Rework BlockHeaderDAO.chainTips into two methods: BlockHeaderDAO.{get… (#2443)

* Rework BlockHeaderDAO.chainTips into two methods: BlockHeaderDAO.{getBestChainTips,getForkedChainTips}. getForkedChainTips is needed for reorg situations in the case a block header is received that builds off a stale tip * Deduplicate BlockHeaderDAO.getBlockchains() that are subchains for the best chains
2025-01-18 13:24:25 +01:00 · 2021-01-01 09:04:29 -06:00 · 2021-01-01 09:04:29 -06:00 · b2560c4606
commit b2560c4606
parent 4a41bacaa0
6 changed files with 222 additions and 31 deletions
--- a/chain-test/src/test/scala/org/bitcoins/chain/models/BlockHeaderDAOTest.scala
+++ b/chain-test/src/test/scala/org/bitcoins/chain/models/BlockHeaderDAOTest.scala
@ -124,14 +124,14 @@ class BlockHeaderDAOTest extends ChainDbUnitTest {
      }
  }

-  it must "retrieve the chain tip saved in the database" in {
+  it must "retrieve the best chain tip saved in the database" in {
    blockHeaderDAO: BlockHeaderDAO =>
      val blockHeader = BlockHeaderHelper.buildNextHeader(genesisHeaderDb)

      val createdF = blockHeaderDAO.create(blockHeader)

      val chainTip1F = createdF.flatMap { _ =>
-        blockHeaderDAO.chainTips
+        blockHeaderDAO.getBestChainTips
      }

      val assert1F = chainTip1F.map { tips =>
@ -144,24 +144,70 @@ class BlockHeaderDAOTest extends ChainDbUnitTest {
      //insert another header and make sure that is the new last header
      assert1F.flatMap { _ =>
        val created2F = blockHeaderDAO.create(blockHeader2)
-        val chainTip2F = created2F.flatMap(_ => blockHeaderDAO.chainTips)
+        val chainTip2F = created2F.flatMap(_ => blockHeaderDAO.getBestChainTips)

        chainTip2F.map { tips =>
          assert(tips.length == 1)
          assert(tips.head.blockHeader.hash == blockHeader2.blockHeader.hash)
        }
      }
-
  }

  it must "return the genesis block when retrieving block headers from an empty database" in {
    blockHeaderDAO: BlockHeaderDAO =>
-      val chainTipsF = blockHeaderDAO.chainTips
+      val chainTipsF = blockHeaderDAO.getBestChainTips
      chainTipsF.map { tips =>
        assert(tips.headOption == Some(genesisHeaderDb))
      }
  }

+  it must "retrieve all chainTips in the last difficulty interval, not just the heaviest chain tip" in {
+    blockHeaderDAO: BlockHeaderDAO =>
+      val reorgFixtureF = buildBlockHeaderDAOCompetingHeaders(blockHeaderDAO)
+
+      //now we have 2 competing tips, chainTips should return both competing headers
+      val firstAssertionF = for {
+        reorgFixture <- reorgFixtureF
+        headerDb1 = reorgFixture.headerDb1
+        headerDb2 = reorgFixture.headerDb2
+        chainTips <- blockHeaderDAO.getForkedChainTips
+      } yield {
+        assert(chainTips.length == 2)
+        assert(chainTips.contains(headerDb1))
+        assert(chainTips.contains(headerDb2))
+      }
+
+      //ok, now we are going to build a new header off of headerDb1
+      //however, headerDb2 is _still_ a possible chainTip that we can reorg
+      //too. So we should still have both of them returned
+      for {
+        _ <- firstAssertionF
+        reorgFixture <- reorgFixtureF
+        headerD = BlockHeaderHelper.buildNextHeader(reorgFixture.headerDb1)
+        _ <- reorgFixture.blockHeaderDAO.create(headerD)
+        chainTips <- blockHeaderDAO.getForkedChainTips
+      } yield {
+        assert(chainTips.length == 2)
+        assert(chainTips.contains(reorgFixture.headerDb1))
+        assert(chainTips.contains(reorgFixture.headerDb2))
+      }
+  }
+
+  it must "deduplicate blockchains so in reorg situations we do not return duplicates" in {
+    blockHeaderDAO: BlockHeaderDAO =>
+      val reorgFixtureF = buildBlockHeaderDAOCompetingHeaders(blockHeaderDAO)
+
+      //now we have 2 competing tips, so we should return 2 chains
+      val firstAssertionF = for {
+        _ <- reorgFixtureF
+        chains <- blockHeaderDAO.getBlockchains()
+      } yield {
+        assert(chains.length == 2)
+      }
+
+      firstAssertionF
+  }
+
  it must "retrieve a block header by height" in {
    blockHeaderDAO: BlockHeaderDAO =>
      val blockHeader = BlockHeaderHelper.buildNextHeader(genesisHeaderDb)
@ -331,7 +377,7 @@ class BlockHeaderDAOTest extends ChainDbUnitTest {

    for {
      _ <- blockerHeaderDAO.createAll(Vector(db1, db2))
-      tips <- blockerHeaderDAO.chainTips
+      tips <- blockerHeaderDAO.getBestChainTips
    } yield assert(tips == Vector(db2))
  }

@ -347,10 +393,20 @@ class BlockHeaderDAOTest extends ChainDbUnitTest {

      for {
        _ <- blockerHeaderDAO.create(db)
-        tips <- blockerHeaderDAO.chainTips
+        tips <- blockerHeaderDAO.getBestChainTips
      } yield assert(tips == Vector(db))
  }

+  it must "get blockchains from the genesis header" in {
+    blockHeaderDAO: BlockHeaderDAO =>
+      val blockchainsF = blockHeaderDAO.getBlockchains()
+      for {
+        blockchains <- blockchainsF
+      } yield {
+        assert(blockchains.length == 1)
+      }
+  }
+
  it must "successfully getBlockchainsBetweenHeights" in {
    blockerHeaderDAO: BlockHeaderDAO =>
      val duplicate3 = BlockHeader(
--- a/chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala
+++ b/chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala
@ -87,7 +87,7 @@ class ChainHandler(
  }

  override def getBestBlockHeader(): Future[BlockHeaderDb] = {
-    val tipsF: Future[Vector[BlockHeaderDb]] = blockHeaderDAO.chainTips
+    val tipsF: Future[Vector[BlockHeaderDb]] = blockHeaderDAO.getBestChainTips
    for {
      tips <- tipsF
      chains = tips.map(t => Blockchain.fromHeaders(Vector(t)))
@ -172,11 +172,13 @@ class ChainHandler(
  override def processHeaders(
      headers: Vector[BlockHeader]): Future[ChainApi] = {
    val blockchainsF = blockHeaderDAO.getBlockchains()
-    for {
+    val resultF = for {
      blockchains <- blockchainsF
      newChainApi <-
        processHeadersWithChains(headers = headers, blockchains = blockchains)
    } yield newChainApi
+
+    resultF
  }

  /**
@ -673,7 +675,7 @@ class ChainHandler(
      case None => FutureUtil.none
      case Some(blockHeight) =>
        for {
-          tips <- blockHeaderDAO.chainTips
+          tips <- blockHeaderDAO.getBestChainTips
          getNAncestorsFs = tips.map { tip =>
            blockHeaderDAO.getNAncestors(tip.hashBE, tip.height - blockHeight)
          }
--- a/chain/src/main/scala/org/bitcoins/chain/blockchain/sync/ChainSync.scala
+++ b/chain/src/main/scala/org/bitcoins/chain/blockchain/sync/ChainSync.scala
@ -26,7 +26,7 @@ abstract class ChainSync extends ChainVerificationLogger {
      getBestBlockHashFunc: () => Future[DoubleSha256DigestBE])(implicit
      ec: ExecutionContext): Future[ChainApi] = {
    val currentTipsF: Future[Vector[BlockHeaderDb]] = {
-      chainHandler.blockHeaderDAO.chainTips
+      chainHandler.blockHeaderDAO.getBestChainTips
    }

    //TODO: We are implicitly trusting whatever
--- a/chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDAO.scala
+++ b/chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDAO.scala
@ -269,7 +269,7 @@ case class BlockHeaderDAO()(implicit

  /** Returns the block height of the block with the most work from our database */
  def bestHeight: Future[Int] = {
-    chainTips.map { tips =>
+    getBestChainTips.map { tips =>
      tips.maxByOption(_.chainWork).map(_.height).getOrElse(0)
    }
  }
@ -301,8 +301,30 @@ case class BlockHeaderDAO()(implicit
    safeDatabase.runVec(aggregate)
  }

-  def chainTips: Future[Vector[BlockHeaderDb]] = {
-    logger.debug(s"Getting chain tips from database")
+  /** Retrieves chain tips my finding duplicates at a certain height
+    * within a given range. This indicates there was a contentious chain tip
+    * at some point.
+    *
+    * It's important to note that this query will NOT return the current best chain tip
+    * unless that current chain tips is contentious
+    *
+    * @param lowestHeight the height we will look backwards until. This is inclusive
+    */
+  private def forkedChainTips(
+      lowestHeight: Int): Future[Vector[BlockHeaderDb]] = {
+    val headersQ = table.filter(_.height >= lowestHeight)
+    val headersF = safeDatabase.runVec(headersQ.result)
+    for {
+      headers <- headersF
+      byHeight = headers.groupBy(_.height)
+      //now find instances where we have duplicate headers at a given height
+      //this indicates there was at one point a fork
+      forks = byHeight.filter(_._2.length > 1)
+    } yield forks.flatMap(_._2).toVector
+  }
+
+  /** Returns the block header with the most accumulated work */
+  def getBestChainTips: Future[Vector[BlockHeaderDb]] = {
    val aggregate = {
      maxWorkQuery.flatMap { work =>
        logger.debug(s"Max block work: $work")
@ -314,7 +336,39 @@ case class BlockHeaderDAO()(implicit
      }
    }

-    safeDatabase.runVec(aggregate)
+    safeDatabase
+      .runVec(aggregate)
+  }
+
+  /** Retrieves all possible chainTips from the database. Note this does NOT retrieve
+    * the BEST chain tips. If you need those please call [[getBestChainTips]]. This method
+    * will search backwards [[appConfig.chain.difficultyChangeInterval]] blocks looking
+    * for all forks that we have in our chainstate.
+    *
+    * We will then return all conflicting headers.
+    *
+    * Note:
+    * This method does NOT try and remove headers that are in the best chain. This means
+    * half the returned headers from this method will be in the best chain. To figure out
+    * which headers are in the best chain, you will need to walk backwards from [[getBestChainTips]]
+    * figuring out which headers are a part of the best chain.
+    */
+  def getForkedChainTips: Future[Vector[BlockHeaderDb]] = {
+    val mHeight = maxHeight
+    val lowestHeightF = mHeight.map { h =>
+      val lowest = h - appConfig.chain.difficultyChangeInterval
+      Math.max(lowest, 0)
+    }
+
+    //what to do about tips that are in the best chain?
+    val tipsF = for {
+      lowestHeight <- lowestHeightF
+      result <- forkedChainTips(lowestHeight)
+    } yield {
+      result
+    }
+
+    tipsF
  }

  /** Returns competing blockchains that are contained in our BlockHeaderDAO
@ -328,13 +382,40 @@ case class BlockHeaderDAO()(implicit
    */
  def getBlockchains()(implicit
      ec: ExecutionContext): Future[Vector[Blockchain]] = {
-    val chainTipsF = chainTips
-    chainTipsF.flatMap { tips =>
+    val chainTipsF = getForkedChainTips
+    val bestTipF = getBestChainTips
+    val staleChainsF = chainTipsF.flatMap { tips =>
      val nestedFuture: Vector[Future[Option[Blockchain]]] = tips.map { tip =>
        getBlockchainFrom(tip)
      }
      Future.sequence(nestedFuture).map(_.flatten)
    }
+
+    val bestChainsF = bestTipF.flatMap { tips =>
+      val nestedFuture: Vector[Future[Option[Blockchain]]] = tips.map { tip =>
+        getBlockchainFrom(tip)
+      }
+      Future.sequence(nestedFuture).map(_.flatten)
+    }
+
+    for {
+      staleChains <- staleChainsF
+      bestChains <- bestChainsF
+      //we need to check the stale chains tips to see if it is contained
+      //in our best chains. If it is, that means the stale chain
+      //is a subchain of a best chain. We need to discard it if
+      //if that is the case to avoid duplicates
+      filtered = staleChains.filterNot { c =>
+        bestChains.exists { best =>
+          best.findAtHeight(c.tip.height) match {
+            case Some(h) => h == c.tip
+            case None    => false
+          }
+        }
+      }
+    } yield {
+      bestChains ++ filtered
+    }
  }

  /** Retrieves a blockchain with the best tip being the given header */
--- a/testkit/src/main/scala/org/bitcoins/testkit/chain/ChainUnitTest.scala
+++ b/testkit/src/main/scala/org/bitcoins/testkit/chain/ChainUnitTest.scala
@ -20,10 +20,13 @@ import org.bitcoins.rpc.client.common.{BitcoindRpcClient, BitcoindVersion}
 import org.bitcoins.rpc.client.v19.BitcoindV19RpcClient
 import org.bitcoins.testkit.chain.ChainUnitTest.createChainHandler
 import org.bitcoins.testkit.chain.fixture._
+import org.bitcoins.testkit.chain.models.{
+  ReorgFixtureBlockHeaderDAO,
+  ReorgFixtureChainApi
+}
 import org.bitcoins.testkit.fixtures.BitcoinSFixture
 import org.bitcoins.testkit.node.CachedChainAppConfig
 import org.bitcoins.testkit.rpc.BitcoindRpcTestUtil
-
 import org.bitcoins.testkit.util.ScalaTestUtil
 import org.bitcoins.testkit.{chain, BitcoinSTestAppConfig}
 import org.bitcoins.zmq.ZMQSubscriber
@ -388,26 +391,39 @@ trait ChainUnitTest
    }
  }

-  case class ReorgFixture(
-      chainApi: ChainApi,
-      headerDb1: BlockHeaderDb,
-      headerDb2: BlockHeaderDb,
-      oldBestBlockHeader: BlockHeaderDb) {
-    lazy val header1: BlockHeader = headerDb1.blockHeader
-    lazy val header2: BlockHeader = headerDb2.blockHeader
-  }
-
  /** Builds two competing headers off of the [[ChainHandler.getBestBlockHash best chain tip]] */
  def buildChainHandlerCompetingHeaders(
-      chainHandler: ChainHandler): Future[ReorgFixture] = {
+      chainHandler: ChainHandler): Future[ReorgFixtureChainApi] = {
    for {
      oldBestTip <- chainHandler.getBestBlockHeader()
      (newHeaderB, newHeaderC) = buildCompetingHeaders(oldBestTip)
-      newChainApi <- chainHandler.processHeaders(Vector(newHeaderB, newHeaderC))
+      newChainApi <- chainHandler.processHeader(newHeaderB)
+      newChainApi2 <- newChainApi.processHeader(newHeaderC)
      newHeaderDbB <- newChainApi.getHeader(newHeaderB.hashBE)
-      newHeaderDbC <- newChainApi.getHeader(newHeaderC.hashBE)
+      newHeaderDbC <- newChainApi2.getHeader(newHeaderC.hashBE)
    } yield {
-      ReorgFixture(newChainApi, newHeaderDbB.get, newHeaderDbC.get, oldBestTip)
+      ReorgFixtureChainApi(newChainApi2,
+                           newHeaderDbB.get,
+                           newHeaderDbC.get,
+                           oldBestTip)
+    }
+  }
+
+  /** Builds two competing headers off of [[BlockHeaderDAO.chainTips]]. */
+  def buildBlockHeaderDAOCompetingHeaders(
+      blockHeaderDAO: BlockHeaderDAO): Future[ReorgFixtureBlockHeaderDAO] = {
+    val handler = ChainHandler.fromDatabase(blockHeaderDAO,
+                                            CompactFilterHeaderDAO(),
+                                            CompactFilterDAO())
+    val chainFixtureF = buildChainHandlerCompetingHeaders(handler)
+    for {
+      chainFixture <- chainFixtureF
+    } yield {
+      ReorgFixtureBlockHeaderDAO(blockHeaderDAO = blockHeaderDAO,
+                                 headerDb1 = chainFixture.headerDb1,
+                                 headerDb2 = chainFixture.headerDb2,
+                                 oldBestBlockHeader =
+                                   chainFixture.oldBestBlockHeader)
    }
  }
 }
--- a/testkit/src/main/scala/org/bitcoins/testkit/chain/models/ReorgFixture.scala
+++ b/testkit/src/main/scala/org/bitcoins/testkit/chain/models/ReorgFixture.scala
@ -0,0 +1,36 @@
+package org.bitcoins.testkit.chain.models
+
+import org.bitcoins.chain.models.BlockHeaderDAO
+import org.bitcoins.core.api.chain.ChainApi
+import org.bitcoins.core.api.chain.db.BlockHeaderDb
+import org.bitcoins.core.protocol.blockchain.BlockHeader
+
+/** A trait that contains a reorg scenario that looks like this
+  *                       headerDb1
+  *                    /
+  * oldBestBlockHeader
+  *                    \
+  *                      headerDb2
+  */
+sealed trait ReorgFixture {
+  def headerDb1: BlockHeaderDb
+  def headerDb2: BlockHeaderDb
+  def oldBestBlockHeader: BlockHeaderDb
+
+  lazy val header1: BlockHeader = headerDb1.blockHeader
+  lazy val header2: BlockHeader = headerDb2.blockHeader
+}
+
+case class ReorgFixtureChainApi(
+    chainApi: ChainApi,
+    headerDb1: BlockHeaderDb,
+    headerDb2: BlockHeaderDb,
+    oldBestBlockHeader: BlockHeaderDb)
+    extends ReorgFixture
+
+case class ReorgFixtureBlockHeaderDAO(
+    blockHeaderDAO: BlockHeaderDAO,
+    headerDb1: BlockHeaderDb,
+    headerDb2: BlockHeaderDb,
+    oldBestBlockHeader: BlockHeaderDb)
+    extends ReorgFixture