Implement best block hash correctly (#1452)

* Implement best block hash correctly

* Handle no common history

* Fix getBlockProof to be BigInt instead of UInt32

* Fix warnings

* Make get best filter use chainwork

* Remove unused function, test isMissingChainWork

* Fix comparisons for chain work in sql

* Fix migrations from rebase

* Fix CI error

* Fix postgresql

bitcoind-rpc-test/src/test/scala/org/bitcoins/rpc/v18/BitcoindV18RpcClientTest.scala

@@ -100,7 +100,7 @@ class BitcoindV18RpcClientTest extends BitcoindRpcTest {
   it should "successfully submit a header" in {
     val genesisHeader = RegTestNetChainParams.genesisBlock.blockHeader
     val genesisHeaderDb =
-      BlockHeaderDbHelper.fromBlockHeader(height = 1, genesisHeader)
+      BlockHeaderDbHelper.fromBlockHeader(height = 1, BigInt(0), genesisHeader)
     val nextHeader = BlockHeaderHelper.buildNextHeader(genesisHeaderDb)
     clientF.flatMap(client =>
       client.submitHeader(nextHeader.blockHeader).map(_ => succeed))

chain-test/src/test/scala/org/bitcoins/chain/blockchain/ChainHandlerTest.scala

@@ -4,6 +4,7 @@ import akka.actor.ActorSystem
 import org.bitcoins.chain.api.ChainApi
 import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.chain.models.{BlockHeaderDb, BlockHeaderDbHelper}
+import org.bitcoins.chain.pow.Pow
 import org.bitcoins.core.gcs.{BlockFilter, FilterHeader}
 import org.bitcoins.core.number.{Int32, UInt32}
 import org.bitcoins.core.p2p.CompactFilterMessage
@@ -26,6 +27,7 @@ import org.bitcoins.testkit.util.{FileUtil, ScalaTestUtil}
 import org.scalatest.{Assertion, FutureOutcome}
 import play.api.libs.json.Json
 
+import scala.annotation.tailrec
 import scala.concurrent.Future
 import scala.io.BufferedSource
 
@@ -78,6 +80,37 @@ class ChainHandlerTest extends ChainDbUnitTest {
       foundHeaderF.map(found => assert(found.get == newValidHeader))
   }
 
+  it must "have getBestBlockHash return the header with the most work, not the highest" in {
+    tempHandler: ChainHandler =>
+      val dummyHeader =
+        BlockHeaderDbHelper.fromBlockHeader(1,
+                                            BigInt(0),
+                                            ChainTestUtil.blockHeader562462)
+
+      val highestHeader =
+        BlockHeaderDbHelper.fromBlockHeader(2,
+                                            BigInt(0),
+                                            ChainTestUtil.blockHeader562463)
+
+      val headerWithMostWork =
+        BlockHeaderDbHelper.fromBlockHeader(1,
+                                            BigInt(1000),
+                                            ChainTestUtil.blockHeader562464)
+
+      val tallestBlockchain =
+        Blockchain(Vector(highestHeader, dummyHeader, genesis))
+      val mostWorkChain = Blockchain(Vector(headerWithMostWork, genesis))
+
+      val chainHandler =
+        tempHandler.copy(blockchains = Vector(tallestBlockchain, mostWorkChain))
+
+      for {
+        hash <- chainHandler.getBestBlockHash()
+      } yield {
+        assert(hash == headerWithMostWork.blockHeader.hashBE)
+      }
+  }
+
   it must "have an in-order seed" in { _ =>
     val source = FileUtil.getFileAsSource("block_headers.json")
     val arrStr = source.getLines.next
@@ -108,14 +141,17 @@ class ChainHandlerTest extends ChainDbUnitTest {
 
       val firstBlockHeaderDb =
         BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 2,
+                                            BigInt(0),
                                             ChainTestUtil.blockHeader562462)
 
       val secondBlockHeaderDb =
         BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 1,
+                                            BigInt(0),
                                             ChainTestUtil.blockHeader562463)
 
       val thirdBlockHeaderDb =
         BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE,
+                                            BigInt(0),
                                             ChainTestUtil.blockHeader562464)
 
       /*
@@ -160,16 +196,26 @@ class ChainHandlerTest extends ChainDbUnitTest {
           ChainUnitTest.FIRST_POW_CHANGE - ChainUnitTest.FIRST_BLOCK_HEIGHT)
 
       val firstBlockHeaderDb =
-        BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 2,
+        BlockHeaderDbHelper.fromBlockHeader(
-                                            ChainTestUtil.blockHeader562462)
+          ChainUnitTest.FIRST_POW_CHANGE - 2,
+          Pow.getBlockProof(ChainTestUtil.blockHeader562462),
+          ChainTestUtil.blockHeader562462)
 
-      val secondBlockHeaderDb =
+      val secondBlockHeaderDb = {
+        val chainWork = firstBlockHeaderDb.chainWork + Pow.getBlockProof(
+          ChainTestUtil.blockHeader562463)
         BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 1,
+                                            chainWork,
                                             ChainTestUtil.blockHeader562463)
+      }
 
-      val thirdBlockHeaderDb =
+      val thirdBlockHeaderDb = {
+        val chainWork = secondBlockHeaderDb.chainWork + Pow.getBlockProof(
+          ChainTestUtil.blockHeader562464)
         BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE,
+                                            chainWork,
                                             ChainTestUtil.blockHeader562464)
+      }
 
       /*
        * We need to insert one block before the first POW check because it is used on the next
@@ -460,33 +506,72 @@ class ChainHandlerTest extends ChainDbUnitTest {
       }
   }
 
+  it must "properly recalculate chain work" in { tempHandler: ChainHandler =>
+    val headersWithNoWork = Vector(
+      BlockHeaderDbHelper.fromBlockHeader(3,
+                                          BigInt(0),
+                                          ChainTestUtil.blockHeader562464),
+      BlockHeaderDbHelper.fromBlockHeader(2,
+                                          BigInt(0),
+                                          ChainTestUtil.blockHeader562463),
+      BlockHeaderDbHelper.fromBlockHeader(1,
+                                          BigInt(0),
+                                          ChainTestUtil.blockHeader562462)
+    )
+
+    val blockchain = Blockchain(headersWithNoWork :+ genesis)
+
+    val chainHandler = tempHandler.copy(blockchains = Vector(blockchain))
+
+    for {
+      _ <- chainHandler.blockHeaderDAO.createAll(headersWithNoWork)
+      isMissingWork <- chainHandler.isMissingChainWork
+      _ = assert(isMissingWork)
+      newHandler <- chainHandler.recalculateChainWork
+      headerDb <- newHandler.getBestBlockHeader()
+    } yield {
+      assert(headerDb.height == headersWithNoWork.head.height)
+      assert(headerDb.hashBE == headersWithNoWork.head.hashBE)
+      assert(headerDb.chainWork == BigInt(12885098501L))
+    }
+  }
+
   final def processHeaders(
       processorF: Future[ChainApi],
       headers: Vector[BlockHeader],
       height: Int): Future[Assertion] = {
     val processedHeadersF = processorF.flatMap(_.processHeaders(headers))
 
+    @tailrec
     def loop(
         remainingHeaders: Vector[BlockHeader],
+        prevHeaderDbOpt: Option[BlockHeaderDb],
         height: Int,
         accum: Vector[Future[Assertion]]): Vector[Future[Assertion]] = {
       remainingHeaders match {
         case header +: headersTail =>
           val getHeaderF = processedHeadersF.flatMap(_.getHeader(header.hashBE))
+
+          val chainWork = prevHeaderDbOpt match {
+            case None => Pow.getBlockProof(header)
+            case Some(prevHeader) =>
+              prevHeader.chainWork + Pow.getBlockProof(header)
+          }
+
           val expectedBlockHeaderDb =
-            BlockHeaderDbHelper.fromBlockHeader(height, header)
+            BlockHeaderDbHelper.fromBlockHeader(height, chainWork, header)
           val assertionF =
             getHeaderF.map(headerOpt =>
               assert(headerOpt.contains(expectedBlockHeaderDb)))
           val newAccum = accum.:+(assertionF)
-          loop(headersTail, height + 1, newAccum)
+          loop(headersTail, Some(expectedBlockHeaderDb), height + 1, newAccum)
         case Vector() =>
           accum
       }
     }
 
     val vecFutAssert: Vector[Future[Assertion]] =
-      loop(headers, height, Vector.empty)
+      loop(headers, None, height, Vector.empty)
 
     ScalaTestUtil.toAssertF(vecFutAssert)
   }

chain-test/src/test/scala/org/bitcoins/chain/models/BlockHeaderDAOTest.scala

@@ -10,6 +10,7 @@ import org.bitcoins.testkit.chain.{
   ChainUnitTest
 }
 import org.scalatest.FutureOutcome
+import scodec.bits._
 
 import scala.concurrent.Future
 

chain-test/src/test/scala/org/bitcoins/chain/pow/BitcoinPowTest.scala

@@ -3,7 +3,10 @@ package org.bitcoins.chain.pow
 import akka.actor.ActorSystem
 import org.bitcoins.chain.blockchain.Blockchain
 import org.bitcoins.chain.config.ChainAppConfig
-import org.bitcoins.core.protocol.blockchain.MainNetChainParams
+import org.bitcoins.core.protocol.blockchain.{
+  MainNetChainParams,
+  TestNetChainParams
+}
 import org.bitcoins.testkit.chain.fixture.{ChainFixture, ChainFixtureTag}
 import org.bitcoins.testkit.chain.{
   ChainDbUnitTest,
@@ -78,4 +81,24 @@ class BitcoinPowTest extends ChainDbUnitTest {
 
       seqF.map(_ => succeed)
   }
+
+  it must "getBlockProof correctly for the testnet genesis block" inFixtured {
+    case ChainFixture.Empty =>
+      Future {
+        val header = TestNetChainParams.genesisBlock.blockHeader
+        val proof = Pow.getBlockProof(header)
+
+        assert(proof == BigInt(4295032833L))
+      }
+  }
+
+  it must "getBlockProof correctly for the mainnet genesis block" inFixtured {
+    case ChainFixture.Empty =>
+      Future {
+        val header = MainNetChainParams.genesisBlock.blockHeader
+        val proof = Pow.getBlockProof(header)
+
+        assert(proof == BigInt(4295032833L))
+      }
+  }
 }

chain-test/src/test/scala/org/bitcoins/chain/validation/TipValidationTest.scala

@@ -1,14 +1,15 @@
 package org.bitcoins.chain.validation
 
 import akka.actor.ActorSystem
-import org.bitcoins.chain.blockchain.Blockchain
-import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDbHelper}
+import org.bitcoins.chain.pow.Pow
 import org.bitcoins.core.protocol.blockchain.BlockHeader
 import org.bitcoins.testkit.chain.{BlockHeaderHelper, ChainDbUnitTest}
 import org.scalatest.{Assertion, FutureOutcome}
 
 class TipValidationTest extends ChainDbUnitTest {
+  import org.bitcoins.chain.blockchain.Blockchain
+  import org.bitcoins.chain.config.ChainAppConfig
 
   override type FixtureParam = BlockHeaderDAO
 
@@ -29,7 +30,10 @@ class TipValidationTest extends ChainDbUnitTest {
 
   it must "connect two blocks with that are valid" in { bhDAO =>
     val newValidTipDb =
-      BlockHeaderDbHelper.fromBlockHeader(566093, newValidTip)
+      BlockHeaderDbHelper.fromBlockHeader(
+        566093,
+        currentTipDb.chainWork + Pow.getBlockProof(newValidTip),
+        newValidTip)
     val expected = TipUpdateResult.Success(newValidTipDb)
 
     runTest(newValidTip, expected, blockchain)

chain/src/main/resources/postgresql/chain/migration/V2__add_block_header_chain_work.sql

@@ -0,0 +1 @@
+ALTER TABLE block_headers ADD COLUMN chain_work numeric(78,0) NOT NULL DEFAULT 0;

chain/src/main/resources/sqlite/chain/migration/V2__add_block_header_chain_work.sql

@@ -0,0 +1 @@
+ALTER TABLE "block_headers" ADD COLUMN "chain_work" VARBINARY(32) NOT NULL DEFAULT "00";

chain/src/main/scala/org/bitcoins/chain/blockchain/BaseBlockChain.scala

@@ -39,7 +39,7 @@ private[blockchain] trait BaseBlockChain extends SeqWrapper[BlockHeaderDb] {
   protected[blockchain] def compObjectfromHeaders(
       headers: scala.collection.immutable.Seq[BlockHeaderDb]): Blockchain
 
-  val tip: BlockHeaderDb = headers.head
+  val tip: BlockHeaderDb = headers.maxBy(_.height)
 
   /** The height of the chain */
   val height: Int = tip.height

chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala

@@ -4,6 +4,7 @@ import org.bitcoins.chain.ChainVerificationLogger
 import org.bitcoins.chain.api.ChainApi
 import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.chain.models._
+import org.bitcoins.chain.pow.Pow
 import org.bitcoins.core.api.ChainQueryApi.FilterResponse
 import org.bitcoins.core.gcs.FilterHeader
 import org.bitcoins.core.number.UInt32
@@ -45,22 +46,33 @@ case class ChainHandler(
   /** @inheritdoc */
   override def getBlockCount(): Future[Int] = {
     logger.debug(s"Querying for block count")
-    blockHeaderDAO.maxHeight.map { height =>
+    blockHeaderDAO.bestHeight.map { height =>
       logger.debug(s"getBlockCount result: count=$height")
       height
     }
   }
 
   override def getBestBlockHeader(): Future[BlockHeaderDb] = {
-    for {
+    logger.debug(s"Querying for best block hash")
-      hash <- getBestBlockHash()
+    //https://bitcoin.org/en/glossary/block-chain
-      headerOpt <- getHeader(hash)
+    val groupedChains = blockchains.groupBy(_.tip.chainWork)
-    } yield headerOpt match {
+    val maxWork = groupedChains.keys.max
-      case None =>
+    val chains = groupedChains(maxWork)
-        throw new RuntimeException(
+
-          s"We found best hash=${hash.hex} but could not retrieve the full header!!!")
+    val bestHeader: BlockHeaderDb = chains match {
-      case Some(header) => header
+      case Vector() =>
+        // This should never happen
+        val errMsg = s"Did not find blockchain with work $maxWork"
+        logger.error(errMsg)
+        throw new RuntimeException(errMsg)
+      case chain +: Vector() =>
+        chain.tip
+      case chain +: rest =>
+        logger.warn(
+          s"We have multiple competing blockchains: ${(chain +: rest).map(_.tip.hashBE.hex).mkString(", ")}")
+        chain.tip
     }
+    Future.successful(bestHeader)
   }
 
   /** @inheritdoc */
@@ -110,28 +122,7 @@ case class ChainHandler(
     * @inheritdoc
     */
   override def getBestBlockHash(): Future[DoubleSha256DigestBE] = {
-    logger.debug(s"Querying for best block hash")
+    getBestBlockHeader().map(_.hashBE)
-    //naive implementation, this is looking for the tip with the _most_ proof of work
-    //this does _not_ mean that it is on the chain that has the most work
-    //TODO: Enhance this in the future to return the "heaviest" header
-    //https://bitcoin.org/en/glossary/block-chain
-    val groupedChains = blockchains.groupBy(_.tip.height)
-    val maxHeight = groupedChains.keys.max
-    val chains = groupedChains(maxHeight)
-
-    val hashBE: DoubleSha256DigestBE = chains match {
-      case Vector() =>
-        val errMsg = s"Did not find blockchain with height $maxHeight"
-        logger.error(errMsg)
-        throw new RuntimeException(errMsg)
-      case chain +: Vector() =>
-        chain.tip.hashBE
-      case chain +: rest =>
-        logger.warn(
-          s"We have multiple competing blockchains: ${(chain +: rest).map(_.tip.hashBE.hex).mkString(", ")}")
-        chain.tip.hashBE
-    }
-    Future.successful(hashBE)
   }
 
   /** @inheritdoc */
@@ -147,7 +138,7 @@ case class ChainHandler(
           throw UnknownBlockHash(s"Unknown block hash ${prevStopHash}"))
       } yield prevStopHeader.height + 1
     }
-    val blockCountF = getBlockCount
+    val blockCountF = getBlockCount()
     for {
       startHeight <- startHeightF
       blockCount <- blockCountF
@@ -361,8 +352,9 @@ case class ChainHandler(
   /** @inheritdoc */
   override def getFilterHeaderCount: Future[Int] = {
     logger.debug(s"Querying for filter header count")
-    filterHeaderDAO.maxHeight.map { height =>
+    filterHeaderDAO.getBestFilter.map { filterHeader =>
-      logger.debug(s"getFilterHeaderCount result: count=$height")
+      val height = filterHeader.height
+      logger.debug(s"getFilterCount result: count=$height")
       height
     }
   }
@@ -374,28 +366,7 @@ case class ChainHandler(
 
   /** @inheritdoc */
   override def getBestFilterHeader(): Future[CompactFilterHeaderDb] = {
-    //this seems realy brittle, is there a guarantee
+    filterHeaderDAO.getBestFilter
-    //that the highest filter header count is our best filter header?
-    val filterCountF = getFilterHeaderCount()
-    val ourBestFilterHeader = for {
-      count <- filterCountF
-      filterHeader <- getFilterHeadersAtHeight(count)
-    } yield {
-      //TODO: Figure out what the best way to select
-      //the best filter header is if we have competing
-      //chains. (Same thing applies to getBestBlockHash()
-      //for now, just do the dumb thing and pick the first one
-      filterHeader match {
-        case tip1 +: _ +: _ =>
-          tip1
-        case tip +: _ =>
-          tip
-        case Vector() =>
-          sys.error(s"No filter headers found in database!")
-      }
-    }
-
-    ourBestFilterHeader
   }
 
   /** @inheritdoc */
@@ -406,7 +377,8 @@ case class ChainHandler(
   /** @inheritdoc */
   override def getFilterCount: Future[Int] = {
     logger.debug(s"Querying for filter count")
-    filterDAO.maxHeight.map { height =>
+    filterDAO.getBestFilter.map { filter =>
+      val height = filter.height
       logger.debug(s"getFilterCount result: count=$height")
       height
     }
@@ -486,6 +458,101 @@ case class ChainHandler(
     }
     loop(Future.successful(to), Vector.empty)
   }
+
+  def isMissingChainWork: Future[Boolean] = {
+    for {
+      first100 <- blockHeaderDAO.getBetweenHeights(0, 100)
+      first100MissingWork = first100.nonEmpty && first100.exists(
+        _.chainWork == BigInt(0))
+      isMissingWork <- {
+        if (first100MissingWork) {
+          Future.successful(true)
+        } else {
+          for {
+            height <- getBestHashBlockHeight()
+            last100 <- blockHeaderDAO.getBetweenHeights(height - 100, height)
+            last100MissingWork = last100.nonEmpty && last100.exists(
+              _.chainWork == BigInt(0))
+          } yield last100MissingWork
+        }
+      }
+
+    } yield isMissingWork
+  }
+
+  def recalculateChainWork: Future[ChainHandler] = {
+    logger.info("Calculating chain work for previous blocks")
+
+    val batchSize = chainConfig.chain.difficultyChangeInterval
+    def loop(
+        currentChainWork: BigInt,
+        remainingHeaders: Vector[BlockHeaderDb],
+        accum: Vector[BlockHeaderDb]): Future[Vector[BlockHeaderDb]] = {
+      if (remainingHeaders.isEmpty) {
+        blockHeaderDAO.upsertAll(accum.takeRight(batchSize))
+      } else {
+        val header = remainingHeaders.head
+
+        val newChainWork = currentChainWork + Pow.getBlockProof(
+          header.blockHeader)
+        val newHeader = header.copy(chainWork = newChainWork)
+
+        // Add the last batch to the database and create log
+        if (header.height % batchSize == 0) {
+          logger.info(
+            s"Recalculating chain work... current height: ${header.height}")
+          val updated = accum :+ newHeader
+          // updated the latest batch
+          blockHeaderDAO
+            .upsertAll(updated.takeRight(batchSize))
+            .flatMap(
+              _ =>
+                loop(
+                  newChainWork,
+                  remainingHeaders.tail,
+                  updated.takeRight(batchSize)
+                ))
+        } else {
+          loop(newChainWork, remainingHeaders.tail, accum :+ newHeader)
+        }
+      }
+    }
+
+    for {
+      tips <- blockHeaderDAO.chainTipsByHeight
+      fullChains <- FutureUtil.sequentially(tips)(tip =>
+        blockHeaderDAO.getFullBlockchainFrom(tip))
+
+      commonHistory = fullChains.foldLeft(fullChains.head.headers)(
+        (accum, chain) => chain.intersect(accum).toVector)
+      sortedHeaders = commonHistory.sortBy(_.height)
+
+      diffedChains = fullChains.map { blockchain =>
+        val sortedFullHeaders = blockchain.headers.sortBy(_.height)
+        // Remove the common blocks
+        sortedFullHeaders.diff(sortedHeaders)
+      }
+
+      commonWithWork <- loop(BigInt(0), sortedHeaders, Vector.empty)
+      finalCommon = Vector(sortedHeaders.head) ++ commonWithWork.takeRight(
+        batchSize)
+      commonChainWork = finalCommon.lastOption
+        .map(_.chainWork)
+        .getOrElse(BigInt(0))
+      newBlockchains <- FutureUtil.sequentially(diffedChains) { blockchain =>
+        loop(commonChainWork, blockchain, Vector.empty)
+          .map { newHeaders =>
+            val relevantHeaders =
+              (finalCommon ++ newHeaders).takeRight(
+                chainConfig.chain.difficultyChangeInterval)
+            Blockchain(relevantHeaders)
+          }
+      }
+    } yield {
+      logger.info("Finished calculating chain work")
+      this.copy(blockchains = newBlockchains.toVector)
+    }
+  }
 }
 
 object ChainHandler {

chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala

@@ -68,6 +68,7 @@ case class ChainAppConfig(
       } else {
         val genesisHeader =
           BlockHeaderDbHelper.fromBlockHeader(height = 0,
+                                              chainWork = BigInt(0),
                                               bh =
                                                 chain.genesisBlock.blockHeader)
         val blockHeaderDAO = BlockHeaderDAO()(ec, appConfig)
@@ -100,9 +101,9 @@ object ChainAppConfig extends AppConfigFactory[ChainAppConfig] {
   /** Constructs a chain verification configuration from the default Bitcoin-S
     * data directory and given list of configuration overrides.
     */
-  override def fromDatadir(datadir: Path, useLogbackConf: Boolean, confs: Vector[Config])(
+  override def fromDatadir(
-      implicit ec: ExecutionContext): ChainAppConfig =
+      datadir: Path,
-    ChainAppConfig(datadir,
+      useLogbackConf: Boolean,
-                   useLogbackConf,
+      confs: Vector[Config])(implicit ec: ExecutionContext): ChainAppConfig =
-                   confs: _*)
+    ChainAppConfig(datadir, useLogbackConf, confs: _*)
 }

chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDAO.scala

@@ -22,7 +22,14 @@ case class BlockHeaderDAO()(
 
   import profile.api._
   private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
-  import mappers._
+  import mappers.{doubleSha256DigestBEMapper, int32Mapper, uInt32Mapper}
+
+  implicit private val bigIntMapper: BaseColumnType[BigInt] =
+    if (appConfig.driverName == "postgresql") {
+      mappers.bigIntPostgresMapper
+    } else {
+      mappers.bigIntMapper
+    }
 
   override val table =
     profile.api.TableQuery[BlockHeaderTable]
@@ -126,6 +133,19 @@ case class BlockHeaderDAO()(
     table.filter(_.height === height).result
   }
 
+  /** Retrieves a [[BlockHeaderDb]] with the given chain work */
+  def getAtChainWork(work: BigInt): Future[Vector[BlockHeaderDb]] = {
+    val query = getAtChainWorkQuery(work)
+    safeDatabase.runVec(query)
+  }
+
+  def getAtChainWorkQuery(work: BigInt): profile.StreamingProfileAction[
+    Seq[BlockHeaderDb],
+    BlockHeaderDb,
+    Effect.Read] = {
+    table.filter(_.chainWork === work).result
+  }
+
   /** Gets Block Headers between (inclusive) start height and stop hash, could be out of order */
   def getBetweenHeightAndHash(
       startHeight: Int,
@@ -223,10 +243,24 @@ case class BlockHeaderDAO()(
     query
   }
 
-  /** Returns the chainTips in our database. This can be multiple headers if we have
+  /** Returns the block height of the block with the most work from our database */
-    * competing blockchains (fork) */
+  def bestHeight: Future[Int] = {
-  def chainTips: Future[Vector[BlockHeaderDb]] = {
+    chainTips.map(_.maxBy(_.chainWork).height)
-    logger.debug(s"Getting chaintips from: ${dbConfig.config}")
+  }
+
+  private val maxWorkQuery: profile.ProfileAction[
+    BigInt,
+    NoStream,
+    Effect.Read] = {
+    val query = table.map(_.chainWork).max.getOrElse(BigInt(0)).result
+    query
+  }
+
+  /** Returns the chainTips in our database calculated by max height, not work.
+    * This should only be used if the chain work has not been calculated
+    */
+  def chainTipsByHeight: Future[Vector[BlockHeaderDb]] = {
+    logger.debug(s"Getting chain tips from: ${dbConfig.config}")
     val aggregate = {
       maxHeightQuery.flatMap { height =>
         logger.debug(s"Max block height: $height")
@@ -241,6 +275,22 @@ case class BlockHeaderDAO()(
     safeDatabase.runVec(aggregate)
   }
 
+  def chainTips: Future[Vector[BlockHeaderDb]] = {
+    logger.debug(s"Getting chain tips from: ${dbConfig.config}")
+    val aggregate = {
+      maxWorkQuery.flatMap { work =>
+        logger.debug(s"Max block work: $work")
+        val atChainWork = getAtChainWorkQuery(work)
+        atChainWork.map { headers =>
+          logger.debug(s"Headers at $work: $headers")
+        }
+        atChainWork
+      }
+    }
+
+    safeDatabase.runVec(aggregate)
+  }
+
   /** Returns competing blockchains that are contained in our BlockHeaderDAO
     * Each chain returns the last [[org.bitcoins.core.protocol.blockchain.ChainParams.difficultyChangeInterval difficutly interval]]
     * block headers as defined by the network we are on. For instance, on bitcoin mainnet this will be 2016 block headers.
@@ -270,6 +320,13 @@ case class BlockHeaderDAO()(
     headersF.map(headers => Blockchain.fromHeaders(headers.reverse))
   }
 
+  /** Retrieves a full blockchain with the best tip being the given header */
+  def getFullBlockchainFrom(header: BlockHeaderDb)(
+      implicit ec: ExecutionContext): Future[Blockchain] = {
+    val headersF = getBetweenHeights(from = 0, to = header.height)
+    headersF.map(headers => Blockchain.fromHeaders(headers.reverse))
+  }
+
   /** Finds a [[org.bitcoins.chain.models.BlockHeaderDb block header]] that satisfies the given predicate, else returns None */
   def find(f: BlockHeaderDb => Boolean)(
       implicit ec: ExecutionContext): Future[Option[BlockHeaderDb]] = {
@@ -310,6 +367,8 @@ case class BlockHeaderDAO()(
 
     def hex = column[String]("hex")
 
+    def chainWork: Rep[BigInt] = column[BigInt]("chain_work")
+
     /** The sql index for searching based on [[height]] */
     def heightIndex = index("block_headers_height_index", height)
 
@@ -324,7 +383,8 @@ case class BlockHeaderDAO()(
        time,
        nBits,
        nonce,
-       hex).<>(BlockHeaderDb.tupled, BlockHeaderDb.unapply)
+       hex,
+       chainWork).<>(BlockHeaderDb.tupled, BlockHeaderDb.unapply)
     }
 
   }

chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDb.scala

@@ -13,7 +13,8 @@ case class BlockHeaderDb(
     time: UInt32,
     nBits: UInt32,
     nonce: UInt32,
-    hex: String) {
+    hex: String,
+    chainWork: BigInt) {
 
   lazy val blockHeader: BlockHeader = {
     val blockHeader = BlockHeader.fromHex(hex)
@@ -30,7 +31,10 @@ case class BlockHeaderDb(
 
 object BlockHeaderDbHelper {
 
-  def fromBlockHeader(height: Int, bh: BlockHeader): BlockHeaderDb = {
+  def fromBlockHeader(
+      height: Int,
+      chainWork: BigInt,
+      bh: BlockHeader): BlockHeaderDb = {
     BlockHeaderDb(
       height = height,
       hashBE = bh.hashBE,
@@ -40,7 +44,8 @@ object BlockHeaderDbHelper {
       nBits = bh.nBits,
       nonce = bh.nonce,
       version = bh.version,
-      hex = bh.hex
+      hex = bh.hex,
+      chainWork = chainWork
     )
   }
 }

chain/src/main/scala/org/bitcoins/chain/models/CompactFilterDAO.scala

@@ -14,8 +14,18 @@ case class CompactFilterDAO()(
     extends CRUD[CompactFilterDb, DoubleSha256DigestBE]
     with SlickUtil[CompactFilterDb, DoubleSha256DigestBE] {
   val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
-  import mappers._
+  import mappers.{
+    byteVectorMapper,
+    doubleSha256DigestBEMapper,
+    filterTypeMapper
+  }
   import profile.api._
+  implicit private val bigIntMapper: BaseColumnType[BigInt] =
+    if (appConfig.driverName == "postgresql") {
+      mappers.bigIntPostgresMapper
+    } else {
+      mappers.bigIntMapper
+    }
 
   class CompactFilterTable(tag: Tag)
       extends Table[CompactFilterDb](tag, "cfilters") {
@@ -43,6 +53,11 @@ case class CompactFilterDAO()(
     TableQuery[CompactFilterTable]
   }
 
+  private lazy val blockHeaderTable: profile.api.TableQuery[
+    BlockHeaderDAO#BlockHeaderTable] = {
+    BlockHeaderDAO().table
+  }
+
   override def createAll(
       filters: Vector[CompactFilterDb]): Future[Vector[CompactFilterDb]] = {
     createAllNoAutoInc(ts = filters, database = safeDatabase)
@@ -109,4 +124,15 @@ case class CompactFilterDAO()(
     Effect.Read] = {
     table.filter(header => header.height >= from && header.height <= to).result
   }
+
+  def getBestFilter: Future[CompactFilterDb] = {
+    val join = table join blockHeaderTable on (_.blockHash === _.hash)
+    val query = join.groupBy(_._1).map {
+      case (filter, headers) =>
+        filter -> headers.map(_._2.chainWork).max
+    }
+    safeDatabase
+      .runVec(query.result)
+      .map(_.maxBy(_._2.getOrElse(BigInt(0)))._1)
+  }
 }

chain/src/main/scala/org/bitcoins/chain/models/CompactFilterHeaderDAO.scala

@@ -13,7 +13,13 @@ case class CompactFilterHeaderDAO()(
     with SlickUtil[CompactFilterHeaderDb, DoubleSha256DigestBE] {
   import profile.api._
   val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
-  import mappers._
+  import mappers.doubleSha256DigestBEMapper
+  implicit private val bigIntMapper: BaseColumnType[BigInt] =
+    if (appConfig.driverName == "postgresql") {
+      mappers.bigIntPostgresMapper
+    } else {
+      mappers.bigIntMapper
+    }
 
   class CompactFilterHeaderTable(tag: Tag)
       extends Table[CompactFilterHeaderDb](tag, "cfheaders") {
@@ -42,6 +48,11 @@ case class CompactFilterHeaderDAO()(
     TableQuery[CompactFilterHeaderTable]
   }
 
+  private lazy val blockHeaderTable: profile.api.TableQuery[
+    BlockHeaderDAO#BlockHeaderTable] = {
+    BlockHeaderDAO().table
+  }
+
   override def createAll(filterHeaders: Vector[CompactFilterHeaderDb]): Future[
     Vector[CompactFilterHeaderDb]] = {
     createAllNoAutoInc(ts = filterHeaders, database = safeDatabase)
@@ -106,4 +117,15 @@ case class CompactFilterHeaderDAO()(
     query
   }
 
+  def getBestFilter: Future[CompactFilterHeaderDb] = {
+    val join = table join blockHeaderTable on (_.blockHash === _.hash)
+    val query = join.groupBy(_._1).map {
+      case (filter, headers) =>
+        filter -> headers.map(_._2.chainWork).max
+    }
+    safeDatabase
+      .runVec(query.result)
+      .map(_.maxBy(_._2.getOrElse(BigInt(0)))._1)
+  }
+
 }

chain/src/main/scala/org/bitcoins/chain/pow/Pow.scala

@@ -123,6 +123,16 @@ sealed abstract class Pow {
       newTarget
     }
   }
+
+  def getBlockProof(header: BlockHeader): BigInt = {
+    val target = NumberUtil.targetExpansion(header.nBits)
+
+    if (target.isNegative || target.isOverflow) {
+      BigInt(0)
+    } else {
+      (BigInt(1) << 256) / (target.difficulty + BigInt(1))
+    }
+  }
 }
 
 object Pow extends Pow

chain/src/main/scala/org/bitcoins/chain/validation/TipValidation.scala

@@ -47,6 +47,7 @@ sealed abstract class TipValidation extends ChainVerificationLogger {
       } else {
         val headerDb = BlockHeaderDbHelper.fromBlockHeader(
           height = currentTip.height + 1,
+          chainWork = currentTip.chainWork + Pow.getBlockProof(newPotentialTip),
           bh = newPotentialTip
         )
         TipUpdateResult.Success(headerDb)

core/src/main/scala/org/bitcoins/core/number/NumberType.scala

@@ -45,6 +45,8 @@ sealed abstract class Number[T <: Number[T]]
   override def *(factor: BigInt): T = apply(underlying * factor)
   override def *(num: T): T = apply(underlying * num.underlying)
 
+  def /(num: T): T = apply(underlying / num.underlying)
+
   override def compare(num: T): Int = underlying compare num.underlying
 
   def <<(num: Int): T = this.<<(apply(num))

db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala

@@ -48,7 +48,7 @@ class DbManagementTest extends BitcoinSAsyncTest with EmbeddedPg {
                                         dbConfig(ProjectType.Chain))
     val chainDbManagement = createChainDbManagement(chainAppConfig)
     val result = chainDbManagement.migrate()
-    assert(result == 1)
+    assert(result == 2)
   }
 
   it must "run migrations for wallet db" in {

db-commons/src/main/scala/org/bitcoins/db/DbCommonsColumnMappers.scala

@@ -67,6 +67,15 @@ class DbCommonsColumnMappers(val profile: JdbcProfile) {
       DoubleSha256DigestBE.fromHex
     )
 
+  implicit val bigIntMapper: BaseColumnType[BigInt] =
+    MappedColumnType
+      .base[BigInt, Array[Byte]](_.toByteArray.dropWhile(_ == 0x00),
+                                 BigInt(1, _))
+
+  implicit val bigIntPostgresMapper: BaseColumnType[BigInt] =
+    MappedColumnType
+      .base[BigInt, BigDecimal](BigDecimal(_), _.toBigInt)
+
   implicit val ecPublicKeyMapper: BaseColumnType[ECPublicKey] =
     MappedColumnType.base[ECPublicKey, String](_.hex, ECPublicKey.fromHex)
 

node/src/main/scala/org/bitcoins/node/Node.scala

@@ -11,6 +11,7 @@ import org.bitcoins.chain.models.{
 import org.bitcoins.core.api.{ChainQueryApi, NodeApi}
 import org.bitcoins.core.p2p.{NetworkPayload, TypeIdentifier}
 import org.bitcoins.core.protocol.transaction.Transaction
+import org.bitcoins.core.util.FutureUtil
 import org.bitcoins.crypto.{DoubleSha256Digest, DoubleSha256DigestBE}
 import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.node.models.{
@@ -126,6 +127,14 @@ trait Node extends NodeApi with ChainQueryApi with P2PLogger {
     val start = System.currentTimeMillis()
     for {
       _ <- nodeAppConfig.initialize()
+      // get chainApi so we don't need to call chainApiFromDb on every call
+      chainApi <- chainApiFromDb
+
+      isMissingChainWork <- chainApi.isMissingChainWork
+      _ <- if (isMissingChainWork) {
+        chainApi.recalculateChainWork
+      } else FutureUtil.unit
+
       node <- {
         val isInitializedF = for {
           _ <- peerMsgSenderF.map(_.connect())
@@ -136,14 +145,13 @@ trait Node extends NodeApi with ChainQueryApi with P2PLogger {
           logger.error(s"Failed to connect with peer=$peer with err=${err}"))
 
         isInitializedF.map { _ =>
-          logger.info(s"Our peer=${peer} has been initialized")
+          logger.info(s"Our peer=$peer has been initialized")
           logger.info(s"Our node has been full started. It took=${System
             .currentTimeMillis() - start}ms")
           this
         }
       }
-      // get chainApi so we don't need to call chainApiFromDb on every call
+
-      chainApi <- chainApiFromDb
       bestHash <- chainApi.getBestBlockHash()
       bestHeight <- chainApi.getBestHashBlockHeight()
       filterCount <- chainApi.getFilterCount
@@ -195,6 +203,7 @@ trait Node extends NodeApi with ChainQueryApi with P2PLogger {
       header <- chainApi
         .getHeader(hash)
         .map(_.get) // .get is safe since this is an internal call
+
     } yield {
       peerMsgSenderF.map(_.sendGetHeadersMessage(hash.flip))
       logger.info(s"Starting sync node, height=${header.height} hash=$hash")

testkit/src/main/scala/org/bitcoins/testkit/chain/BlockHeaderHelper.scala

@@ -1,6 +1,7 @@
 package org.bitcoins.testkit.chain
 
 import org.bitcoins.chain.models.{BlockHeaderDb, BlockHeaderDbHelper}
+import org.bitcoins.chain.pow.Pow
 import org.bitcoins.chain.validation.TipValidation
 import org.bitcoins.core.number.{Int32, UInt32}
 import org.bitcoins.core.protocol.blockchain.BlockHeader
@@ -29,7 +30,9 @@ abstract class BlockHeaderHelper {
   }
 
   val header1Db: BlockHeaderDb = {
-    BlockHeaderDbHelper.fromBlockHeader(566093, header1)
+    BlockHeaderDbHelper.fromBlockHeader(566093,
+                                        Pow.getBlockProof(header1),
+                                        header1)
   }
 
   /**
@@ -44,7 +47,8 @@ abstract class BlockHeaderHelper {
   }
 
   val header2Db: BlockHeaderDb = {
-    BlockHeaderDbHelper.fromBlockHeader(566092, header2)
+    val chainWork = header1Db.chainWork + Pow.getBlockProof(header2)
+    BlockHeaderDbHelper.fromBlockHeader(566092, chainWork, header2)
   }
 
   lazy val twoValidHeaders: Vector[BlockHeader] = {
@@ -129,7 +133,10 @@ abstract class BlockHeaderHelper {
     if (TipValidation.isBadNonce(blockHeader)) {
       buildNextHeader(prevHeader)
     } else {
-      BlockHeaderDbHelper.fromBlockHeader(prevHeader.height + 1, blockHeader)
+      val chainWork = prevHeader.chainWork + Pow.getBlockProof(blockHeader)
+      BlockHeaderDbHelper.fromBlockHeader(prevHeader.height + 1,
+                                          chainWork,
+                                          blockHeader)
     }
   }
 }

testkit/src/main/scala/org/bitcoins/testkit/chain/ChainTestUtil.scala

@@ -1,6 +1,7 @@
 package org.bitcoins.testkit.chain
 
 import org.bitcoins.chain.models._
+import org.bitcoins.chain.pow.Pow
 import org.bitcoins.core.gcs.{BlockFilter, FilterHeader, GolombFilter}
 import org.bitcoins.core.protocol.blockchain.{
   BlockHeader,
@@ -15,7 +16,10 @@ sealed abstract class ChainTestUtil {
   lazy val regTestHeader: BlockHeader =
     regTestChainParams.genesisBlock.blockHeader
   lazy val regTestGenesisHeaderDb: BlockHeaderDb = {
-    BlockHeaderDbHelper.fromBlockHeader(height = 0, bh = regTestHeader)
+    BlockHeaderDbHelper.fromBlockHeader(height = 0,
+                                        chainWork =
+                                          Pow.getBlockProof(regTestHeader),
+                                        bh = regTestHeader)
   }
   lazy val regTestGenesisHeaderCompactFilter: GolombFilter =
     BlockFilter.apply(regTestChainParams.genesisBlock, Vector.empty)
@@ -55,26 +59,36 @@ sealed abstract class ChainTestUtil {
       "000000200cd536b3eb1cd9c028e081f1455006276b293467c3e5170000000000000000007bc1b27489db01c85d38a4bc6d2280611e9804f506d83ad00d2a33ebd663992f76c7725c505b2e174fb90f55")
 
     lazy val blockHeaderDb564480 =
-      BlockHeaderDbHelper.fromBlockHeader(564480, blockHeader564480)
+      BlockHeaderDbHelper.fromBlockHeader(564480,
+                                          Pow.getBlockProof(blockHeader564480),
+                                          blockHeader564480)
 
     lazy val blockHeader566494 = BlockHeader.fromHex(
       "00000020ea2cb07d670ddb7a158e72ddfcfd9e1b9bf4459278bb240000000000000000004fb33054d79de69bb84b4d5c7dd87d80473c416320427a882c72108f7e43fd0c3d3e855c505b2e178f328fe2")
 
     lazy val blockHeaderDb566494 =
-      BlockHeaderDbHelper.fromBlockHeader(566594, blockHeader566494)
+      BlockHeaderDbHelper.fromBlockHeader(566594,
+                                          Pow.getBlockProof(blockHeader566494),
+                                          blockHeader566494)
 
     lazy val blockHeader566495 = BlockHeader.fromHex(
       "000000202164d8c4e5246ab003fdebe36c697b9418aa454ec4190d00000000000000000059134ad5aaad38a0e75946c7d4cb09b3ad45b459070195dd564cde193cf0ef29c33e855c505b2e17f61af734")
 
-    lazy val blockHeaderDb566495 =
+    lazy val blockHeaderDb566495 = {
-      BlockHeaderDbHelper.fromBlockHeader(566495, blockHeader566495)
+      val chainWork = blockHeaderDb566494.chainWork + Pow.getBlockProof(
+        blockHeader566495)
+      BlockHeaderDbHelper.fromBlockHeader(566495, chainWork, blockHeader566495)
+    }
 
     //https://blockstream.info/block/00000000000000000015fea169c62eb0a1161aba36932ca32bc3785cbb3480bf
     lazy val blockHeader566496 = BlockHeader.fromHex(
       "000000201b61e8961710991a47ff8187d946d93e4fb33569c09622000000000000000000d0098658f53531e6e67fc9448986b5a8f994da42d746079eabe10f55e561e243103f855c17612e1735c4afdb")
 
-    lazy val blockHeaderDb566496 =
+    lazy val blockHeaderDb566496 = {
-      BlockHeaderDbHelper.fromBlockHeader(566496, blockHeader566496)
+      val chainWork = blockHeaderDb566495.chainWork + Pow.getBlockProof(
+        blockHeader566496)
+      BlockHeaderDbHelper.fromBlockHeader(566496, chainWork, blockHeader566496)
+    }
   }
 }
 

testkit/src/main/scala/org/bitcoins/testkit/chain/ChainUnitTest.scala

@@ -10,6 +10,7 @@ import org.bitcoins.chain.blockchain.ChainHandler
 import org.bitcoins.chain.blockchain.sync.ChainSync
 import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.chain.models._
+import org.bitcoins.chain.pow.Pow
 import org.bitcoins.core.protocol.blockchain.{Block, BlockHeader}
 import org.bitcoins.crypto.DoubleSha256DigestBE
 import org.bitcoins.db.AppConfig
@@ -355,9 +356,20 @@ object ChainUnitTest extends ChainVerificationLogger {
         logger.error(s"Failed to parse headers from block_headers.json: $err")
         Future.failed(new RuntimeException(err.toString))
       case JsSuccess(headers, _) =>
+        var prevHeaderOpt: Option[BlockHeaderDb] = None
         val dbHeaders = headers.zipWithIndex.map {
           case (header, height) =>
-            BlockHeaderDbHelper.fromBlockHeader(height + OFFSET, header)
+            val chainWork = prevHeaderOpt match {
+              case None => Pow.getBlockProof(header)
+              case Some(prevHeader) =>
+                prevHeader.chainWork + Pow.getBlockProof(header)
+            }
+
+            val newHeader = BlockHeaderDbHelper.fromBlockHeader(height + OFFSET,
+                                                                chainWork,
+                                                                header)
+            prevHeaderOpt = Some(newHeader)
+            newHeader
         }
 
         @tailrec