mirrors/bitcoin-s
1
0
Fork 0
mirror of https://github.com/bitcoin-s/bitcoin-s.git synced 2024-11-19 09:52:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Implement best block hash correctly (#1452)

Browse Source 
* Implement best block hash correctly

* Handle no common history

* Fix getBlockProof to be BigInt instead of UInt32

* Fix warnings

* Make get best filter use chainwork

* Remove unused function, test isMissingChainWork

* Fix comparisons for chain work in sql

* Fix migrations from rebase

* Fix CI error

* Fix postgresql
This commit is contained in:
Ben Carman 2020-06-04 12:05:10 -05:00 committed by GitHub
parent 2f53379f16
commit 61d9f0efba
23 changed files with 461 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bitcoind-rpc-test/src/test/scala/org/bitcoins/rpc/v18/BitcoindV18RpcClientTest.scala
View File

@ -100,7 +100,7 @@ class BitcoindV18RpcClientTest extends BitcoindRpcTest {
it should "successfully submit a header" in {
val genesisHeader = RegTestNetChainParams.genesisBlock.blockHeader
val genesisHeaderDb =
BlockHeaderDbHelper.fromBlockHeader(height = 1, genesisHeader)
BlockHeaderDbHelper.fromBlockHeader(height = 1, BigInt(0), genesisHeader)
val nextHeader = BlockHeaderHelper.buildNextHeader(genesisHeaderDb)
clientF.flatMap(client =>
client.submitHeader(nextHeader.blockHeader).map(_ => succeed))

99
chain-test/src/test/scala/org/bitcoins/chain/blockchain/ChainHandlerTest.scala
View File

@ -4,6 +4,7 @@ import akka.actor.ActorSystem
import org.bitcoins.chain.api.ChainApi
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.chain.models.{BlockHeaderDb, BlockHeaderDbHelper}
import org.bitcoins.chain.pow.Pow
import org.bitcoins.core.gcs.{BlockFilter, FilterHeader}
import org.bitcoins.core.number.{Int32, UInt32}
import org.bitcoins.core.p2p.CompactFilterMessage
@ -26,6 +27,7 @@ import org.bitcoins.testkit.util.{FileUtil, ScalaTestUtil}
import org.scalatest.{Assertion, FutureOutcome}
import play.api.libs.json.Json
import scala.annotation.tailrec
import scala.concurrent.Future
import scala.io.BufferedSource
@ -78,6 +80,37 @@ class ChainHandlerTest extends ChainDbUnitTest {
foundHeaderF.map(found => assert(found.get == newValidHeader))
}
it must "have getBestBlockHash return the header with the most work, not the highest" in {
tempHandler: ChainHandler =>
val dummyHeader =
BlockHeaderDbHelper.fromBlockHeader(1,
BigInt(0),
ChainTestUtil.blockHeader562462)
val highestHeader =
BlockHeaderDbHelper.fromBlockHeader(2,
BigInt(0),
ChainTestUtil.blockHeader562463)
val headerWithMostWork =
BlockHeaderDbHelper.fromBlockHeader(1,
BigInt(1000),
ChainTestUtil.blockHeader562464)
val tallestBlockchain =
Blockchain(Vector(highestHeader, dummyHeader, genesis))
val mostWorkChain = Blockchain(Vector(headerWithMostWork, genesis))
val chainHandler =
tempHandler.copy(blockchains = Vector(tallestBlockchain, mostWorkChain))
for {
hash <- chainHandler.getBestBlockHash()
} yield {
assert(hash == headerWithMostWork.blockHeader.hashBE)
}
}
it must "have an in-order seed" in { _ =>
val source = FileUtil.getFileAsSource("block_headers.json")
val arrStr = source.getLines.next
@ -108,14 +141,17 @@ class ChainHandlerTest extends ChainDbUnitTest {
val firstBlockHeaderDb =
BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 2,
BigInt(0),
ChainTestUtil.blockHeader562462)
val secondBlockHeaderDb =
BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 1,
BigInt(0),
ChainTestUtil.blockHeader562463)
val thirdBlockHeaderDb =
BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE,
BigInt(0),
ChainTestUtil.blockHeader562464)
/*
@ -160,16 +196,26 @@ class ChainHandlerTest extends ChainDbUnitTest {
ChainUnitTest.FIRST_POW_CHANGE - ChainUnitTest.FIRST_BLOCK_HEIGHT)
val firstBlockHeaderDb =
BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 2,
ChainTestUtil.blockHeader562462)
BlockHeaderDbHelper.fromBlockHeader(
ChainUnitTest.FIRST_POW_CHANGE - 2,
Pow.getBlockProof(ChainTestUtil.blockHeader562462),
ChainTestUtil.blockHeader562462)
val secondBlockHeaderDb =
val secondBlockHeaderDb = {
val chainWork = firstBlockHeaderDb.chainWork + Pow.getBlockProof(
ChainTestUtil.blockHeader562463)
BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE - 1,
chainWork,
ChainTestUtil.blockHeader562463)
}
val thirdBlockHeaderDb =
val thirdBlockHeaderDb = {
val chainWork = secondBlockHeaderDb.chainWork + Pow.getBlockProof(
ChainTestUtil.blockHeader562464)
BlockHeaderDbHelper.fromBlockHeader(ChainUnitTest.FIRST_POW_CHANGE,
chainWork,
ChainTestUtil.blockHeader562464)
}
/*
* We need to insert one block before the first POW check because it is used on the next
@ -460,33 +506,72 @@ class ChainHandlerTest extends ChainDbUnitTest {
}
}
it must "properly recalculate chain work" in { tempHandler: ChainHandler =>
val headersWithNoWork = Vector(
BlockHeaderDbHelper.fromBlockHeader(3,
BigInt(0),
ChainTestUtil.blockHeader562464),
BlockHeaderDbHelper.fromBlockHeader(2,
BigInt(0),
ChainTestUtil.blockHeader562463),
BlockHeaderDbHelper.fromBlockHeader(1,
BigInt(0),
ChainTestUtil.blockHeader562462)
)
val blockchain = Blockchain(headersWithNoWork :+ genesis)
val chainHandler = tempHandler.copy(blockchains = Vector(blockchain))
for {
_ <- chainHandler.blockHeaderDAO.createAll(headersWithNoWork)
isMissingWork <- chainHandler.isMissingChainWork
_ = assert(isMissingWork)
newHandler <- chainHandler.recalculateChainWork
headerDb <- newHandler.getBestBlockHeader()
} yield {
assert(headerDb.height == headersWithNoWork.head.height)
assert(headerDb.hashBE == headersWithNoWork.head.hashBE)
assert(headerDb.chainWork == BigInt(12885098501L))
}
}
final def processHeaders(
processorF: Future[ChainApi],
headers: Vector[BlockHeader],
height: Int): Future[Assertion] = {
val processedHeadersF = processorF.flatMap(_.processHeaders(headers))
@tailrec
def loop(
remainingHeaders: Vector[BlockHeader],
prevHeaderDbOpt: Option[BlockHeaderDb],
height: Int,
accum: Vector[Future[Assertion]]): Vector[Future[Assertion]] = {
remainingHeaders match {
case header +: headersTail =>
val getHeaderF = processedHeadersF.flatMap(_.getHeader(header.hashBE))
val chainWork = prevHeaderDbOpt match {
case None => Pow.getBlockProof(header)
case Some(prevHeader) =>
prevHeader.chainWork + Pow.getBlockProof(header)
}
val expectedBlockHeaderDb =
BlockHeaderDbHelper.fromBlockHeader(height, header)
BlockHeaderDbHelper.fromBlockHeader(height, chainWork, header)
val assertionF =
getHeaderF.map(headerOpt =>
assert(headerOpt.contains(expectedBlockHeaderDb)))
val newAccum = accum.:+(assertionF)
loop(headersTail, height + 1, newAccum)
loop(headersTail, Some(expectedBlockHeaderDb), height + 1, newAccum)
case Vector() =>
accum
}
}
val vecFutAssert: Vector[Future[Assertion]] =
loop(headers, height, Vector.empty)
loop(headers, None, height, Vector.empty)
ScalaTestUtil.toAssertF(vecFutAssert)
}

1
chain-test/src/test/scala/org/bitcoins/chain/models/BlockHeaderDAOTest.scala
View File

@ -10,6 +10,7 @@ import org.bitcoins.testkit.chain.{
ChainUnitTest
}
import org.scalatest.FutureOutcome
import scodec.bits._
import scala.concurrent.Future

25
chain-test/src/test/scala/org/bitcoins/chain/pow/BitcoinPowTest.scala
View File

@ -3,7 +3,10 @@ package org.bitcoins.chain.pow
import akka.actor.ActorSystem
import org.bitcoins.chain.blockchain.Blockchain
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.core.protocol.blockchain.MainNetChainParams
import org.bitcoins.core.protocol.blockchain.{
MainNetChainParams,
TestNetChainParams
}
import org.bitcoins.testkit.chain.fixture.{ChainFixture, ChainFixtureTag}
import org.bitcoins.testkit.chain.{
ChainDbUnitTest,
@ -78,4 +81,24 @@ class BitcoinPowTest extends ChainDbUnitTest {
seqF.map(_ => succeed)
}
it must "getBlockProof correctly for the testnet genesis block" inFixtured {
case ChainFixture.Empty =>
Future {
val header = TestNetChainParams.genesisBlock.blockHeader
val proof = Pow.getBlockProof(header)
assert(proof == BigInt(4295032833L))
}
}
it must "getBlockProof correctly for the mainnet genesis block" inFixtured {
case ChainFixture.Empty =>
Future {
val header = MainNetChainParams.genesisBlock.blockHeader
val proof = Pow.getBlockProof(header)
assert(proof == BigInt(4295032833L))
}
}
}

10
chain-test/src/test/scala/org/bitcoins/chain/validation/TipValidationTest.scala
View File

@ -1,14 +1,15 @@
package org.bitcoins.chain.validation
import akka.actor.ActorSystem
import org.bitcoins.chain.blockchain.Blockchain
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDbHelper}
import org.bitcoins.chain.pow.Pow
import org.bitcoins.core.protocol.blockchain.BlockHeader
import org.bitcoins.testkit.chain.{BlockHeaderHelper, ChainDbUnitTest}
import org.scalatest.{Assertion, FutureOutcome}
class TipValidationTest extends ChainDbUnitTest {
import org.bitcoins.chain.blockchain.Blockchain
import org.bitcoins.chain.config.ChainAppConfig
override type FixtureParam = BlockHeaderDAO
@ -29,7 +30,10 @@ class TipValidationTest extends ChainDbUnitTest {
it must "connect two blocks with that are valid" in { bhDAO =>
val newValidTipDb =
BlockHeaderDbHelper.fromBlockHeader(566093, newValidTip)
BlockHeaderDbHelper.fromBlockHeader(
566093,
currentTipDb.chainWork + Pow.getBlockProof(newValidTip),
newValidTip)
val expected = TipUpdateResult.Success(newValidTipDb)
runTest(newValidTip, expected, blockchain)

1
chain/src/main/resources/postgresql/chain/migration/V2__add_block_header_chain_work.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE block_headers ADD COLUMN chain_work numeric(78,0) NOT NULL DEFAULT 0;

1
chain/src/main/resources/sqlite/chain/migration/V2__add_block_header_chain_work.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE "block_headers" ADD COLUMN "chain_work" VARBINARY(32) NOT NULL DEFAULT "00";

2
chain/src/main/scala/org/bitcoins/chain/blockchain/BaseBlockChain.scala
View File

@ -39,7 +39,7 @@ private[blockchain] trait BaseBlockChain extends SeqWrapper[BlockHeaderDb] {
protected[blockchain] def compObjectfromHeaders(
headers: scala.collection.immutable.Seq[BlockHeaderDb]): Blockchain
val tip: BlockHeaderDb = headers.head
val tip: BlockHeaderDb = headers.maxBy(_.height)
/** The height of the chain */
val height: Int = tip.height

181
chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala
View File

@ -4,6 +4,7 @@ import org.bitcoins.chain.ChainVerificationLogger
import org.bitcoins.chain.api.ChainApi
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.chain.models._
import org.bitcoins.chain.pow.Pow
import org.bitcoins.core.api.ChainQueryApi.FilterResponse
import org.bitcoins.core.gcs.FilterHeader
import org.bitcoins.core.number.UInt32
@ -45,22 +46,33 @@ case class ChainHandler(
/** @inheritdoc */
override def getBlockCount(): Future[Int] = {
logger.debug(s"Querying for block count")
blockHeaderDAO.maxHeight.map { height =>
blockHeaderDAO.bestHeight.map { height =>
logger.debug(s"getBlockCount result: count=$height")
height
}
}
override def getBestBlockHeader(): Future[BlockHeaderDb] = {
for {
hash <- getBestBlockHash()
headerOpt <- getHeader(hash)
} yield headerOpt match {
case None =>
throw new RuntimeException(
s"We found best hash=${hash.hex} but could not retrieve the full header!!!")
case Some(header) => header
logger.debug(s"Querying for best block hash")
//https://bitcoin.org/en/glossary/block-chain
val groupedChains = blockchains.groupBy(_.tip.chainWork)
val maxWork = groupedChains.keys.max
val chains = groupedChains(maxWork)
val bestHeader: BlockHeaderDb = chains match {
case Vector() =>
// This should never happen
val errMsg = s"Did not find blockchain with work $maxWork"
logger.error(errMsg)
throw new RuntimeException(errMsg)
case chain +: Vector() =>
chain.tip
case chain +: rest =>
logger.warn(
s"We have multiple competing blockchains: ${(chain +: rest).map(_.tip.hashBE.hex).mkString(", ")}")
chain.tip
}
Future.successful(bestHeader)
}
/** @inheritdoc */
@ -110,28 +122,7 @@ case class ChainHandler(
* @inheritdoc
*/
override def getBestBlockHash(): Future[DoubleSha256DigestBE] = {
logger.debug(s"Querying for best block hash")
//naive implementation, this is looking for the tip with the _most_ proof of work
//this does _not_ mean that it is on the chain that has the most work
//TODO: Enhance this in the future to return the "heaviest" header
//https://bitcoin.org/en/glossary/block-chain
val groupedChains = blockchains.groupBy(_.tip.height)
val maxHeight = groupedChains.keys.max
val chains = groupedChains(maxHeight)
val hashBE: DoubleSha256DigestBE = chains match {
case Vector() =>
val errMsg = s"Did not find blockchain with height $maxHeight"
logger.error(errMsg)
throw new RuntimeException(errMsg)
case chain +: Vector() =>
chain.tip.hashBE
case chain +: rest =>
logger.warn(
s"We have multiple competing blockchains: ${(chain +: rest).map(_.tip.hashBE.hex).mkString(", ")}")
chain.tip.hashBE
}
Future.successful(hashBE)
getBestBlockHeader().map(_.hashBE)
}
/** @inheritdoc */
@ -147,7 +138,7 @@ case class ChainHandler(
throw UnknownBlockHash(s"Unknown block hash ${prevStopHash}"))
} yield prevStopHeader.height + 1
}
val blockCountF = getBlockCount
val blockCountF = getBlockCount()
for {
startHeight <- startHeightF
blockCount <- blockCountF
@ -361,8 +352,9 @@ case class ChainHandler(
/** @inheritdoc */
override def getFilterHeaderCount: Future[Int] = {
logger.debug(s"Querying for filter header count")
filterHeaderDAO.maxHeight.map { height =>
logger.debug(s"getFilterHeaderCount result: count=$height")
filterHeaderDAO.getBestFilter.map { filterHeader =>
val height = filterHeader.height
logger.debug(s"getFilterCount result: count=$height")
height
}
}
@ -374,28 +366,7 @@ case class ChainHandler(
/** @inheritdoc */
override def getBestFilterHeader(): Future[CompactFilterHeaderDb] = {
//this seems realy brittle, is there a guarantee
//that the highest filter header count is our best filter header?
val filterCountF = getFilterHeaderCount()
val ourBestFilterHeader = for {
count <- filterCountF
filterHeader <- getFilterHeadersAtHeight(count)
} yield {
//TODO: Figure out what the best way to select
//the best filter header is if we have competing
//chains. (Same thing applies to getBestBlockHash()
//for now, just do the dumb thing and pick the first one
filterHeader match {
case tip1 +: _ +: _ =>
tip1
case tip +: _ =>
tip
case Vector() =>
sys.error(s"No filter headers found in database!")
}
}
ourBestFilterHeader
filterHeaderDAO.getBestFilter
}
/** @inheritdoc */
@ -406,7 +377,8 @@ case class ChainHandler(
/** @inheritdoc */
override def getFilterCount: Future[Int] = {
logger.debug(s"Querying for filter count")
filterDAO.maxHeight.map { height =>
filterDAO.getBestFilter.map { filter =>
val height = filter.height
logger.debug(s"getFilterCount result: count=$height")
height
}
@ -486,6 +458,101 @@ case class ChainHandler(
}
loop(Future.successful(to), Vector.empty)
}
def isMissingChainWork: Future[Boolean] = {
for {
first100 <- blockHeaderDAO.getBetweenHeights(0, 100)
first100MissingWork = first100.nonEmpty && first100.exists(
_.chainWork == BigInt(0))
isMissingWork <- {
if (first100MissingWork) {
Future.successful(true)
} else {
for {
height <- getBestHashBlockHeight()
last100 <- blockHeaderDAO.getBetweenHeights(height - 100, height)
last100MissingWork = last100.nonEmpty && last100.exists(
_.chainWork == BigInt(0))
} yield last100MissingWork
}
}
} yield isMissingWork
}
def recalculateChainWork: Future[ChainHandler] = {
logger.info("Calculating chain work for previous blocks")
val batchSize = chainConfig.chain.difficultyChangeInterval
def loop(
currentChainWork: BigInt,
remainingHeaders: Vector[BlockHeaderDb],
accum: Vector[BlockHeaderDb]): Future[Vector[BlockHeaderDb]] = {
if (remainingHeaders.isEmpty) {
blockHeaderDAO.upsertAll(accum.takeRight(batchSize))
} else {
val header = remainingHeaders.head
val newChainWork = currentChainWork + Pow.getBlockProof(
header.blockHeader)
val newHeader = header.copy(chainWork = newChainWork)
// Add the last batch to the database and create log
if (header.height % batchSize == 0) {
logger.info(
s"Recalculating chain work... current height: ${header.height}")
val updated = accum :+ newHeader
// updated the latest batch
blockHeaderDAO
.upsertAll(updated.takeRight(batchSize))
.flatMap(
_ =>
loop(
newChainWork,
remainingHeaders.tail,
updated.takeRight(batchSize)
))
} else {
loop(newChainWork, remainingHeaders.tail, accum :+ newHeader)
}
}
}
for {
tips <- blockHeaderDAO.chainTipsByHeight
fullChains <- FutureUtil.sequentially(tips)(tip =>
blockHeaderDAO.getFullBlockchainFrom(tip))
commonHistory = fullChains.foldLeft(fullChains.head.headers)(
(accum, chain) => chain.intersect(accum).toVector)
sortedHeaders = commonHistory.sortBy(_.height)
diffedChains = fullChains.map { blockchain =>
val sortedFullHeaders = blockchain.headers.sortBy(_.height)
// Remove the common blocks
sortedFullHeaders.diff(sortedHeaders)
}
commonWithWork <- loop(BigInt(0), sortedHeaders, Vector.empty)
finalCommon = Vector(sortedHeaders.head) ++ commonWithWork.takeRight(
batchSize)
commonChainWork = finalCommon.lastOption
.map(_.chainWork)
.getOrElse(BigInt(0))
newBlockchains <- FutureUtil.sequentially(diffedChains) { blockchain =>
loop(commonChainWork, blockchain, Vector.empty)
.map { newHeaders =>
val relevantHeaders =
(finalCommon ++ newHeaders).takeRight(
chainConfig.chain.difficultyChangeInterval)
Blockchain(relevantHeaders)
}
}
} yield {
logger.info("Finished calculating chain work")
this.copy(blockchains = newBlockchains.toVector)
}
}
}
object ChainHandler {

11
chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala
View File

@ -68,6 +68,7 @@ case class ChainAppConfig(
} else {
val genesisHeader =
BlockHeaderDbHelper.fromBlockHeader(height = 0,
chainWork = BigInt(0),
bh =
chain.genesisBlock.blockHeader)
val blockHeaderDAO = BlockHeaderDAO()(ec, appConfig)
@ -100,9 +101,9 @@ object ChainAppConfig extends AppConfigFactory[ChainAppConfig] {
/** Constructs a chain verification configuration from the default Bitcoin-S
* data directory and given list of configuration overrides.
*/
override def fromDatadir(datadir: Path, useLogbackConf: Boolean, confs: Vector[Config])(
implicit ec: ExecutionContext): ChainAppConfig =
ChainAppConfig(datadir,
useLogbackConf,
confs: _*)
override def fromDatadir(
datadir: Path,
useLogbackConf: Boolean,
confs: Vector[Config])(implicit ec: ExecutionContext): ChainAppConfig =
ChainAppConfig(datadir, useLogbackConf, confs: _*)
}

72
chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDAO.scala
View File

@ -22,7 +22,14 @@ case class BlockHeaderDAO()(
import profile.api._
private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
import mappers._
import mappers.{doubleSha256DigestBEMapper, int32Mapper, uInt32Mapper}
implicit private val bigIntMapper: BaseColumnType[BigInt] =
if (appConfig.driverName == "postgresql") {
mappers.bigIntPostgresMapper
} else {
mappers.bigIntMapper
}
override val table =
profile.api.TableQuery[BlockHeaderTable]
@ -126,6 +133,19 @@ case class BlockHeaderDAO()(
table.filter(_.height === height).result
}
/** Retrieves a [[BlockHeaderDb]] with the given chain work */
def getAtChainWork(work: BigInt): Future[Vector[BlockHeaderDb]] = {
val query = getAtChainWorkQuery(work)
safeDatabase.runVec(query)
}
def getAtChainWorkQuery(work: BigInt): profile.StreamingProfileAction[
Seq[BlockHeaderDb],
BlockHeaderDb,
Effect.Read] = {
table.filter(_.chainWork === work).result
}
/** Gets Block Headers between (inclusive) start height and stop hash, could be out of order */
def getBetweenHeightAndHash(
startHeight: Int,
@ -223,10 +243,24 @@ case class BlockHeaderDAO()(
query
}
/** Returns the chainTips in our database. This can be multiple headers if we have
* competing blockchains (fork) */
def chainTips: Future[Vector[BlockHeaderDb]] = {
logger.debug(s"Getting chaintips from: ${dbConfig.config}")
/** Returns the block height of the block with the most work from our database */
def bestHeight: Future[Int] = {
chainTips.map(_.maxBy(_.chainWork).height)
}
private val maxWorkQuery: profile.ProfileAction[
BigInt,
NoStream,
Effect.Read] = {
val query = table.map(_.chainWork).max.getOrElse(BigInt(0)).result
query
}
/** Returns the chainTips in our database calculated by max height, not work.
* This should only be used if the chain work has not been calculated
*/
def chainTipsByHeight: Future[Vector[BlockHeaderDb]] = {
logger.debug(s"Getting chain tips from: ${dbConfig.config}")
val aggregate = {
maxHeightQuery.flatMap { height =>
logger.debug(s"Max block height: $height")
@ -241,6 +275,22 @@ case class BlockHeaderDAO()(
safeDatabase.runVec(aggregate)
}
def chainTips: Future[Vector[BlockHeaderDb]] = {
logger.debug(s"Getting chain tips from: ${dbConfig.config}")
val aggregate = {
maxWorkQuery.flatMap { work =>
logger.debug(s"Max block work: $work")
val atChainWork = getAtChainWorkQuery(work)
atChainWork.map { headers =>
logger.debug(s"Headers at $work: $headers")
}
atChainWork
}
}
safeDatabase.runVec(aggregate)
}
/** Returns competing blockchains that are contained in our BlockHeaderDAO
* Each chain returns the last [[org.bitcoins.core.protocol.blockchain.ChainParams.difficultyChangeInterval difficutly interval]]
* block headers as defined by the network we are on. For instance, on bitcoin mainnet this will be 2016 block headers.
@ -270,6 +320,13 @@ case class BlockHeaderDAO()(
headersF.map(headers => Blockchain.fromHeaders(headers.reverse))
}
/** Retrieves a full blockchain with the best tip being the given header */
def getFullBlockchainFrom(header: BlockHeaderDb)(
implicit ec: ExecutionContext): Future[Blockchain] = {
val headersF = getBetweenHeights(from = 0, to = header.height)
headersF.map(headers => Blockchain.fromHeaders(headers.reverse))
}
/** Finds a [[org.bitcoins.chain.models.BlockHeaderDb block header]] that satisfies the given predicate, else returns None */
def find(f: BlockHeaderDb => Boolean)(
implicit ec: ExecutionContext): Future[Option[BlockHeaderDb]] = {
@ -310,6 +367,8 @@ case class BlockHeaderDAO()(
def hex = column[String]("hex")
def chainWork: Rep[BigInt] = column[BigInt]("chain_work")
/** The sql index for searching based on [[height]] */
def heightIndex = index("block_headers_height_index", height)
@ -324,7 +383,8 @@ case class BlockHeaderDAO()(
time,
nBits,
nonce,
hex).<>(BlockHeaderDb.tupled, BlockHeaderDb.unapply)
hex,
chainWork).<>(BlockHeaderDb.tupled, BlockHeaderDb.unapply)
}
}

11
chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDb.scala
View File

@ -13,7 +13,8 @@ case class BlockHeaderDb(
time: UInt32,
nBits: UInt32,
nonce: UInt32,
hex: String) {
hex: String,
chainWork: BigInt) {
lazy val blockHeader: BlockHeader = {
val blockHeader = BlockHeader.fromHex(hex)
@ -30,7 +31,10 @@ case class BlockHeaderDb(
object BlockHeaderDbHelper {
def fromBlockHeader(height: Int, bh: BlockHeader): BlockHeaderDb = {
def fromBlockHeader(
height: Int,
chainWork: BigInt,
bh: BlockHeader): BlockHeaderDb = {
BlockHeaderDb(
height = height,
hashBE = bh.hashBE,
@ -40,7 +44,8 @@ object BlockHeaderDbHelper {
nBits = bh.nBits,
nonce = bh.nonce,
version = bh.version,
hex = bh.hex
hex = bh.hex,
chainWork = chainWork
)
}
}

28
chain/src/main/scala/org/bitcoins/chain/models/CompactFilterDAO.scala
View File

@ -14,8 +14,18 @@ case class CompactFilterDAO()(
extends CRUD[CompactFilterDb, DoubleSha256DigestBE]
with SlickUtil[CompactFilterDb, DoubleSha256DigestBE] {
val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
import mappers._
import mappers.{
byteVectorMapper,
doubleSha256DigestBEMapper,
filterTypeMapper
}
import profile.api._
implicit private val bigIntMapper: BaseColumnType[BigInt] =
if (appConfig.driverName == "postgresql") {
mappers.bigIntPostgresMapper
} else {
mappers.bigIntMapper
}
class CompactFilterTable(tag: Tag)
extends Table[CompactFilterDb](tag, "cfilters") {
@ -43,6 +53,11 @@ case class CompactFilterDAO()(
TableQuery[CompactFilterTable]
}
private lazy val blockHeaderTable: profile.api.TableQuery[
BlockHeaderDAO#BlockHeaderTable] = {
BlockHeaderDAO().table
}
override def createAll(
filters: Vector[CompactFilterDb]): Future[Vector[CompactFilterDb]] = {
createAllNoAutoInc(ts = filters, database = safeDatabase)
@ -109,4 +124,15 @@ case class CompactFilterDAO()(
Effect.Read] = {
table.filter(header => header.height >= from && header.height <= to).result
}
def getBestFilter: Future[CompactFilterDb] = {
val join = table join blockHeaderTable on (_.blockHash === _.hash)
val query = join.groupBy(_._1).map {
case (filter, headers) =>
filter -> headers.map(_._2.chainWork).max
}
safeDatabase
.runVec(query.result)
.map(_.maxBy(_._2.getOrElse(BigInt(0)))._1)
}
}

24
chain/src/main/scala/org/bitcoins/chain/models/CompactFilterHeaderDAO.scala
View File

@ -13,7 +13,13 @@ case class CompactFilterHeaderDAO()(
with SlickUtil[CompactFilterHeaderDb, DoubleSha256DigestBE] {
import profile.api._
val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
import mappers._
import mappers.doubleSha256DigestBEMapper
implicit private val bigIntMapper: BaseColumnType[BigInt] =
if (appConfig.driverName == "postgresql") {
mappers.bigIntPostgresMapper
} else {
mappers.bigIntMapper
}
class CompactFilterHeaderTable(tag: Tag)
extends Table[CompactFilterHeaderDb](tag, "cfheaders") {
@ -42,6 +48,11 @@ case class CompactFilterHeaderDAO()(
TableQuery[CompactFilterHeaderTable]
}
private lazy val blockHeaderTable: profile.api.TableQuery[
BlockHeaderDAO#BlockHeaderTable] = {
BlockHeaderDAO().table
}
override def createAll(filterHeaders: Vector[CompactFilterHeaderDb]): Future[
Vector[CompactFilterHeaderDb]] = {
createAllNoAutoInc(ts = filterHeaders, database = safeDatabase)
@ -106,4 +117,15 @@ case class CompactFilterHeaderDAO()(
query
}
def getBestFilter: Future[CompactFilterHeaderDb] = {
val join = table join blockHeaderTable on (_.blockHash === _.hash)
val query = join.groupBy(_._1).map {
case (filter, headers) =>
filter -> headers.map(_._2.chainWork).max
}
safeDatabase
.runVec(query.result)
.map(_.maxBy(_._2.getOrElse(BigInt(0)))._1)
}
}

10
chain/src/main/scala/org/bitcoins/chain/pow/Pow.scala
View File

@ -123,6 +123,16 @@ sealed abstract class Pow {
newTarget
}
}
def getBlockProof(header: BlockHeader): BigInt = {
val target = NumberUtil.targetExpansion(header.nBits)
if (target.isNegative || target.isOverflow) {
BigInt(0)
} else {
(BigInt(1) << 256) / (target.difficulty + BigInt(1))
}
}
}
object Pow extends Pow

1
chain/src/main/scala/org/bitcoins/chain/validation/TipValidation.scala
View File

@ -47,6 +47,7 @@ sealed abstract class TipValidation extends ChainVerificationLogger {
} else {
val headerDb = BlockHeaderDbHelper.fromBlockHeader(
height = currentTip.height + 1,
chainWork = currentTip.chainWork + Pow.getBlockProof(newPotentialTip),
bh = newPotentialTip
)
TipUpdateResult.Success(headerDb)

2
core/src/main/scala/org/bitcoins/core/number/NumberType.scala
View File

@ -45,6 +45,8 @@ sealed abstract class Number[T <: Number[T]]
override def *(factor: BigInt): T = apply(underlying * factor)
override def *(num: T): T = apply(underlying * num.underlying)
def /(num: T): T = apply(underlying / num.underlying)
override def compare(num: T): Int = underlying compare num.underlying
def <<(num: Int): T = this.<<(apply(num))

2
db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala
View File

@ -48,7 +48,7 @@ class DbManagementTest extends BitcoinSAsyncTest with EmbeddedPg {
dbConfig(ProjectType.Chain))
val chainDbManagement = createChainDbManagement(chainAppConfig)
val result = chainDbManagement.migrate()
assert(result == 1)
assert(result == 2)
}
it must "run migrations for wallet db" in {

9
db-commons/src/main/scala/org/bitcoins/db/DbCommonsColumnMappers.scala
View File

@ -67,6 +67,15 @@ class DbCommonsColumnMappers(val profile: JdbcProfile) {
DoubleSha256DigestBE.fromHex
)
implicit val bigIntMapper: BaseColumnType[BigInt] =
MappedColumnType
.base[BigInt, Array[Byte]](_.toByteArray.dropWhile(_ == 0x00),
BigInt(1, _))
implicit val bigIntPostgresMapper: BaseColumnType[BigInt] =
MappedColumnType
.base[BigInt, BigDecimal](BigDecimal(_), _.toBigInt)
implicit val ecPublicKeyMapper: BaseColumnType[ECPublicKey] =
MappedColumnType.base[ECPublicKey, String](_.hex, ECPublicKey.fromHex)

15
node/src/main/scala/org/bitcoins/node/Node.scala
View File

@ -11,6 +11,7 @@ import org.bitcoins.chain.models.{
import org.bitcoins.core.api.{ChainQueryApi, NodeApi}
import org.bitcoins.core.p2p.{NetworkPayload, TypeIdentifier}
import org.bitcoins.core.protocol.transaction.Transaction
import org.bitcoins.core.util.FutureUtil
import org.bitcoins.crypto.{DoubleSha256Digest, DoubleSha256DigestBE}
import org.bitcoins.node.config.NodeAppConfig
import org.bitcoins.node.models.{
@ -126,6 +127,14 @@ trait Node extends NodeApi with ChainQueryApi with P2PLogger {
val start = System.currentTimeMillis()
for {
_ <- nodeAppConfig.initialize()
// get chainApi so we don't need to call chainApiFromDb on every call
chainApi <- chainApiFromDb
isMissingChainWork <- chainApi.isMissingChainWork
_ <- if (isMissingChainWork) {
chainApi.recalculateChainWork
} else FutureUtil.unit
node <- {
val isInitializedF = for {
_ <- peerMsgSenderF.map(_.connect())
@ -136,14 +145,13 @@ trait Node extends NodeApi with ChainQueryApi with P2PLogger {
logger.error(s"Failed to connect with peer=$peer with err=${err}"))
isInitializedF.map { _ =>
logger.info(s"Our peer=${peer} has been initialized")
logger.info(s"Our peer=$peer has been initialized")
logger.info(s"Our node has been full started. It took=${System
.currentTimeMillis() - start}ms")
this
}
}
// get chainApi so we don't need to call chainApiFromDb on every call
chainApi <- chainApiFromDb
bestHash <- chainApi.getBestBlockHash()
bestHeight <- chainApi.getBestHashBlockHeight()
filterCount <- chainApi.getFilterCount
@ -195,6 +203,7 @@ trait Node extends NodeApi with ChainQueryApi with P2PLogger {
header <- chainApi
.getHeader(hash)
.map(_.get) // .get is safe since this is an internal call
} yield {
peerMsgSenderF.map(_.sendGetHeadersMessage(hash.flip))
logger.info(s"Starting sync node, height=${header.height} hash=$hash")

13
testkit/src/main/scala/org/bitcoins/testkit/chain/BlockHeaderHelper.scala
View File

@ -1,6 +1,7 @@
package org.bitcoins.testkit.chain
import org.bitcoins.chain.models.{BlockHeaderDb, BlockHeaderDbHelper}
import org.bitcoins.chain.pow.Pow
import org.bitcoins.chain.validation.TipValidation
import org.bitcoins.core.number.{Int32, UInt32}
import org.bitcoins.core.protocol.blockchain.BlockHeader
@ -29,7 +30,9 @@ abstract class BlockHeaderHelper {
}
val header1Db: BlockHeaderDb = {
BlockHeaderDbHelper.fromBlockHeader(566093, header1)
BlockHeaderDbHelper.fromBlockHeader(566093,
Pow.getBlockProof(header1),
header1)
}
/**
@ -44,7 +47,8 @@ abstract class BlockHeaderHelper {
}
val header2Db: BlockHeaderDb = {
BlockHeaderDbHelper.fromBlockHeader(566092, header2)
val chainWork = header1Db.chainWork + Pow.getBlockProof(header2)
BlockHeaderDbHelper.fromBlockHeader(566092, chainWork, header2)
}
lazy val twoValidHeaders: Vector[BlockHeader] = {
@ -129,7 +133,10 @@ abstract class BlockHeaderHelper {
if (TipValidation.isBadNonce(blockHeader)) {
buildNextHeader(prevHeader)
} else {
BlockHeaderDbHelper.fromBlockHeader(prevHeader.height + 1, blockHeader)
val chainWork = prevHeader.chainWork + Pow.getBlockProof(blockHeader)
BlockHeaderDbHelper.fromBlockHeader(prevHeader.height + 1,
chainWork,
blockHeader)
}
}
}

28
testkit/src/main/scala/org/bitcoins/testkit/chain/ChainTestUtil.scala
View File

@ -1,6 +1,7 @@
package org.bitcoins.testkit.chain
import org.bitcoins.chain.models._
import org.bitcoins.chain.pow.Pow
import org.bitcoins.core.gcs.{BlockFilter, FilterHeader, GolombFilter}
import org.bitcoins.core.protocol.blockchain.{
BlockHeader,
@ -15,7 +16,10 @@ sealed abstract class ChainTestUtil {
lazy val regTestHeader: BlockHeader =
regTestChainParams.genesisBlock.blockHeader
lazy val regTestGenesisHeaderDb: BlockHeaderDb = {
BlockHeaderDbHelper.fromBlockHeader(height = 0, bh = regTestHeader)
BlockHeaderDbHelper.fromBlockHeader(height = 0,
chainWork =
Pow.getBlockProof(regTestHeader),
bh = regTestHeader)
}
lazy val regTestGenesisHeaderCompactFilter: GolombFilter =
BlockFilter.apply(regTestChainParams.genesisBlock, Vector.empty)
@ -55,26 +59,36 @@ sealed abstract class ChainTestUtil {
"000000200cd536b3eb1cd9c028e081f1455006276b293467c3e5170000000000000000007bc1b27489db01c85d38a4bc6d2280611e9804f506d83ad00d2a33ebd663992f76c7725c505b2e174fb90f55")
lazy val blockHeaderDb564480 =
BlockHeaderDbHelper.fromBlockHeader(564480, blockHeader564480)
BlockHeaderDbHelper.fromBlockHeader(564480,
Pow.getBlockProof(blockHeader564480),
blockHeader564480)
lazy val blockHeader566494 = BlockHeader.fromHex(
"00000020ea2cb07d670ddb7a158e72ddfcfd9e1b9bf4459278bb240000000000000000004fb33054d79de69bb84b4d5c7dd87d80473c416320427a882c72108f7e43fd0c3d3e855c505b2e178f328fe2")
lazy val blockHeaderDb566494 =
BlockHeaderDbHelper.fromBlockHeader(566594, blockHeader566494)
BlockHeaderDbHelper.fromBlockHeader(566594,
Pow.getBlockProof(blockHeader566494),
blockHeader566494)
lazy val blockHeader566495 = BlockHeader.fromHex(
"000000202164d8c4e5246ab003fdebe36c697b9418aa454ec4190d00000000000000000059134ad5aaad38a0e75946c7d4cb09b3ad45b459070195dd564cde193cf0ef29c33e855c505b2e17f61af734")
lazy val blockHeaderDb566495 =
BlockHeaderDbHelper.fromBlockHeader(566495, blockHeader566495)
lazy val blockHeaderDb566495 = {
val chainWork = blockHeaderDb566494.chainWork + Pow.getBlockProof(
blockHeader566495)
BlockHeaderDbHelper.fromBlockHeader(566495, chainWork, blockHeader566495)
}
//https://blockstream.info/block/00000000000000000015fea169c62eb0a1161aba36932ca32bc3785cbb3480bf
lazy val blockHeader566496 = BlockHeader.fromHex(
"000000201b61e8961710991a47ff8187d946d93e4fb33569c09622000000000000000000d0098658f53531e6e67fc9448986b5a8f994da42d746079eabe10f55e561e243103f855c17612e1735c4afdb")
lazy val blockHeaderDb566496 =
BlockHeaderDbHelper.fromBlockHeader(566496, blockHeader566496)
lazy val blockHeaderDb566496 = {
val chainWork = blockHeaderDb566495.chainWork + Pow.getBlockProof(
blockHeader566496)
BlockHeaderDbHelper.fromBlockHeader(566496, chainWork, blockHeader566496)
}
}
}

14
testkit/src/main/scala/org/bitcoins/testkit/chain/ChainUnitTest.scala
View File

@ -10,6 +10,7 @@ import org.bitcoins.chain.blockchain.ChainHandler
import org.bitcoins.chain.blockchain.sync.ChainSync
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.chain.models._
import org.bitcoins.chain.pow.Pow
import org.bitcoins.core.protocol.blockchain.{Block, BlockHeader}
import org.bitcoins.crypto.DoubleSha256DigestBE
import org.bitcoins.db.AppConfig
@ -355,9 +356,20 @@ object ChainUnitTest extends ChainVerificationLogger {
logger.error(s"Failed to parse headers from block_headers.json: $err")
Future.failed(new RuntimeException(err.toString))
case JsSuccess(headers, _) =>
var prevHeaderOpt: Option[BlockHeaderDb] = None
val dbHeaders = headers.zipWithIndex.map {
case (header, height) =>
BlockHeaderDbHelper.fromBlockHeader(height + OFFSET, header)
val chainWork = prevHeaderOpt match {
case None => Pow.getBlockProof(header)
case Some(prevHeader) =>
prevHeader.chainWork + Pow.getBlockProof(header)
}
val newHeader = BlockHeaderDbHelper.fromBlockHeader(height + OFFSET,
chainWork,
header)
prevHeaderOpt = Some(newHeader)
newHeader
}
@tailrec