2024 10 23 merkle vector (#5734)

* Rework MerkleBlock/PartialMerkleTree data structures to use Vector

* Fix base case for Merkle.build()
This commit is contained in:
Chris Stewart 2024-10-23 16:14:33 -05:00 committed by GitHub
parent 07f17cfedf
commit e419b18d9c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 105 additions and 95 deletions

View File

@ -20,7 +20,7 @@ class MerkleTest extends BitcoinSUnitTest {
}
it must "fail to compute the merkle root for a block which has 0 transactions" in {
val transactions = Seq()
val transactions = Vector.empty
Try(Merkle.computeMerkleRoot(transactions)).isFailure must be(true)
}
@ -44,7 +44,7 @@ class MerkleTest extends BitcoinSUnitTest {
)
)
val transactions = Seq(coinbaseTx, tx1)
val transactions = Vector(coinbaseTx, tx1)
Merkle.computeMerkleRoot(transactions).hex must be(
BytesUtil.flipEndianness(
"8fb300e3fdb6f30a4c67233b997f99fdd518b968b9a3fd65857bfe78b2600719"
@ -78,7 +78,7 @@ class MerkleTest extends BitcoinSUnitTest {
"d8c9d6a13a7fb8236833b1e93d298f4626deeb78b2f1814aa9a779961c08ce39"
)
)
val transactions = Seq(coinbaseTx, tx1, tx2)
val transactions = Vector(coinbaseTx, tx1, tx2)
Merkle.computeMerkleRoot(transactions).hex must be(
BytesUtil.flipEndianness(
@ -103,7 +103,7 @@ class MerkleTest extends BitcoinSUnitTest {
"01000000010b6072b386d4a773235237f64c1126ac3b240c84b917a3909ba1c43ded5f51f4000000008c493046022100bb1ad26df930a51cce110cf44f7a48c3c561fd977500b1ae5d6b6fd13d0b3f4a022100c5b42951acedff14abba2736fd574bdb465f3e6f8da12e2c5303954aca7f78f3014104a7135bfe824c97ecc01ec7d7e336185c81e2aa2c41ab175407c09484ce9694b44953fcb751206564a9c24dd094d42fdbfdd5aad3e063ce6af4cfaaea4ea14fbbffffffff0140420f00000000001976a91439aa3d569e06a1d7926dc4be1193c99bf2eb9ee088ac00000000"
)
val transactions = Seq(coinbaseTx, tx1, tx2, tx3)
val transactions = Vector(coinbaseTx, tx1, tx2, tx3)
Merkle.computeMerkleRoot(transactions).hex must be(
BytesUtil.flipEndianness(
@ -156,7 +156,7 @@ class MerkleTest extends BitcoinSUnitTest {
)
)
val transactions = Seq(coinbaseTx, tx1, tx2, tx3, tx4)
val transactions = Vector(coinbaseTx, tx1, tx2, tx3, tx4)
Merkle.computeMerkleRoot(transactions).hex must be(
BytesUtil.flipEndianness(
"36b38854f9adf76b4646ab2c0f949846408cfab2c045f110d01f84f4122c5add"
@ -256,7 +256,7 @@ class MerkleTest extends BitcoinSUnitTest {
)
val transactions =
Seq(coinbaseTx, tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8, tx9, tx10)
Vector(coinbaseTx, tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8, tx9, tx10)
Merkle.computeMerkleRoot(transactions).hex must be(
BytesUtil.flipEndianness(

View File

@ -463,17 +463,15 @@ class MerkleBlockTests extends BitcoinSUnitTest {
// TODO: This is *extremely* slow, this is currently the longest running property we have taking about 6 minutes to run
// I think it is the generator MerkleGenerator.merkleBlockWithInsertTxIds
it must "contains all inserted txids when we directly create a merkle block from the txids && " +
"contains all txids matched by a bloom filter && " +
"serialization symmetry" in {
forAll(MerkleGenerator.merkleBlockWithInsertedTxIds) {
case (merkleBlock: MerkleBlock, _, txIds: Seq[DoubleSha256Digest]) =>
val extractedMatches = merkleBlock.partialMerkleTree.extractMatches
assert(
extractedMatches == txIds &&
extractedMatches.intersect(txIds) == txIds &&
MerkleBlock(merkleBlock.hex) == merkleBlock
)
}
it must "contains all inserted txids when we directly create a merkle block from the txids" in {
forAll(MerkleGenerator.merkleBlockWithInsertedTxIds) {
case (merkleBlock: MerkleBlock, _, txIds: Seq[DoubleSha256Digest]) =>
val extractedMatches = merkleBlock.partialMerkleTree.extractMatches
assert(
extractedMatches == txIds &&
extractedMatches.intersect(txIds) == txIds &&
MerkleBlock(merkleBlock.hex) == merkleBlock
)
}
}
}

View File

@ -225,7 +225,7 @@ class PartialMerkleTreeTests extends BitcoinSUnitTest {
}
it must "build a partial merkle tree with no matches and 1 transaction in the original block" in {
val txMatches = Seq(
val txMatches = Vector(
(
false,
DoubleSha256Digest(
@ -249,7 +249,7 @@ class PartialMerkleTreeTests extends BitcoinSUnitTest {
}
it must "build a partial merkle tree with 1 match and 2 transactions inside the original block" in {
val txMatches = Seq(
val txMatches = Vector(
(
false,
DoubleSha256Digest(
@ -294,7 +294,7 @@ class PartialMerkleTreeTests extends BitcoinSUnitTest {
}
it must "calculate bits correctly for a tree of height 1" in {
val matches = List(
val matches = Vector(
(
true,
DoubleSha256Digest(
@ -318,7 +318,7 @@ class PartialMerkleTreeTests extends BitcoinSUnitTest {
it must "correctly compute a merkle tree that has an odd amount of txids on the merkle tree" in {
// this test is meant to prevent these failures on travis ci
// https://travis-ci.org/bitcoin-s/bitcoin-s-core/builds/205812075#L2774
val hashes: Seq[DoubleSha256Digest] = List(
val hashes: Vector[DoubleSha256Digest] = Vector(
DoubleSha256Digest(
"1563b82f187da1067f5000dabe3a4f4ae8650e207aa163e1d25ded8175e2bae1"
),

View File

@ -40,7 +40,7 @@ class RawMerkleBlockSerializerTest extends BitcoinSUnitTest {
BitVector.bits(
Vector(false, false, false, false, false,
false, false, false)),
List(DoubleSha256Digest(
Vector(DoubleSha256Digest(
"442abdc8e74ad35ebd9571f88fda91ff511dcda8d241a5aed52cea1e00d69e03"))
)
),
@ -97,7 +97,7 @@ class RawMerkleBlockSerializerTest extends BitcoinSUnitTest {
List(true, true, true, false, true, true,
false, true, false, false, false, false,
false, false, false, false)),
List(
Vector(
DoubleSha256Digest(
"e4aeaf729035a7fb939e12c4f6a2072a9b2e7da784207ce7852d398593210a45"),
DoubleSha256Digest(

View File

@ -34,17 +34,20 @@ trait Merkle {
* @return
* the merkle root for the sequence of transactions
*/
def computeMerkleRoot(transactions: Seq[Transaction]): DoubleSha256Digest =
transactions match {
case Nil =>
throw new IllegalArgumentException(
"We cannot have zero transactions in the block. There always should be ATLEAST one - the coinbase tx")
case h +: Nil => h.txId
case _ +: _ =>
val leafs = transactions.map(tx => LeafDoubleSha256Digest(tx.txId))
val merkleTree = build(leafs, Nil)
merkleTree.value.get
def computeMerkleRoot(
transactions: Vector[Transaction]): DoubleSha256Digest = {
val result = if (transactions.isEmpty) {
throw new IllegalArgumentException(
"We cannot have zero transactions in the block. There always should be ATLEAST one - the coinbase tx")
} else if (transactions.length == 1) {
transactions.head.txId
} else {
val leafs = transactions.map(tx => LeafDoubleSha256Digest(tx.txId))
val merkleTree = build(leafs, Vector.empty)
merkleTree.value.get
}
result
}
/** Builds a [[MerkleTree]] from sequence of sub merkle trees. This subTrees
* can be individual txids (leafs) or full blown subtrees
@ -57,28 +60,30 @@ trait Merkle {
*/
@tailrec
final def build(
subTrees: Seq[MerkleTree],
accum: Seq[MerkleTree]): MerkleTree =
subTrees match {
case Nil =>
if (accum.size == 1) accum.head
else if (accum.isEmpty)
throw new IllegalArgumentException(
"Should never have sub tree size of zero, this implies there was zero hashes given")
else build(accum.reverse, Nil)
case h +: h1 +: t =>
val newTree = computeTree(h, h1)
build(t, newTree +: accum)
case h +: t =>
// means that we have an odd amount of txids, this means we duplicate the last hash in the tree
val newTree = computeTree(h, h)
build(t, newTree +: accum)
subTrees: Vector[MerkleTree],
accum: Vector[MerkleTree]): MerkleTree = {
if (subTrees.isEmpty) {
if (accum.length == 1) accum.head
else if (accum.isEmpty) {
throw new IllegalArgumentException(
"Should never have sub tree size of zero, this implies there was zero hashes given")
} else {
build(accum.reverse, Vector.empty)
}
} else if (subTrees.length >= 2) {
val newTree = computeTree(subTrees.head, subTrees(1))
build(subTrees.drop(2), newTree +: accum)
} else {
// means that we have an odd amount of txids, this means we duplicate the last hash in the tree
val newTree = computeTree(subTrees.head, subTrees.head)
build(subTrees.drop(1), newTree +: accum)
}
}
/** Builds a merkle tree from a sequence of hashes */
def build(hashes: Seq[DoubleSha256Digest]): MerkleTree = {
def build(hashes: Vector[DoubleSha256Digest]): MerkleTree = {
val leafs = hashes.map(LeafDoubleSha256Digest.apply)
build(leafs, Nil)
build(leafs, Vector.empty)
}
/** Computes the merkle tree of two sub merkle trees */

View File

@ -173,7 +173,7 @@ sealed abstract class ChainParams {
TransactionConstants.lockTime)
val prevBlockHash = DoubleSha256Digest(
"0000000000000000000000000000000000000000000000000000000000000000")
val merkleRootHash = Merkle.computeMerkleRoot(Seq(tx))
val merkleRootHash = Merkle.computeMerkleRoot(Vector(tx))
val genesisBlockHeader =
BlockHeader(version, prevBlockHash, merkleRootHash, time, nBits, nonce)
val genesisBlock = Block(genesisBlockHeader, Vector(tx))

View File

@ -26,7 +26,7 @@ sealed abstract class MerkleBlock extends NetworkElement {
/** One or more hashes of both transactions and merkle nodes used to build the
* partial merkle tree
*/
def hashes: Seq[DoubleSha256Digest] = partialMerkleTree.hashes
def hashes: Vector[DoubleSha256Digest] = partialMerkleTree.hashes
/** The
* [[org.bitcoins.core.protocol.blockchain.PartialMerkleTree PartialMerkleTree]]
@ -34,7 +34,7 @@ sealed abstract class MerkleBlock extends NetworkElement {
*/
def partialMerkleTree: PartialMerkleTree
def bytes = RawMerkleBlockSerializer.write(this)
override def bytes: ByteVector = RawMerkleBlockSerializer.write(this)
}
object MerkleBlock extends Factory[MerkleBlock] {
@ -64,10 +64,10 @@ object MerkleBlock extends Factory[MerkleBlock] {
def apply(block: Block, filter: BloomFilter): (MerkleBlock, BloomFilter) = {
@tailrec
def loop(
remainingTxs: Seq[Transaction],
remainingTxs: Vector[Transaction],
accumFilter: BloomFilter,
txMatches: Seq[(Boolean, DoubleSha256Digest)])
: (Seq[(Boolean, DoubleSha256Digest)], BloomFilter) = {
txMatches: Vector[(Boolean, DoubleSha256Digest)])
: (Vector[(Boolean, DoubleSha256Digest)], BloomFilter) = {
if (remainingTxs.isEmpty) (txMatches.reverse, accumFilter)
else {
val tx = remainingTxs.head
@ -76,7 +76,7 @@ object MerkleBlock extends Factory[MerkleBlock] {
loop(remainingTxs.tail, newFilter, newTxMatches)
}
}
val (matchedTxs, newFilter) = loop(block.transactions, filter, Nil)
val (matchedTxs, newFilter) = loop(block.transactions, filter, Vector.empty)
val partialMerkleTree = PartialMerkleTree(matchedTxs)
val txCount = UInt32(block.transactions.size)
(MerkleBlock(block.blockHeader, txCount, partialMerkleTree), newFilter)
@ -90,9 +90,9 @@ object MerkleBlock extends Factory[MerkleBlock] {
// https://github.com/bitcoin/bitcoin/blob/master/src/merkleblock.cpp#L40
@tailrec
def loop(
remainingTxs: Seq[Transaction],
txMatches: Seq[(Boolean, DoubleSha256Digest)])
: (Seq[(Boolean, DoubleSha256Digest)]) = {
remainingTxs: Vector[Transaction],
txMatches: Vector[(Boolean, DoubleSha256Digest)])
: (Vector[(Boolean, DoubleSha256Digest)]) = {
if (remainingTxs.isEmpty) txMatches.reverse
else {
val tx = remainingTxs.head
@ -101,7 +101,7 @@ object MerkleBlock extends Factory[MerkleBlock] {
}
}
val txMatches = loop(block.transactions, Nil)
val txMatches = loop(block.transactions, Vector.empty)
val partialMerkleTree = PartialMerkleTree(txMatches)
val txCount = UInt32(block.transactions.size)
@ -118,7 +118,7 @@ object MerkleBlock extends Factory[MerkleBlock] {
def apply(
blockHeader: BlockHeader,
txCount: UInt32,
hashes: Seq[DoubleSha256Digest],
hashes: Vector[DoubleSha256Digest],
bits: BitVector): MerkleBlock = {
val partialMerkleTree = PartialMerkleTree(txCount, hashes, bits)
MerkleBlock(blockHeader, txCount, partialMerkleTree)

View File

@ -51,7 +51,7 @@ sealed trait PartialMerkleTree {
def bits: BitVector
/** The hashes used to create the binary tree */
def hashes: Seq[DoubleSha256Digest]
def hashes: Vector[DoubleSha256Digest]
/** Extracts the txids that were matched inside of the bloom filter used to
* create this partial merkle tree
@ -63,8 +63,8 @@ sealed trait PartialMerkleTree {
remainingBits: BitVector,
height: Int,
pos: Int,
accumMatches: Seq[DoubleSha256Digest])
: (Seq[DoubleSha256Digest], BitVector) = {
accumMatches: Vector[DoubleSha256Digest])
: (Vector[DoubleSha256Digest], BitVector) = {
if (height == maxHeight)
extractLeafMatch(accumMatches, remainingBits, subTree)
else {
@ -104,7 +104,7 @@ sealed trait PartialMerkleTree {
} else (accumMatches, remainingBits.tail)
}
}
val (matches, remainingBits) = loop(tree, bits, 0, 0, Nil)
val (matches, remainingBits) = loop(tree, bits, 0, 0, Vector.empty)
require(
PartialMerkleTree.usedAllBits(bits, remainingBits),
"We should not have any remaining matches " +
@ -117,10 +117,10 @@ sealed trait PartialMerkleTree {
* merkle tree
*/
private def extractLeafMatch(
accumMatches: Seq[DoubleSha256Digest],
accumMatches: Vector[DoubleSha256Digest],
remainingBits: BitVector,
subTree: BinaryTreeDoubleSha256Digest)
: (Seq[DoubleSha256Digest], BitVector) = {
: (Vector[DoubleSha256Digest], BitVector) = {
if (remainingBits.head) {
// means we have a txid node that matched the filter
subTree match {
@ -144,14 +144,14 @@ object PartialMerkleTree {
tree: BinaryTreeDoubleSha256Digest,
transactionCount: UInt32,
bits: BitVector,
hashes: Seq[DoubleSha256Digest])
hashes: Vector[DoubleSha256Digest])
extends PartialMerkleTree {
require(bits.size % 8 == 0,
"As per BIP37, bits must be padded to the nearest byte")
}
def apply(
txMatches: Seq[(Boolean, DoubleSha256Digest)]): PartialMerkleTree = {
txMatches: Vector[(Boolean, DoubleSha256Digest)]): PartialMerkleTree = {
val txIds = txMatches.map(_._2)
val (bits, hashes) = build(txMatches)
val tree = reconstruct(txIds.size, hashes, bits)
@ -167,8 +167,8 @@ object PartialMerkleTree {
* to reconstruct this partial merkle tree, and the hashes needed to be
* inserted according to the flags inside of bits
*/
private def build(txMatches: Seq[(Boolean, DoubleSha256Digest)])
: (BitVector, Seq[DoubleSha256Digest]) = {
private def build(txMatches: Vector[(Boolean, DoubleSha256Digest)])
: (BitVector, Vector[DoubleSha256Digest]) = {
val maxHeight = calcMaxHeight(txMatches.size)
/** This loops through our merkle tree building `bits` so we can instruct
@ -192,9 +192,9 @@ object PartialMerkleTree {
*/
def loop(
bits: BitVector,
hashes: Seq[DoubleSha256Digest],
hashes: Vector[DoubleSha256Digest],
height: Int,
pos: Int): (BitVector, Seq[DoubleSha256Digest]) = {
pos: Int): (BitVector, Vector[DoubleSha256Digest]) = {
val parentOfMatch =
matchesTx(maxHeight, maxHeight - height, pos, txMatches)
val newBits = parentOfMatch +: bits
@ -213,7 +213,7 @@ object PartialMerkleTree {
} else (leftBits, leftHashes)
}
}
val (bits, hashes) = loop(BitVector.empty, Nil, maxHeight, 0)
val (bits, hashes) = loop(BitVector.empty, Vector.empty, maxHeight, 0)
// pad the bit array to the nearest byte as required by BIP37
val bitsNeeded =
if (bits.size % 8 == 0) 0 else (8 - (bits.size % 8)) + bits.size
@ -276,7 +276,7 @@ object PartialMerkleTree {
*/
def apply(
transactionCount: UInt32,
hashes: Seq[DoubleSha256Digest],
hashes: Vector[DoubleSha256Digest],
bits: BitVector): PartialMerkleTree = {
val tree = reconstruct(transactionCount.toInt, hashes, bits)
PartialMerkleTree(tree, transactionCount, bits, hashes)
@ -299,7 +299,7 @@ object PartialMerkleTree {
tree: BinaryTreeDoubleSha256Digest,
transactionCount: UInt32,
bits: BitVector,
hashes: Seq[DoubleSha256Digest]): PartialMerkleTree = {
hashes: Vector[DoubleSha256Digest]): PartialMerkleTree = {
PartialMerkleTreeImpl(tree, transactionCount, bits, hashes)
}
@ -309,16 +309,17 @@ object PartialMerkleTree {
*/
private def reconstruct(
numTransaction: Int,
hashes: Seq[DoubleSha256Digest],
hashes: Vector[DoubleSha256Digest],
bits: BitVector): BinaryTreeDoubleSha256Digest = {
val maxHeight = calcMaxHeight(numTransaction)
// TODO: Optimize to tailrec function
def loop(
remainingHashes: Seq[DoubleSha256Digest],
remainingHashes: Vector[DoubleSha256Digest],
remainingMatches: BitVector,
height: Int,
pos: Int)
: (BinaryTreeDoubleSha256Digest, Seq[DoubleSha256Digest], BitVector) = {
pos: Int): (BinaryTreeDoubleSha256Digest,
Vector[DoubleSha256Digest],
BitVector) = {
if (height == maxHeight) {
// means we have a txid node
(LeafDoubleSha256Digest(remainingHashes.head),

View File

@ -74,22 +74,22 @@ sealed abstract class RawMerkleBlockSerializer
*/
private def parseTransactionHashes(
bytes: ByteVector,
hashCount: CompactSizeUInt): (Seq[DoubleSha256Digest], ByteVector) = {
hashCount: CompactSizeUInt): (Vector[DoubleSha256Digest], ByteVector) = {
@tailrec
def loop(
remainingHashes: Long,
remainingBytes: ByteVector,
accum: List[DoubleSha256Digest])
: (Seq[DoubleSha256Digest], ByteVector) = {
accum: Vector[DoubleSha256Digest])
: (Vector[DoubleSha256Digest], ByteVector) = {
if (remainingHashes <= 0) (accum.reverse, remainingBytes)
else {
val (hashBytes, newRemainingBytes) = remainingBytes.splitAt(32)
loop(remainingHashes - 1,
newRemainingBytes,
DoubleSha256Digest(hashBytes) :: accum)
DoubleSha256Digest(hashBytes) +: accum)
}
}
loop(hashCount.num.toInt, bytes, Nil)
loop(hashCount.num.toInt, bytes, Vector.empty)
}
}

View File

@ -66,7 +66,9 @@ sealed abstract class BlockchainElementsGenerator {
): Gen[BlockHeader] =
for {
numTxs <- Gen.choose(1, 5)
txs <- Gen.listOfN(numTxs, TransactionGenerators.transaction)
txs <- Gen
.listOfN(numTxs, TransactionGenerators.transaction)
.map(_.toVector)
header <- blockHeader(previousBlockHash, nBits, txs)
} yield header
@ -77,7 +79,7 @@ sealed abstract class BlockchainElementsGenerator {
def blockHeader(
previousBlockHash: DoubleSha256Digest,
nBits: UInt32,
txs: Seq[Transaction]
txs: Vector[Transaction]
): Gen[BlockHeader] =
for {
version <- NumberGenerator.int32s
@ -97,7 +99,7 @@ sealed abstract class BlockchainElementsGenerator {
* [[org.bitcoins.core.protocol.blockchain.BlockHeader BlockHeader]] that has
* a merkle root hash corresponding to the given txs
*/
def blockHeader(txs: Seq[Transaction]): Gen[BlockHeader] =
def blockHeader(txs: Vector[Transaction]): Gen[BlockHeader] =
for {
previousBlockHash <- CryptoGenerators.doubleSha256Digest
nBits <- NumberGenerator.uInt32s

View File

@ -24,7 +24,9 @@ abstract class MerkleGenerator {
block <- BlockchainElementsGenerator.block(txs)
txIds = txs.map(_.txId)
merkleBlock = MerkleBlock(block, txIds)
} yield (merkleBlock, block, txIds)
} yield {
(merkleBlock, block, txIds)
}
/** Returns a
* [[org.bitcoins.core.protocol.blockchain.MerkleBlock MerkleBlock]]
@ -65,7 +67,7 @@ abstract class MerkleGenerator {
* indicating if the txid was matched
*/
def partialMerkleTree
: Gen[(PartialMerkleTree, Seq[(Boolean, DoubleSha256Digest)])] =
: Gen[(PartialMerkleTree, Vector[(Boolean, DoubleSha256Digest)])] =
for {
randomNum <- Gen.choose(1, 25)
txMatches <- txIdsWithMatchIndication(randomNum)
@ -87,8 +89,10 @@ abstract class MerkleGenerator {
*/
def txIdsWithMatchIndication(
num: Int
): Gen[Seq[(Boolean, DoubleSha256Digest)]] =
Gen.listOfN(num, txIdWithMatchIndication)
): Gen[Vector[(Boolean, DoubleSha256Digest)]] =
Gen
.listOfN(num, txIdWithMatchIndication)
.map(_.toVector)
}
object MerkleGenerator extends MerkleGenerator