diff --git a/core/src/main/java/com/google/bitcoin/core/BloomFilter.java b/core/src/main/java/com/google/bitcoin/core/BloomFilter.java index c19ed2ae4..b5b61419c 100644 --- a/core/src/main/java/com/google/bitcoin/core/BloomFilter.java +++ b/core/src/main/java/com/google/bitcoin/core/BloomFilter.java @@ -16,11 +16,15 @@ package com.google.bitcoin.core; +import com.google.bitcoin.script.Script; +import com.google.bitcoin.script.ScriptChunk; import com.google.common.base.Objects; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.Math.*; @@ -157,9 +161,12 @@ public class BloomFilter extends Message { private static int rotateLeft32(int x, int r) { return (x << r) | (x >>> (32 - r)); } - - private int hash(int hashNum, byte[] object) { - // The following is MurmurHash3 (x86_32), see http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp + + /** + * Applies the MurmurHash3 (x86_32) algorithm to the given data. + * See this C++ code for the original. + */ + public static int murmurHash3(byte[] data, long nTweak, int hashNum, byte[] object) { int h1 = (int)(hashNum * 0xFBA4C795L + nTweak); final int c1 = 0xcc9e2d51; final int c2 = 0x1b873593; @@ -214,22 +221,22 @@ public class BloomFilter extends Message { * Returns true if the given object matches the filter either because it was inserted, or because we have a * false-positive. */ - public boolean contains(byte[] object) { + public synchronized boolean contains(byte[] object) { for (int i = 0; i < hashFuncs; i++) { - if (!Utils.checkBitLE(data, hash(i, object))) + if (!Utils.checkBitLE(data, murmurHash3(data, nTweak, i, object))) return false; } return true; } /** Insert the given arbitrary data into the filter */ - public void insert(byte[] object) { + public synchronized void insert(byte[] object) { for (int i = 0; i < hashFuncs; i++) - Utils.setBitLE(data, hash(i, object)); + Utils.setBitLE(data, murmurHash3(data, nTweak, i, object)); } /** Inserts the given key and equivalent hashed form (for the address). */ - public void insert(ECKey key) { + public synchronized void insert(ECKey key) { insert(key.getPubKey()); insert(key.getPubKeyHash()); } @@ -241,7 +248,7 @@ public class BloomFilter extends Message { * Solved blocks will then be send just as Merkle trees of tx hashes, meaning a constant 32 bytes of data for each * transaction instead of 100-300 bytes as per usual. */ - public void setMatchAll() { + public synchronized void setMatchAll() { data = new byte[] {(byte) 0xff}; } @@ -249,7 +256,7 @@ public class BloomFilter extends Message { * Copies filter into this. Filter must have the same size, hash function count and nTweak or an * IllegalArgumentException will be thrown. */ - public void merge(BloomFilter filter) { + public synchronized void merge(BloomFilter filter) { if (!this.matchesAll() && !filter.matchesAll()) { checkArgument(filter.data.length == this.data.length && filter.hashFuncs == this.hashFuncs && @@ -265,15 +272,69 @@ public class BloomFilter extends Message { * Returns true if this filter will match anything. See {@link com.google.bitcoin.core.BloomFilter#setMatchAll()} * for when this can be a useful thing to do. */ - public boolean matchesAll() { + public synchronized boolean matchesAll() { for (byte b : data) if (b != (byte) 0xff) return false; return true; } + + /** + * The update flag controls how application of the filter to a block modifies the filter. See the enum javadocs + * for information on what occurs and when. + */ + public synchronized BloomUpdate getUpdateFlag() { + if (nFlags == 0) + return BloomUpdate.UPDATE_NONE; + else if (nFlags == 1) + return BloomUpdate.UPDATE_ALL; + else if (nFlags == 2) + return BloomUpdate.UPDATE_P2PUBKEY_ONLY; + else + throw new IllegalStateException("Unknown flag combination"); + } + + /** + * Creates a new FilteredBlock from the given Block, using this filter to select transactions. Matches can cause the + * filter to be updated with the matched element, this ensures that when a filter is applied to a block, spends of + * matched transactions are also matched. However it means this filter can be mutated by the operation. + */ + public synchronized FilteredBlock applyAndUpdate(Block block) { + List txns = block.getTransactions(); + List txHashes = new ArrayList(txns.size()); + byte[] bits = new byte[(int) Math.ceil(txns.size() / 8.0)]; + for (int i = 0; i < txns.size(); i++) { + txHashes.add(txns.get(i).getHash()); + if (applyAndUpdate(txns.get(i))) + Utils.setBitLE(bits, i); + } + PartialMerkleTree pmt = PartialMerkleTree.buildFromLeaves(block.getParams(), bits, txHashes); + return new FilteredBlock(block.getParams(), block.cloneAsHeader(), pmt); + } + + public synchronized boolean applyAndUpdate(Transaction tx) { + if (contains(tx.getHash().getBytes())) + return true; + boolean found = false; + BloomUpdate flag = getUpdateFlag(); + for (TransactionOutput output : tx.getOutputs()) { + Script script = output.getScriptPubKey(); + for (ScriptChunk chunk : script.getChunks()) { + if (!chunk.isPushData()) + continue; + if (contains(chunk.data)) { + boolean isSendingToPubKeys = script.isSentToRawPubKey() || script.isSentToMultiSig(); + if (flag == BloomUpdate.UPDATE_ALL || (flag == BloomUpdate.UPDATE_P2PUBKEY_ONLY && isSendingToPubKeys)) + insert(output.getOutPointFor().bitcoinSerialize()); + found = true; + } + } + } + return found; + } @Override - public boolean equals(Object o) { + public synchronized boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; BloomFilter other = (BloomFilter) o; @@ -283,7 +344,7 @@ public class BloomFilter extends Message { } @Override - public int hashCode() { + public synchronized int hashCode() { return Objects.hashCode(hashFuncs, nTweak, Arrays.hashCode(data)); } } diff --git a/core/src/main/java/com/google/bitcoin/core/FilteredBlock.java b/core/src/main/java/com/google/bitcoin/core/FilteredBlock.java index 81da0b1a0..52d30fc3a 100644 --- a/core/src/main/java/com/google/bitcoin/core/FilteredBlock.java +++ b/core/src/main/java/com/google/bitcoin/core/FilteredBlock.java @@ -29,7 +29,6 @@ public class FilteredBlock extends Message { public static final int MIN_PROTOCOL_VERSION = 70000; private Block header; - // The PartialMerkleTree of transactions private PartialMerkleTree merkleTree; private List cachedTransactionHashes = null; @@ -40,7 +39,13 @@ public class FilteredBlock extends Message { public FilteredBlock(NetworkParameters params, byte[] payloadBytes) throws ProtocolException { super(params, payloadBytes, 0); } - + + public FilteredBlock(NetworkParameters params, Block header, PartialMerkleTree pmt) { + super(params); + this.header = header; + this.merkleTree = pmt; + } + @Override public void bitcoinSerializeToStream(OutputStream stream) throws IOException { if (header.transactions == null) diff --git a/core/src/main/java/com/google/bitcoin/core/PartialMerkleTree.java b/core/src/main/java/com/google/bitcoin/core/PartialMerkleTree.java index 37c3d0750..894218a19 100644 --- a/core/src/main/java/com/google/bitcoin/core/PartialMerkleTree.java +++ b/core/src/main/java/com/google/bitcoin/core/PartialMerkleTree.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static com.google.bitcoin.core.Utils.*; + /** *

A data structure that contains proofs of block inclusion for one or more transactions, in an efficient manner.

* @@ -62,14 +64,44 @@ public class PartialMerkleTree extends Message { public PartialMerkleTree(NetworkParameters params, byte[] payloadBytes, int offset) throws ProtocolException { super(params, payloadBytes, offset); } - + + /** + * Constructs a new PMT with the given bit set (little endian) and the raw list of hashes including internal hashes, + * taking ownership of the list. + */ + public PartialMerkleTree(NetworkParameters params, byte[] bits, List hashes, int origTxCount) { + super(params); + this.matchedChildBits = bits; + this.hashes = hashes; + this.transactionCount = origTxCount; + } + + /** + * Calculates a PMT given the list of leaf hashes and which leaves need to be included. The relevant interior hashes + * are calculated and a new PMT returned. + */ + public static PartialMerkleTree buildFromLeaves(NetworkParameters params, byte[] includeBits, List allLeafHashes) { + // Calculate height of the tree. + int height = 0; + while (getTreeWidth(allLeafHashes.size(), height) > 1) + height++; + List bitList = new ArrayList(); + List hashes = new ArrayList(); + traverseAndBuild(height, 0, allLeafHashes, includeBits, bitList, hashes); + byte[] bits = new byte[(int)Math.ceil(bitList.size() / 8.0)]; + for (int i = 0; i < bitList.size(); i++) + if (bitList.get(i)) + Utils.setBitLE(bits, i); + return new PartialMerkleTree(params, bits, hashes, allLeafHashes.size()); + } + @Override public void bitcoinSerializeToStream(OutputStream stream) throws IOException { - Utils.uint32ToByteStreamLE(transactionCount, stream); + uint32ToByteStreamLE(transactionCount, stream); stream.write(new VarInt(hashes.size()).encode()); for (Sha256Hash hash : hashes) - stream.write(Utils.reverseBytes(hash.getBytes())); + stream.write(reverseBytes(hash.getBytes())); stream.write(new VarInt(matchedChildBits.length).encode()); stream.write(matchedChildBits); @@ -86,18 +118,62 @@ public class PartialMerkleTree extends Message { int nFlagBytes = (int) readVarInt(); matchedChildBits = readBytes(nFlagBytes); - + length = cursor - offset; } - + + // Based on CPartialMerkleTree::TraverseAndBuild in Bitcoin Core. + private static void traverseAndBuild(int height, int pos, List allLeafHashes, byte[] includeBits, + List matchedChildBits, List resultHashes) { + boolean parentOfMatch = false; + // Is this node a parent of at least one matched hash? + for (int p = pos << height; p < (pos+1) << height && p < allLeafHashes.size(); p++) { + if (Utils.checkBitLE(includeBits, p)) { + parentOfMatch = true; + break; + } + } + // Store as a flag bit. + matchedChildBits.add(parentOfMatch); + if (height == 0 || !parentOfMatch) { + // If at height 0, or nothing interesting below, store hash and stop. + resultHashes.add(calcHash(height, pos, allLeafHashes)); + } else { + // Otherwise descend into the subtrees. + int h = height - 1; + int p = pos * 2; + traverseAndBuild(h, p, allLeafHashes, includeBits, matchedChildBits, resultHashes); + if (p + 1 < getTreeWidth(allLeafHashes.size(), h)) + traverseAndBuild(h, p + 1, allLeafHashes, includeBits, matchedChildBits, resultHashes); + } + } + + private static Sha256Hash calcHash(int height, int pos, List hashes) { + if (height == 0) { + // Hash at height 0 is just the regular tx hash itself. + return hashes.get(pos); + } + int h = height - 1; + int p = pos * 2; + Sha256Hash left = calcHash(h, p, hashes); + // Calculate right hash if not beyond the end of the array - copy left hash otherwise. + Sha256Hash right; + if (p + 1 < getTreeWidth(hashes.size(), h)) { + right = calcHash(h, p + 1, hashes); + } else { + right = left; + } + return combineLeftRight(left.getBytes(), right.getBytes()); + } + @Override protected void parseLite() { } // helper function to efficiently calculate the number of nodes at given height in the merkle tree - private int getTreeWidth(int height) { - return (transactionCount+(1 << height)-1) >> height; + private static int getTreeWidth(int transactionCount, int height) { + return (transactionCount + (1 << height) - 1) >> height; } private static class ValuesUsed { @@ -111,30 +187,35 @@ public class PartialMerkleTree extends Message { // overflowed the bits array - failure throw new VerificationException("CPartialMerkleTree overflowed its bits array"); } - boolean parentOfMatch = Utils.checkBitLE(matchedChildBits, used.bitsUsed++); + boolean parentOfMatch = checkBitLE(matchedChildBits, used.bitsUsed++); if (height == 0 || !parentOfMatch) { // if at height 0, or nothing interesting below, use stored hash and do not descend if (used.hashesUsed >= hashes.size()) { // overflowed the hash array - failure throw new VerificationException("CPartialMerkleTree overflowed its hash array"); } + Sha256Hash hash = hashes.get(used.hashesUsed++); if (height == 0 && parentOfMatch) // in case of height 0, we have a matched txid - matchedHashes.add(hashes.get(used.hashesUsed)); - return hashes.get(used.hashesUsed++); + matchedHashes.add(hash); + return hash; } else { // otherwise, descend into the subtrees to extract matched txids and hashes - byte[] left = recursiveExtractHashes(height-1, pos*2, used, matchedHashes).getBytes(), right; - if (pos*2+1 < getTreeWidth(height-1)) - right = recursiveExtractHashes(height-1, pos*2+1, used, matchedHashes).getBytes(); + byte[] left = recursiveExtractHashes(height - 1, pos * 2, used, matchedHashes).getBytes(), right; + if (pos * 2 + 1 < getTreeWidth(transactionCount, height-1)) + right = recursiveExtractHashes(height - 1, pos * 2 + 1, used, matchedHashes).getBytes(); else right = left; // and combine them before returning - return new Sha256Hash(Utils.reverseBytes(Utils.doubleDigestTwoBuffers( - Utils.reverseBytes(left), 0, 32, - Utils.reverseBytes(right), 0, 32))); + return combineLeftRight(left, right); } } - + + private static Sha256Hash combineLeftRight(byte[] left, byte[] right) { + return new Sha256Hash(reverseBytes(doubleDigestTwoBuffers( + reverseBytes(left), 0, 32, + reverseBytes(right), 0, 32))); + } + /** * Extracts tx hashes that are in this merkle tree * and returns the merkle root of this tree. @@ -164,7 +245,7 @@ public class PartialMerkleTree extends Message { throw new VerificationException("Got a CPartialMerkleTree with fewer matched bits than hashes"); // calculate height of tree int height = 0; - while (getTreeWidth(height) > 1) + while (getTreeWidth(transactionCount, height) > 1) height++; // traverse the partial tree ValuesUsed used = new ValuesUsed(); diff --git a/core/src/main/java/com/google/bitcoin/core/PeerGroup.java b/core/src/main/java/com/google/bitcoin/core/PeerGroup.java index 2a3ba8a8f..e1572715c 100644 --- a/core/src/main/java/com/google/bitcoin/core/PeerGroup.java +++ b/core/src/main/java/com/google/bitcoin/core/PeerGroup.java @@ -143,7 +143,9 @@ public class PeerGroup extends AbstractExecutionThreadService implements Transac final double rate = checkNotNull(chain).getFalsePositiveRate(); final double target = bloomFilterMerger.getBloomFilterFPRate() * MAX_FP_RATE_INCREASE; if (rate > target) { - log.info("Force update Bloom filter due to high false positive rate ({} vs {})", rate, target); + // TODO: Avoid hitting this path if the remote peer didn't acknowledge applying a new filter yet. + if (log.isDebugEnabled()) + log.debug("Force update Bloom filter due to high false positive rate ({} vs {})", rate, target); recalculateFastCatchupAndFilter(FilterRecalculateMode.FORCE_SEND_FOR_REFRESH); } } diff --git a/core/src/main/java/com/google/bitcoin/testing/FakeTxBuilder.java b/core/src/main/java/com/google/bitcoin/testing/FakeTxBuilder.java index 3f1a5f88f..f29e39254 100644 --- a/core/src/main/java/com/google/bitcoin/testing/FakeTxBuilder.java +++ b/core/src/main/java/com/google/bitcoin/testing/FakeTxBuilder.java @@ -208,4 +208,14 @@ public class FakeTxBuilder { b.solve(); return b; } + + public static Block makeSolvedTestBlock(Block prev, Address to, Transaction... transactions) throws BlockStoreException { + Block b = prev.createNextBlock(to); + // Coinbase tx already exists. + for (Transaction tx : transactions) { + b.addTransaction(tx); + } + b.solve(); + return b; + } } diff --git a/core/src/test/java/com/google/bitcoin/core/FilteredBlockAndPartialMerkleTreeTests.java b/core/src/test/java/com/google/bitcoin/core/FilteredBlockAndPartialMerkleTreeTests.java index 6857bac66..c793788fa 100644 --- a/core/src/test/java/com/google/bitcoin/core/FilteredBlockAndPartialMerkleTreeTests.java +++ b/core/src/test/java/com/google/bitcoin/core/FilteredBlockAndPartialMerkleTreeTests.java @@ -20,6 +20,7 @@ package com.google.bitcoin.core; import com.google.bitcoin.core.TransactionConfidence.ConfidenceType; import com.google.bitcoin.params.UnitTestParams; import com.google.bitcoin.store.MemoryBlockStore; +import com.google.bitcoin.testing.FakeTxBuilder; import com.google.bitcoin.testing.InboundMessageQueuer; import com.google.bitcoin.testing.TestWithPeerGroup; import com.google.bitcoin.wallet.KeyChainGroup; @@ -68,7 +69,25 @@ public class FilteredBlockAndPartialMerkleTreeTests extends TestWithPeerGroup { // Check round tripping. assertEquals(block, new FilteredBlock(params, block.bitcoinSerialize())); } - + + @Test + public void createFilteredBlock() throws Exception { + ECKey key1 = new ECKey(); + ECKey key2 = new ECKey(); + Transaction tx1 = FakeTxBuilder.createFakeTx(params, Coin.COIN, key1); + Transaction tx2 = FakeTxBuilder.createFakeTx(params, Coin.FIFTY_COINS, key2.toAddress(params)); + Block block = FakeTxBuilder.makeSolvedTestBlock(params.getGenesisBlock(), new Address(params, "msg2t2V2sWNd85LccoddtWysBTR8oPnkzW"), tx1, tx2); + BloomFilter filter = new BloomFilter(4, 0.1, 1); + filter.insert(key1); + filter.insert(key2); + FilteredBlock filteredBlock = filter.applyAndUpdate(block); + assertEquals(4, filteredBlock.getTransactionCount()); + // This call triggers verification of the just created data. + List txns = filteredBlock.getTransactionHashes(); + assertTrue(txns.contains(tx1.getHash())); + assertTrue(txns.contains(tx2.getHash())); + } + @Test public void serializeDownloadBlockWithWallet() throws Exception { unitTestParams = UnitTestParams.get();