mirror of
https://github.com/bitcoin-s/bitcoin-s.git
synced 2025-03-14 03:48:07 +01:00
BIP 158 Golomb-Rice Coded Sets and block filters (#481)
* Implemented GCS (WIP) Added some of Chris' tests for golombEncode and golombDecode Fixed GCS.golombDecodeSet Separated GCS object from GolombFilter class Added hanging test for set encoding Fixed property test for encoding and decoding arbitrary sets Added test for GCS.buildGCS and hashing function consistency Added method to construct a Block Filter from a Block Added scaladocs in GCS.scala Added TempUtxoProvider to make BlockFilter testable Wrote a test for GolombFilter.matchesHash Added test for arbitrary data matching in bip 158 filter Added optional right-padding on GCS BitVector, BlockFilter.fromHex, and got started on BlockFilterTest Cleaned up Bip158TestCase.fromJsArray Fixed parsing of test vectors for bip 158 Wrote test for BlockFilter (doesn't pass) Responded to some code review Included CompactSizeUInt in filter Created BitcoinScriptUtil.getDataTokens for output serialization Wrote test for BitcoinScriptUtil.getDataTokens Added test for BitcoinScriptUtil on the genesis block Fixed name :( * Fixed SipHash and block serialization! * Fixed bugs relating to empty scripts, all tests passgit push --force-with-lease nadav 2019-05-28-bip158 ! * Cleaned up BitcoinScriptUtil.getDataTokens and its test * Responded to code review * Factored out common code from BlockFilter apply methods * Some cleanup
This commit is contained in:
Nadav Kohen
Chris Stewart
parent
90b9b6aa9a
commit
5ed0f6d35b
11 changed files with 933 additions and 2 deletions
13
core-test/src/test/resources/testnet-19.json
Normal file
13
core-test/src/test/resources/testnet-19.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
[
|
||||
["Block Height,Block Hash,Block,[Prev Output Scripts for Block],Previous Basic Header,Basic Filter,Basic Header,Notes"],
|
||||
[0,"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000",[],"0000000000000000000000000000000000000000000000000000000000000000","019dfca8","21584579b7eb08997773e5aeff3a7f932700042d0ed2a6129012b7d7ae81b750","Genesis block"],
|
||||
[2,"000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820","0100000006128e87be8b1b4dea47a7247d5528d2702c96826c7a648497e773b800000000e241352e3bec0a95a6217e10c3abb54adfa05abb12c126695595580fb92e222032e7494dffff001d00d235340101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000",[],"d7bdac13a59d745b1add0d2ce852f1a0442e8945fc1bf3848d3cbffd88c24fe1","0174a170","186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0",""],
|
||||
[3,"000000008b896e272758da5297bcd98fdc6d97c9b765ecec401e286dc1fdbe10","0100000020782a005255b657696ea057d5b98f34defcf75196f64f6eeac8026c0000000041ba5afc532aae03151b8aa87b65e1594f97504a768e010c98c0add79216247186e7494dffff001d058dc2b60101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0486e7494d0151062f503253482fffffffff0100f2052a01000000232103f6d9ff4c12959445ca5549c811683bf9c88e637b222dd2e0311154c4c85cf423ac00000000",[],"186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0","016cf7a0","8d63aadf5ab7257cb6d2316a57b16f517bff1c6388f124ec4c04af1212729d2a",""],
|
||||
[15007,"0000000038c44c703bae0f98cdd6bf30922326340a5996cc692aaae8bacf47ad","0100000002394092aa378fe35d7e9ac79c869b975c4de4374cd75eb5484b0e1e00000000eb9b8670abd44ad6c55cee18e3020fb0c6519e7004b01a16e9164867531b67afc33bc94fffff001d123f10050101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e04c33bc94f0115062f503253482fffffffff0100f2052a01000000232103f268e9ae07e0f8cb2f6e901d87c510d650b97230c0365b021df8f467363cafb1ac00000000",[],"18b5c2b0146d2d09d24fb00ff5b52bd0742f36c9e65527abdb9de30c027a4748","013c3710","07384b01311867949e0c046607c66b7a766d338474bb67f66c8ae9dbd454b20e","Tx has non-standard OP_RETURN output followed by opcodes"],
|
||||
[49291,"0000000018b07dca1b28b4b5a119f6d6e71698ce1ed96f143f54179ce177a19c","02000000abfaf47274223ca2fea22797e44498240e482cb4c2f2baea088962f800000000604b5b52c32305b15d7542071d8b04e750a547500005d4010727694b6e72a776e55d0d51ffff001d211806480201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0d038bc0000102062f503253482fffffffff01a078072a01000000232102971dd6034ed0cf52450b608d196c07d6345184fcb14deb277a6b82d526a6163dac0000000001000000081cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff000000009300493046022100866859c21f306538152e83f115bcfbf59ab4bb34887a88c03483a5dff9895f96022100a6dfd83caa609bf0516debc2bf65c3df91813a4842650a1858b3f61cfa8af249014730440220296d4b818bb037d0f83f9f7111665f49532dfdcbec1e6b784526e9ac4046eaa602204acf3a5cb2695e8404d80bf49ab04828bcbe6fc31d25a2844ced7a8d24afbdff01ffffffff1cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff020000009400483045022100e87899175991aa008176cb553c6f2badbb5b741f328c9845fcab89f8b18cae2302200acce689896dc82933015e7230e5230d5cff8a1ffe82d334d60162ac2c5b0c9601493046022100994ad29d1e7b03e41731a4316e5f4992f0d9b6e2efc40a1ccd2c949b461175c502210099b69fdc2db00fbba214f16e286f6a49e2d8a0d5ffc6409d87796add475478d601ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272010000009500493046022100a27400ba52fd842ce07398a1de102f710a10c5599545e6c95798934352c2e4df022100f6383b0b14c9f64b6718139f55b6b9494374755b86bae7d63f5d3e583b57255a01493046022100fdf543292f34e1eeb1703b264965339ec4a450ec47585009c606b3edbc5b617b022100a5fbb1c8de8aaaa582988cdb23622838e38de90bebcaab3928d949aa502a65d401ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272020000009400493046022100ac626ac3051f875145b4fe4cfe089ea895aac73f65ab837b1ac30f5d875874fa022100bc03e79fa4b7eb707fb735b95ff6613ca33adeaf3a0607cdcead4cfd3b51729801483045022100b720b04a5c5e2f61b7df0fcf334ab6fea167b7aaede5695d3f7c6973496adbf1022043328c4cc1cdc3e5db7bb895ccc37133e960b2fd3ece98350f774596badb387201ffffffff23a8733e349c97d6cd90f520fdd084ba15ce0a395aad03cd51370602bb9e5db3010000004a00483045022100e8556b72c5e9c0da7371913a45861a61c5df434dfd962de7b23848e1a28c86ca02205d41ceda00136267281be0974be132ac4cda1459fe2090ce455619d8b91045e901ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a6000000006a473044022040a1c631554b8b210fbdf2a73f191b2851afb51d5171fb53502a3a040a38d2c0022040d11cf6e7b41fe1b66c3d08f6ada1aee07a047cb77f242b8ecc63812c832c9a012102bcfad931b502761e452962a5976c79158a0f6d307ad31b739611dac6a297c256ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a601000000930048304502205b109df098f7e932fbf71a45869c3f80323974a826ee2770789eae178a21bfc8022100c0e75615e53ee4b6e32b9bb5faa36ac539e9c05fa2ae6b6de5d09c08455c8b9601483045022009fb7d27375c47bea23b24818634df6a54ecf72d52e0c1268fb2a2c84f1885de022100e0ed4f15d62e7f537da0d0f1863498f9c7c0c0a4e00e4679588c8d1a9eb20bb801ffffffffa563c3722b7b39481836d5edfc1461f97335d5d1e9a23ade13680d0e2c1c371f030000006c493046022100ecc38ae2b1565643dc3c0dad5e961a5f0ea09cab28d024f92fa05c922924157e022100ebc166edf6fbe4004c72bfe8cf40130263f98ddff728c8e67b113dbd621906a601210211a4ed241174708c07206601b44a4c1c29e5ad8b1f731c50ca7e1d4b2a06dc1fffffffff02d0223a00000000001976a91445db0b779c0b9fa207f12a8218c94fc77aff504588ac80f0fa02000000000000000000",["5221033423007d8f263819a2e42becaaf5b06f34cb09919e06304349d950668209eaed21021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","522102a7ae1e0971fc1689bd66d2a7296da3a1662fd21a53c9e38979e0f090a375c12d21022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","512103b9d1d0e2b4355ec3cdef7c11a5c0beff9e8b8d8372ab4b4e0aaf30e80173001951ae","76a9149144761ebaccd5b4bbdc2a35453585b5637b2f8588ac","522103f1848b40621c5d48471d9784c8174ca060555891ace6d2b03c58eece946b1a9121020ee5d32b54d429c152fdc7b1db84f2074b0564d35400d89d11870f9273ec140c52ae","76a914f4fa1cc7de742d135ea82c17adf0bb9cf5f4fb8388ac"],"ed47705334f4643892ca46396eb3f4196a5e30880589e4009ef38eae895d4a13","0afbc2920af1b027f31f87b592276eb4c32094bb4d3697021b4c6380","b6d98692cec5145f67585f3434ec3c2b3030182e1cb3ec58b855c5c164dfaaa3","Tx pays to empty output script"],
|
||||
[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"d34ef98386f413769502808d4bac5f20f8dfd5bffc9eedafaa71de0eb1f01489","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","c582d51c0ca365e3fcf36c51cb646d7f83a67e867cb4743fd2128e3e022b700c","Tx spends from empty output script"],
|
||||
[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"8f13b9a9c85611635b47906c3053ac53cfcec7211455d4cb0d63dc9acc13d472","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","546c574a0472144bcaf9b6aeabf26372ad87c7af7d1ee0dbfae5e099abeae49c","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"],
|
||||
[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"fe4d230dbb0f4fec9bed23a5283e08baf996e3f32b93f52c7de1f641ddfd04ad","010c0b40","0965a544743bbfa36f254446e75630c09404b3d164a261892372977538928ed5","Coinbase tx has unparseable output script"],
|
||||
[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"31d66d516a9eda7de865df29f6ef6cb8e4bf9309e5dac899968a9a62a5df61e3","0385acb4f0fe889ef0","4e6d564c2a2452065c205dd7eb2791124e0c4e0dbb064c410c24968572589dec","Includes witness data"],
|
||||
[1414221,"0000000000000027b2b3b3381f114f674f481544ff2be37ae3788d7e078383b1","000000204ea88307a7959d8207968f152bedca5a93aefab253f1fb2cfb032a400000000070cebb14ec6dbc27a9dfd066d9849a4d3bac5f674665f73a5fe1de01a022a0c851fda85bf05f4c19a779d1450102000000010000000000000000000000000000000000000000000000000000000000000000ffffffff18034d94154d696e6572476174653030310d000000f238f401ffffffff01c817a804000000000000000000",[],"5e5e12d90693c8e936f01847859404c67482439681928353ca1296982042864e","00","021e8882ef5a0ed932edeebbecfeda1d7ce528ec7b3daa27641acf1189d7b5dc","Empty data"]
|
||||
]
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
package org.bitcoins.core.gcs
|
||||
|
||||
import org.bitcoins.core.crypto.DoubleSha256Digest
|
||||
import org.bitcoins.core.protocol.blockchain.Block
|
||||
import org.bitcoins.core.protocol.script.ScriptPubKey
|
||||
import org.bitcoins.testkit.util.BitcoinSUnitTest
|
||||
import play.api.libs.json.{JsArray, Json}
|
||||
|
||||
import scala.io.Source
|
||||
|
||||
class BlockFilterTest extends BitcoinSUnitTest {
|
||||
behavior of "BlockFilter"
|
||||
|
||||
// https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#appendix-c-test-vectors
|
||||
case class Bip158TestCase(
|
||||
blockHeight: Int,
|
||||
blockHash: DoubleSha256Digest,
|
||||
block: Block,
|
||||
prevOutputScripts: Vector[ScriptPubKey],
|
||||
// TODO prevHeader: BlockFilterHeader,
|
||||
filter: GolombFilter,
|
||||
// TODO header: BlockFilterHeader,
|
||||
notes: String
|
||||
) {
|
||||
|
||||
def runTest(): org.scalatest.Assertion = {
|
||||
val constructedFilter = BlockFilter(block, prevOutputScripts)
|
||||
|
||||
assert(constructedFilter.decodedHashes == filter.decodedHashes,
|
||||
s"Test Notes: $notes")
|
||||
}
|
||||
}
|
||||
|
||||
object Bip158TestCase {
|
||||
|
||||
//["Block Height,Block Hash,Block,[Prev Output Scripts for Block],Previous Basic Header,Basic Filter,Basic Header,Notes"]
|
||||
def fromJsArray(array: JsArray): Bip158TestCase = {
|
||||
val parseResult = for {
|
||||
height <- array(0).validate[Int]
|
||||
blockHash <- array(1).validate[String].map(DoubleSha256Digest.fromHex)
|
||||
|
||||
block <- array(2).validate[String].map(Block.fromHex)
|
||||
|
||||
scriptArray <- array(3).validate[JsArray]
|
||||
scripts = parseScripts(scriptArray)
|
||||
|
||||
//prevHeader <- array(4).validate[String].map(BlockFilterHeader.fromHex)
|
||||
|
||||
filter <- array(5)
|
||||
.validate[String]
|
||||
.map(BlockFilter.fromHex(_, blockHash))
|
||||
|
||||
//header <- array(6).validate[String].map(BlockFilterHeader.fromHex)
|
||||
|
||||
notes <- array(7).validate[String]
|
||||
} yield Bip158TestCase(height, blockHash, block, scripts, filter, notes)
|
||||
|
||||
parseResult.get
|
||||
}
|
||||
|
||||
private def parseScripts(array: JsArray): Vector[ScriptPubKey] = {
|
||||
val hexScripts = array.validate[Vector[String]].get
|
||||
|
||||
hexScripts.map(ScriptPubKey.fromAsmHex)
|
||||
}
|
||||
}
|
||||
|
||||
it must "pass bip 158 test vectors" in {
|
||||
val source = Source.fromURL(getClass.getResource("/testnet-19.json"))
|
||||
|
||||
val vec: Vector[JsArray] =
|
||||
Json.parse(source.mkString).validate[Vector[JsArray]].get.tail
|
||||
val testCases = vec.map(Bip158TestCase.fromJsArray)
|
||||
|
||||
testCases.foreach(_.runTest())
|
||||
}
|
||||
}
|
||||
257
core-test/src/test/scala/org/bitcoins/core/gcs/GCSTest.scala
Normal file
257
core-test/src/test/scala/org/bitcoins/core/gcs/GCSTest.scala
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
package org.bitcoins.core.gcs
|
||||
|
||||
import org.bitcoins.core.number.{UInt64, UInt8}
|
||||
import org.bitcoins.core.util.NumberUtil
|
||||
import org.bitcoins.testkit.core.gen.NumberGenerator
|
||||
import org.bitcoins.testkit.util.BitcoinSUnitTest
|
||||
import org.scalacheck.Gen
|
||||
import scodec.bits.{BinStringSyntax, ByteVector}
|
||||
|
||||
class GCSTest extends BitcoinSUnitTest {
|
||||
behavior of "GCS"
|
||||
|
||||
//https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#golomb-rice-coding
|
||||
it must "encode and decode Golomb Coded Set example 1" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64.zero
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"000")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 2" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64.one
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"001")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 3" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(2)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"010")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 4" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(3)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"011")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 5" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(4)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"1000")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 6" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(5)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"1001")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 7" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(6)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"1010")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 8" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(7)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"1011")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 9" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(8)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"11000")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode Golomb Coded set example 10" in {
|
||||
val p = UInt8(2)
|
||||
val original = UInt64(9)
|
||||
|
||||
val encoding = GCS.golombEncode(item = original, p = p)
|
||||
|
||||
assert(encoding == bin"11001")
|
||||
|
||||
val decode = GCS.golombDecode(codedItem = encoding, p = p)
|
||||
|
||||
assert(decode == original)
|
||||
}
|
||||
|
||||
it must "encode and decode an arbitrary item for an arbitrary p" in {
|
||||
|
||||
def delta: Gen[UInt64] = {
|
||||
//what is a reasonable delta? This is means the delta
|
||||
//can be 1 - 16384
|
||||
//if we do a full uint64 it takes forever to encode it
|
||||
Gen
|
||||
.choose(1, NumberUtil.pow2(14).toInt)
|
||||
.map(UInt64(_))
|
||||
}
|
||||
|
||||
forAll(delta, NumberGenerator.genP) {
|
||||
case (item, p) =>
|
||||
val encoded = GCS.golombEncode(item = item, p = p)
|
||||
val decode = GCS.golombDecode(codedItem = encoded, p = p)
|
||||
|
||||
assert(decode == item)
|
||||
}
|
||||
}
|
||||
|
||||
it must "encode and decode a set of elements already tested" in {
|
||||
val p = UInt8(2)
|
||||
|
||||
// Diffs are 1, 2, 3, 4, 5
|
||||
val sortedItems =
|
||||
Vector(UInt64(0), UInt64(1), UInt64(3), UInt64(6), UInt64(10), UInt64(15))
|
||||
|
||||
val codedSet = GCS.encodeSortedSet(sortedItems, p)
|
||||
|
||||
val coded0 = bin"000"
|
||||
val coded1 = bin"001"
|
||||
val coded2 = bin"010"
|
||||
val coded3 = bin"011"
|
||||
val coded4 = bin"1000"
|
||||
val coded5 = bin"1001"
|
||||
val expectedCodedSet = coded0 ++ coded1 ++ coded2 ++ coded3 ++ coded4 ++ coded5
|
||||
|
||||
assert(codedSet == expectedCodedSet)
|
||||
|
||||
val decodedSet = GCS.golombDecodeSet(codedSet, p)
|
||||
|
||||
assert(decodedSet == sortedItems)
|
||||
}
|
||||
|
||||
it must "encode and decode arbitrary sets of elements for arbitrary p" in {
|
||||
|
||||
def items: Gen[(Vector[UInt64], UInt8)] = {
|
||||
NumberGenerator.genP.flatMap { p =>
|
||||
Gen.choose(1, 1000).flatMap { size =>
|
||||
// If hash's quotient when divided by 2^p is too large, we hang converting to unary
|
||||
val upperBound: Long = 1L << (p.toInt * 1.75).toInt
|
||||
|
||||
val hashGen = Gen
|
||||
.chooseNum(0L, upperBound)
|
||||
.map(UInt64(_))
|
||||
|
||||
Gen.listOfN(size, hashGen).map(_.toVector).map { vec =>
|
||||
(vec, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
forAll(items) {
|
||||
case (items, p) =>
|
||||
val sorted = items.sortWith(_ < _)
|
||||
|
||||
val codedSet = GCS.encodeSortedSet(sorted, p)
|
||||
val decodedSet = GCS.golombDecodeSet(codedSet, p)
|
||||
|
||||
assert(decodedSet == sorted)
|
||||
}
|
||||
}
|
||||
|
||||
it must "encode and decode arbitrary ByteVectors for arbitrary p" in {
|
||||
|
||||
def genP: Gen[UInt8] = {
|
||||
// We have 8 as a lower bound since N in hashToRange is in the order of 1000
|
||||
Gen.choose(8, 32).map(UInt8(_))
|
||||
}
|
||||
|
||||
def genPM: Gen[(UInt8, UInt64)] = genP.flatMap { p =>
|
||||
// If hash's quotient when divided by 2^p is too large, we hang converting to unary
|
||||
val upperBound: Long = p.toInt * 1000
|
||||
|
||||
val mGen = Gen
|
||||
.chooseNum(0L, upperBound)
|
||||
.map(UInt64(_))
|
||||
|
||||
mGen.map(m => (p, m))
|
||||
}
|
||||
|
||||
def genItems: Gen[Vector[ByteVector]] = {
|
||||
Gen.choose(1, 1000).flatMap { size =>
|
||||
Gen.listOfN(size, NumberGenerator.bytevector).map(_.toVector)
|
||||
}
|
||||
}
|
||||
|
||||
def genKey: Gen[ByteVector] =
|
||||
Gen.listOfN(16, NumberGenerator.byte).map(ByteVector(_))
|
||||
|
||||
forAll(genPM, genItems, genKey) {
|
||||
case ((p, m), items, k) =>
|
||||
val hashes = GCS.hashedSetConstruct(items, k, m)
|
||||
val sortedHashes = hashes.sortWith(_ < _)
|
||||
|
||||
val codedSet = GCS.buildGCS(items, k, p, m)
|
||||
val decodedSet = GCS.golombDecodeSet(codedSet, p)
|
||||
|
||||
assert(decodedSet == sortedHashes)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package org.bitcoins.core.gcs
|
||||
|
||||
import org.bitcoins.core.number.{UInt64, UInt8}
|
||||
import org.bitcoins.core.protocol.CompactSizeUInt
|
||||
import org.bitcoins.testkit.core.gen.NumberGenerator
|
||||
import org.bitcoins.testkit.util.BitcoinSUnitTest
|
||||
import org.scalacheck.Gen
|
||||
import scodec.bits.ByteVector
|
||||
|
||||
class GolombFilterTest extends BitcoinSUnitTest {
|
||||
behavior of "GolombFilter"
|
||||
|
||||
it must "match encoded data for arbitrary GCS parameters" in {
|
||||
def genKey: Gen[ByteVector] =
|
||||
Gen.listOfN(16, NumberGenerator.byte).map(ByteVector(_))
|
||||
|
||||
def genPMRand: Gen[(UInt8, UInt64, UInt64)] = NumberGenerator.genP.flatMap {
|
||||
p =>
|
||||
// If hash's quotient when divided by 2^p is too large, we hang converting to unary
|
||||
val upperBound: Long = p.toInt * 1000 + 1
|
||||
|
||||
val mGen = Gen
|
||||
.chooseNum(1L, upperBound)
|
||||
.map(UInt64(_))
|
||||
|
||||
mGen.flatMap { m =>
|
||||
val upperBound = m.toInt * 2 - 2
|
||||
|
||||
val randGen = Gen.chooseNum(0L, upperBound).map(UInt64(_))
|
||||
|
||||
randGen.map(rand => (p, m, rand))
|
||||
}
|
||||
}
|
||||
|
||||
forAll(genKey, genPMRand) {
|
||||
case (k, (p, m, rand)) =>
|
||||
val data1 = rand + UInt64.one
|
||||
val data2 = data1 + UInt64.one
|
||||
val data = Vector(data1, data2)
|
||||
val encodedData = GCS.encodeSortedSet(data, p)
|
||||
val filter =
|
||||
GolombFilter(k, m, p, CompactSizeUInt(UInt64(2)), encodedData)
|
||||
|
||||
assert(!filter.matchesHash(rand))
|
||||
assert(filter.matchesHash(data1))
|
||||
assert(filter.matchesHash(data2))
|
||||
}
|
||||
}
|
||||
|
||||
it must "match arbitrary encoded data for bip 158 GCS parameters" in {
|
||||
val genKey: Gen[ByteVector] =
|
||||
Gen.listOfN(16, NumberGenerator.byte).map(ByteVector(_))
|
||||
|
||||
val genData: Gen[Vector[ByteVector]] = Gen.chooseNum(1, 10000).flatMap {
|
||||
size =>
|
||||
Gen.listOfN(size, NumberGenerator.bytevector).map(_.toVector)
|
||||
}
|
||||
|
||||
val genRandHashes: Gen[Vector[UInt64]] =
|
||||
Gen.listOfN(1000, NumberGenerator.uInt64).map(_.toVector)
|
||||
|
||||
forAll(genKey, genData, genRandHashes) {
|
||||
case (k, data, randHashes) =>
|
||||
val filter = GCS.buildBasicBlockFilter(data, k)
|
||||
val hashes = filter.decodedHashes
|
||||
|
||||
data.foreach(element => assert(filter.matches(element)))
|
||||
|
||||
val hashesNotInData: Vector[UInt64] =
|
||||
randHashes.filterNot(hashes.contains)
|
||||
|
||||
hashesNotInData.foreach(hash => assert(!filter.matchesHash(hash)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,19 +2,25 @@ package org.bitcoins.core.util
|
|||
|
||||
import org.bitcoins.core.crypto.{ECPrivateKey, ECPublicKey}
|
||||
import org.bitcoins.core.protocol.script.{SigVersionBase, SigVersionWitnessV0}
|
||||
import org.bitcoins.core.protocol.transaction.TransactionOutput
|
||||
import org.bitcoins.core.script.constant._
|
||||
import org.bitcoins.core.script.crypto._
|
||||
import org.bitcoins.core.script.flag.ScriptVerifyWitnessPubKeyType
|
||||
import org.bitcoins.core.script.locktime.OP_CHECKLOCKTIMEVERIFY
|
||||
import org.bitcoins.core.script.reserved.{OP_NOP, OP_RESERVED}
|
||||
import org.bitcoins.core.script.result.ScriptErrorWitnessPubKeyType
|
||||
import org.bitcoins.testkit.core.gen.ScriptGenerators
|
||||
import org.scalatest.prop.PropertyChecks
|
||||
import org.scalatest.{FlatSpec, MustMatchers}
|
||||
import scodec.bits.ByteVector
|
||||
|
||||
/**
|
||||
* Created by chris on 3/2/16.
|
||||
*/
|
||||
class BitcoinScriptUtilTest extends FlatSpec with MustMatchers {
|
||||
class BitcoinScriptUtilTest
|
||||
extends FlatSpec
|
||||
with MustMatchers
|
||||
with PropertyChecks {
|
||||
|
||||
//from b30d3148927f620f5b1228ba941c211fdabdae75d0ba0b688a58accbf018f3cc
|
||||
val asm = TestUtil.p2pkhScriptPubKey.asm
|
||||
|
|
@ -32,6 +38,61 @@ class BitcoinScriptUtilTest extends FlatSpec with MustMatchers {
|
|||
.isEmpty must be(true)
|
||||
}
|
||||
|
||||
// https://en.bitcoin.it/wiki/Genesis_block
|
||||
it must "filter out non-data from the genesis coinbase transaction" in {
|
||||
val genesisPK = ECPublicKey.fromHex(
|
||||
"04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f")
|
||||
val output = TransactionOutput.fromHex(
|
||||
"00f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac")
|
||||
val scriptPubKey = output.scriptPubKey
|
||||
assert(
|
||||
BitcoinScriptUtil
|
||||
.getDataTokens(scriptPubKey.asm)
|
||||
.map(_.bytes) == Vector(genesisPK.bytes))
|
||||
}
|
||||
|
||||
it must "filter out all but the data tokens" in {
|
||||
forAll(ScriptGenerators.p2pkhScriptPubKey) {
|
||||
case (p2pkhScript, _) =>
|
||||
assert(
|
||||
BitcoinScriptUtil
|
||||
.getDataTokens(p2pkhScript.asm)
|
||||
.map(_.bytes) == Vector(p2pkhScript.pubKeyHash.bytes))
|
||||
}
|
||||
|
||||
forAll(ScriptGenerators.p2shScriptPubKey) {
|
||||
case (p2shScript, _) =>
|
||||
assert(
|
||||
BitcoinScriptUtil
|
||||
.getDataTokens(p2shScript.asm)
|
||||
.map(_.bytes) == Vector(p2shScript.scriptHash.bytes))
|
||||
}
|
||||
|
||||
forAll(ScriptGenerators.p2wshSPKV0) {
|
||||
case (p2wshScript, _) =>
|
||||
assert(
|
||||
BitcoinScriptUtil
|
||||
.getDataTokens(p2wshScript.asm)
|
||||
.map(_.bytes) == Vector(p2wshScript.scriptHash.bytes))
|
||||
}
|
||||
|
||||
forAll(ScriptGenerators.p2wpkhSPKV0) {
|
||||
case (p2wpkhScript, _) =>
|
||||
assert(
|
||||
BitcoinScriptUtil
|
||||
.getDataTokens(p2wpkhScript.asm)
|
||||
.map(_.bytes) == Vector(p2wpkhScript.pubKeyHash.bytes))
|
||||
}
|
||||
|
||||
forAll(ScriptGenerators.multiSigScriptPubKey) {
|
||||
case (multiSigScript, _) =>
|
||||
assert(
|
||||
BitcoinScriptUtil
|
||||
.getDataTokens(multiSigScript.asm)
|
||||
.map(_.bytes) == multiSigScript.publicKeys.map(_.bytes))
|
||||
}
|
||||
}
|
||||
|
||||
it must "determine if a script op count towards the bitcoin script op code limit" in {
|
||||
BitcoinScriptUtil.countsTowardsScriptOpLimit(OP_1) must be(false)
|
||||
BitcoinScriptUtil.countsTowardsScriptOpLimit(OP_16) must be(false)
|
||||
|
|
|
|||
227
core/src/main/scala/org/bitcoins/core/gcs/GCS.scala
Normal file
227
core/src/main/scala/org/bitcoins/core/gcs/GCS.scala
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
package org.bitcoins.core.gcs
|
||||
|
||||
import org.bitcoins.core.number.{UInt64, UInt8}
|
||||
import org.bitcoins.core.protocol.CompactSizeUInt
|
||||
import org.bouncycastle.crypto.macs.SipHash
|
||||
import org.bouncycastle.crypto.params.KeyParameter
|
||||
import scodec.bits.{BitVector, BinStringSyntax, ByteVector}
|
||||
|
||||
import scala.annotation.tailrec
|
||||
|
||||
// TODO: Replace ByteVector with a type for keys
|
||||
/**
|
||||
* Defines all functionality dealing with Golomb-Coded Sets
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#GolombCoded_Sets]]
|
||||
*/
|
||||
object GCS {
|
||||
|
||||
/**
|
||||
* Given parameters and data, golomb-encodes the data
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#set-construction]]
|
||||
*/
|
||||
def buildGCS(
|
||||
data: Vector[ByteVector],
|
||||
key: ByteVector,
|
||||
p: UInt8,
|
||||
m: UInt64): BitVector = {
|
||||
val hashedValues = hashedSetConstruct(data, key, m)
|
||||
val sortedHashedValues = hashedValues.sortWith(_ < _)
|
||||
encodeSortedSet(sortedHashedValues, p)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given parameters and data, constructs a GolombFilter for that data
|
||||
*/
|
||||
def buildGolombFilter(
|
||||
data: Vector[ByteVector],
|
||||
key: ByteVector,
|
||||
p: UInt8,
|
||||
m: UInt64): GolombFilter = {
|
||||
val encodedData = buildGCS(data, key, p, m)
|
||||
|
||||
GolombFilter(key, m, p, CompactSizeUInt(UInt64(data.length)), encodedData)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given data, constructs a GolombFilter for that data using Basic Block Filter parameters
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#block-filters]]
|
||||
*/
|
||||
def buildBasicBlockFilter(
|
||||
data: Vector[ByteVector],
|
||||
key: ByteVector): GolombFilter = {
|
||||
buildGolombFilter(data, key, BlockFilter.P, BlockFilter.M)
|
||||
}
|
||||
|
||||
private def sipHash(item: ByteVector, key: ByteVector): UInt64 = {
|
||||
// https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#hashing-data-objects
|
||||
val sipHashCParam = 2
|
||||
val sipHashDParam = 4
|
||||
|
||||
val sh = new SipHash(sipHashCParam, sipHashDParam)
|
||||
|
||||
val keyParam = new KeyParameter(key.toArray)
|
||||
|
||||
sh.init(keyParam)
|
||||
|
||||
val offset = 0
|
||||
|
||||
sh.update(item.toArray, offset, item.length.toInt)
|
||||
|
||||
val digest = sh.doFinal()
|
||||
|
||||
UInt64.fromHex(digest.toHexString)
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes the item to the range [0, f)
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#hashing-data-objects]]
|
||||
*/
|
||||
def hashToRange(item: ByteVector, f: UInt64, key: ByteVector): UInt64 = {
|
||||
val hash = sipHash(item, key)
|
||||
|
||||
val bigInt = (hash.toBigInt * f.toBigInt) >> 64
|
||||
|
||||
UInt64(bigInt)
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes the items of a set of items
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#hashing-data-objects]]
|
||||
*/
|
||||
def hashedSetConstruct(
|
||||
rawItems: Vector[ByteVector],
|
||||
key: ByteVector,
|
||||
m: UInt64): Vector[UInt64] = {
|
||||
val n = rawItems.length
|
||||
val f = m * n
|
||||
|
||||
val hashedItemsBuilder = Vector.newBuilder[UInt64]
|
||||
|
||||
rawItems.foreach { item =>
|
||||
val setValue = hashToRange(item, f, key)
|
||||
hashedItemsBuilder.+=(setValue)
|
||||
}
|
||||
|
||||
hashedItemsBuilder.result()
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts num to unary (6 = 1111110)
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#golomb-rice-coding]]
|
||||
*
|
||||
* TODO: protect against large inputs which cause OutOfMemoryErrors and even larger ones which fail on toInt
|
||||
*/
|
||||
def toUnary(num: UInt64): BitVector = {
|
||||
if (num == UInt64.zero) {
|
||||
bin"0"
|
||||
} else {
|
||||
/*
|
||||
* We use the fact that 2^n - 1 = 111...1 (in binary) where there are n 1 digits
|
||||
*/
|
||||
val binUnary = (BigInt(1) << num.toInt) - 1
|
||||
val leftPadded = BitVector(binUnary.toByteArray)
|
||||
val noPadding = dropLeftPadding(leftPadded)
|
||||
|
||||
noPadding.:+(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a hash into a unary prefix and binary suffix
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#golomb-rice-coding]]
|
||||
*/
|
||||
def golombEncode(item: UInt64, p: UInt8): BitVector = {
|
||||
val q = item >> p.toInt
|
||||
|
||||
val prefix = toUnary(q)
|
||||
|
||||
val pBits = item.bytes.toBitVector.takeRight(p.toInt)
|
||||
|
||||
prefix ++ pBits
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes an item off of the front of a BitVector by reversing [[GCS.golombEncode]]
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#golomb-rice-coding]]
|
||||
*/
|
||||
def golombDecode(codedItem: BitVector, p: UInt8): UInt64 = {
|
||||
@tailrec
|
||||
def split(vec: BitVector, accum: UInt64): (UInt64, BitVector) = {
|
||||
if (vec.head) {
|
||||
split(vec.tail, accum + UInt64.one)
|
||||
} else {
|
||||
(accum, vec.tail)
|
||||
}
|
||||
}
|
||||
|
||||
val (q, pBits) = split(codedItem, UInt64.zero)
|
||||
|
||||
val sizeWithPadding = (8 - (p.toInt % 8)) + p.toInt
|
||||
|
||||
val pBitsAsBytes = {
|
||||
val withoutRightPaddingOrData = pBits.take(p.toInt)
|
||||
val withLeftPadding = withoutRightPaddingOrData.padLeft(sizeWithPadding)
|
||||
withLeftPadding.toByteVector
|
||||
}
|
||||
|
||||
(q << p.toInt) + UInt64.fromBytes(pBitsAsBytes)
|
||||
}
|
||||
|
||||
@tailrec
|
||||
private def dropLeftPadding(padded: BitVector): BitVector = {
|
||||
if (padded.isEmpty || padded.head) {
|
||||
padded
|
||||
} else {
|
||||
dropLeftPadding(padded.tail)
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the first hash gcs-encoded at the front of a BitVector, as well as the remaining BitVector */
|
||||
private def golombDecodeItemFromSet(
|
||||
encodedData: BitVector,
|
||||
p: UInt8): (UInt64, BitVector) = {
|
||||
val head = golombDecode(encodedData, p)
|
||||
|
||||
val prefixSize = (head >> p.toInt).toInt + 1
|
||||
|
||||
(head, encodedData.drop(prefixSize + p.toInt))
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes all hashes from golomb-encoded data, reversing [[GCS.encodeSortedSet]]
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#set-queryingdecompression]]
|
||||
*/
|
||||
def golombDecodeSet(encodedData: BitVector, p: UInt8): Vector[UInt64] = {
|
||||
@tailrec
|
||||
def loop(
|
||||
encoded: BitVector,
|
||||
decoded: Vector[UInt64],
|
||||
lastHash: UInt64): Vector[UInt64] = {
|
||||
if (encoded.length < p.toInt + 1) { // Only padding left
|
||||
decoded
|
||||
} else {
|
||||
val (delta, encodedLeft) = golombDecodeItemFromSet(encoded, p)
|
||||
val hash = lastHash + delta
|
||||
|
||||
loop(encodedLeft, decoded.:+(hash), hash)
|
||||
}
|
||||
}
|
||||
|
||||
loop(encoded = encodedData, decoded = Vector.empty, lastHash = UInt64.zero)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a set of ascending hashes, golomb-encodes them
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#set-construction]]
|
||||
*/
|
||||
def encodeSortedSet(hashes: Vector[UInt64], p: UInt8): BitVector = {
|
||||
val (golombStream, _) = hashes.foldLeft((BitVector.empty, UInt64.zero)) {
|
||||
case ((accum, lastHash), hash) =>
|
||||
val delta = hash - lastHash
|
||||
val encoded = golombEncode(delta, p)
|
||||
(accum ++ encoded, hash)
|
||||
}
|
||||
|
||||
golombStream
|
||||
}
|
||||
}
|
||||
162
core/src/main/scala/org/bitcoins/core/gcs/GolombFilter.scala
Normal file
162
core/src/main/scala/org/bitcoins/core/gcs/GolombFilter.scala
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
package org.bitcoins.core.gcs
|
||||
|
||||
import org.bitcoins.core.crypto.DoubleSha256Digest
|
||||
import org.bitcoins.core.number.{UInt64, UInt8}
|
||||
import org.bitcoins.core.protocol.CompactSizeUInt
|
||||
import org.bitcoins.core.protocol.blockchain.Block
|
||||
import org.bitcoins.core.protocol.script.{EmptyScriptPubKey, ScriptPubKey}
|
||||
import org.bitcoins.core.protocol.transaction.{
|
||||
Transaction,
|
||||
TransactionInput,
|
||||
TransactionOutPoint,
|
||||
TransactionOutput
|
||||
}
|
||||
import org.bitcoins.core.script.control.OP_RETURN
|
||||
import org.bitcoins.core.util.BitcoinSUtil
|
||||
import scodec.bits.{BitVector, ByteVector}
|
||||
|
||||
import scala.annotation.tailrec
|
||||
|
||||
/**
|
||||
* Represents a GCS encoded set with all parameters specified
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#golomb-coded-sets]]
|
||||
*
|
||||
* TODO: Replace ByteVector with a type for keys
|
||||
*/
|
||||
case class GolombFilter(
|
||||
key: ByteVector,
|
||||
m: UInt64,
|
||||
p: UInt8,
|
||||
n: CompactSizeUInt,
|
||||
encodedData: BitVector) {
|
||||
lazy val decodedHashes: Vector[UInt64] = GCS.golombDecodeSet(encodedData, p)
|
||||
|
||||
// TODO: Offer alternative that stops decoding when it finds out if data is there
|
||||
def matchesHash(hash: UInt64): Boolean = {
|
||||
@tailrec
|
||||
def binarySearch(
|
||||
from: Int,
|
||||
to: Int,
|
||||
hash: UInt64,
|
||||
set: Vector[UInt64]): Boolean = {
|
||||
if (to < from) {
|
||||
false
|
||||
} else {
|
||||
val index = (to + from) / 2
|
||||
val otherHash = set(index)
|
||||
|
||||
if (hash == otherHash) {
|
||||
true
|
||||
} else if (hash < otherHash) {
|
||||
binarySearch(from, index - 1, hash, set)
|
||||
} else {
|
||||
binarySearch(index + 1, to, hash, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
binarySearch(from = 0, to = n.toInt - 1, hash, decodedHashes)
|
||||
}
|
||||
|
||||
def matches(data: ByteVector): Boolean = {
|
||||
val f = n.num * m
|
||||
val hash = GCS.hashToRange(data, f, key)
|
||||
|
||||
matchesHash(hash)
|
||||
}
|
||||
}
|
||||
|
||||
object BlockFilter {
|
||||
|
||||
/** @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#block-filters]] */
|
||||
val M: UInt64 = UInt64(784931)
|
||||
val P: UInt8 = UInt8(19)
|
||||
|
||||
/**
|
||||
* Returns all ScriptPubKeys from a Block's outputs that are relevant
|
||||
* to BIP 158 Basic Block Filters
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#contents]]
|
||||
*/
|
||||
def getOutputScriptPubKeysFromBlock(block: Block): Vector[ScriptPubKey] = {
|
||||
val transactions: Vector[Transaction] = block.transactions.toVector
|
||||
|
||||
val newOutputs: Vector[TransactionOutput] = transactions.flatMap(_.outputs)
|
||||
|
||||
newOutputs
|
||||
.filterNot(_.scriptPubKey.asm.contains(OP_RETURN))
|
||||
.filterNot(_.scriptPubKey == EmptyScriptPubKey)
|
||||
.map(_.scriptPubKey)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all ScriptPubKeys from a Block's inputs that are relevant
|
||||
* to BIP 158 Basic Block Filters
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#contents]]
|
||||
*/
|
||||
def getInputScriptPubKeysFromBlock(
|
||||
block: Block,
|
||||
utxoProvider: TempUtxoProvider): Vector[ScriptPubKey] = {
|
||||
val transactions: Vector[Transaction] = block.transactions.toVector
|
||||
val noCoinbase: Vector[Transaction] = transactions.tail
|
||||
|
||||
val inputs: Vector[TransactionInput] = noCoinbase.flatMap(_.inputs)
|
||||
val outpointsSpent: Vector[TransactionOutPoint] =
|
||||
inputs.map(_.previousOutput)
|
||||
val prevOutputs: Vector[TransactionOutput] =
|
||||
outpointsSpent.flatMap(utxoProvider.getUtxo)
|
||||
|
||||
prevOutputs
|
||||
.filterNot(_.scriptPubKey == EmptyScriptPubKey)
|
||||
.map(_.scriptPubKey)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a Block and access to the UTXO set, constructs a Block Filter for that block
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#block-filters]]
|
||||
*/
|
||||
def apply(block: Block, utxoProvider: TempUtxoProvider): GolombFilter = {
|
||||
val prevOutputScripts: Vector[ScriptPubKey] =
|
||||
getInputScriptPubKeysFromBlock(block, utxoProvider)
|
||||
|
||||
BlockFilter(block, prevOutputScripts)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a Block and access to the previous output scripts, constructs a Block Filter for that block
|
||||
* @see [[https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki#block-filters]]
|
||||
*/
|
||||
def apply(
|
||||
block: Block,
|
||||
prevOutputScripts: Vector[ScriptPubKey]): GolombFilter = {
|
||||
val key = block.blockHeader.hash.bytes.take(16)
|
||||
|
||||
val newScriptPubKeys: Vector[ByteVector] =
|
||||
getOutputScriptPubKeysFromBlock(block).map(_.asmBytes)
|
||||
|
||||
val prevOutputScriptBytes: Vector[ByteVector] =
|
||||
prevOutputScripts
|
||||
.filterNot(_ == EmptyScriptPubKey)
|
||||
.map(_.asmBytes)
|
||||
|
||||
val allOutputs = (prevOutputScriptBytes ++ newScriptPubKeys).distinct
|
||||
|
||||
GCS.buildBasicBlockFilter(allOutputs, key)
|
||||
}
|
||||
|
||||
def fromBytes(
|
||||
bytes: ByteVector,
|
||||
blockHash: DoubleSha256Digest): GolombFilter = {
|
||||
val (size, filterBytes) = bytes.splitAt(1)
|
||||
val n = CompactSizeUInt.fromBytes(size)
|
||||
|
||||
GolombFilter(blockHash.bytes.take(16),
|
||||
BlockFilter.M,
|
||||
BlockFilter.P,
|
||||
n,
|
||||
filterBytes.toBitVector)
|
||||
}
|
||||
|
||||
def fromHex(hex: String, blockHash: DoubleSha256Digest): GolombFilter = {
|
||||
fromBytes(BitcoinSUtil.decodeHex(hex), blockHash)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package org.bitcoins.core.gcs
|
||||
|
||||
import org.bitcoins.core.protocol.transaction.{
|
||||
TransactionOutPoint,
|
||||
TransactionOutput
|
||||
}
|
||||
|
||||
import scala.collection.mutable
|
||||
|
||||
/**
|
||||
* A temporary abstraction over anything that keeps track of the UTXO set
|
||||
*/
|
||||
abstract class TempUtxoProvider {
|
||||
|
||||
def getUtxo(
|
||||
transactionOutPoint: TransactionOutPoint): Option[TransactionOutput]
|
||||
}
|
||||
|
||||
class TempUtxoProviderImpl extends TempUtxoProvider {
|
||||
|
||||
val utxos: mutable.Map[TransactionOutPoint, TransactionOutput] =
|
||||
mutable.Map.empty
|
||||
|
||||
override def getUtxo(
|
||||
transactionOutPoint: TransactionOutPoint): Option[TransactionOutput] = {
|
||||
utxos.get(transactionOutPoint)
|
||||
}
|
||||
}
|
||||
|
|
@ -744,7 +744,7 @@ object P2WPKHWitnessSPKV0 extends ScriptFactory[P2WPKHWitnessSPKV0] {
|
|||
* [[https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki#p2wsh BIP141]]
|
||||
*/
|
||||
sealed abstract class P2WSHWitnessSPKV0 extends WitnessScriptPubKeyV0 {
|
||||
def scriptHash: Sha256Digest = Sha256Digest(asm(3).bytes)
|
||||
def scriptHash: Sha256Digest = Sha256Digest(asm(2).bytes)
|
||||
override def toString = s"P2WSHWitnessSPKV0($hex)"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,27 @@ trait BitcoinScriptUtil extends BitcoinSLogger {
|
|||
|| op == OP_PUSHDATA4)
|
||||
}
|
||||
|
||||
/** Returns only the data ScriptTokens in a script that are pushed onto the stack */
|
||||
def getDataTokens(asm: Seq[ScriptToken]): Seq[ScriptToken] = {
|
||||
val builder = Vector.newBuilder[ScriptToken]
|
||||
|
||||
asm.zipWithIndex.foreach {
|
||||
case (token, index) =>
|
||||
token match {
|
||||
case OP_PUSHDATA1 | OP_PUSHDATA2 | OP_PUSHDATA4 =>
|
||||
/* OP_PUSH_DATA[1|2|4] says that the next value is [1|2|4] bytes and indicates
|
||||
* how many bytes should be pushed onto the stack (meaning the data is 2 values away)
|
||||
*/
|
||||
builder.+=(asm(index + 2))
|
||||
case _: BytesToPushOntoStack =>
|
||||
builder.+=(asm(index + 1))
|
||||
case _ => ()
|
||||
}
|
||||
}
|
||||
|
||||
builder.result()
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given script token counts towards our max script operations in a script
|
||||
* See
|
||||
|
|
|
|||
|
|
@ -125,6 +125,16 @@ trait NumberGenerator {
|
|||
vector <- Gen.listOfN(n, bool)
|
||||
} yield BitVector.bits(vector)
|
||||
|
||||
/**
|
||||
* Generates a random GCS P parameter.
|
||||
*
|
||||
* Bit parameter for GCS, cannot be more than 32 as we will have a number too large for a UInt64.
|
||||
* @see [[https://github.com/Roasbeef/btcutil/blob/b5d74480bb5b02a15a9266cbeae37ecf9dd6ffca/gcs/gcs.go#L67]]
|
||||
*/
|
||||
def genP: Gen[UInt8] = {
|
||||
Gen.choose(0, 32).map(UInt8(_))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
object NumberGenerator extends NumberGenerator
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue