mirrors/BlueWallet
1
0
Fork 0
mirror of https://github.com/BlueWallet/BlueWallet.git synced 2025-02-22 06:52:41 +01:00
Code Issues Projects Releases Packages Wiki Activity

REF: HD bech32 (BIP84) transaction fetch speedup (batching from electrum server)

Browse source
This commit is contained in:
Overtorment 2019-06-09 18:06:21 +01:00
parent 39f17d4891
commit 3d97ec40e4
4 changed files with 220 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
BlueElectrum.js
View file

@ -226,6 +226,54 @@ async function multiGetUtxoByAddress(addresses, batchsize) {
return ret;
}
async function multiGetHistoryByAddress(addresses, batchsize) {
batchsize = batchsize || 100;
if (!mainClient) throw new Error('Electrum client is not connected');
let ret = {};
let chunks = splitIntoChunks(addresses, batchsize);
for (let chunk of chunks) {
let scripthashes = [];
let scripthash2addr = {};
for (let addr of chunk) {
let script = bitcoin.address.toOutputScript(addr);
let hash = bitcoin.crypto.sha256(script);
let reversedHash = Buffer.from(reverse(hash));
reversedHash = reversedHash.toString('hex');
scripthashes.push(reversedHash);
scripthash2addr[reversedHash] = addr;
}
let results = await mainClient.blockchainScripthash_getHistoryBatch(scripthashes);
for (let history of results) {
ret[scripthash2addr[history.param]] = history.result;
for (let hist of ret[scripthash2addr[history.param]]) {
hist.address = scripthash2addr[history.param];
}
}
}
return ret;
}
async function multiGetTransactionByTxid(txids, batchsize) {
batchsize = batchsize || 100;
if (!mainClient) throw new Error('Electrum client is not connected');
let ret = {};
let chunks = splitIntoChunks(txids, batchsize);
for (let chunk of chunks) {
let results = await mainClient.blockchainTransaction_getBatch(chunk, true);
for (let txdata of results) {
ret[txdata.param] = txdata.result;
}
}
return ret;
}
/**
* Simple waiter till `mainConnected` becomes true (which means
* it Electrum was connected in other function), or timeout 30 sec.
@ -276,6 +324,8 @@ module.exports.waitTillConnected = waitTillConnected;
module.exports.estimateFees = estimateFees;
module.exports.broadcast = broadcast;
module.exports.multiGetUtxoByAddress = multiGetUtxoByAddress;
module.exports.multiGetHistoryByAddress = multiGetHistoryByAddress;
module.exports.multiGetTransactionByTxid = multiGetTransactionByTxid;
module.exports.forceDisconnect = () => {
mainClient.keepAlive = () => {}; // dirty hack to make it stop reconnecting

111
class/hd-segwit-bech32-wallet.js
View file

@ -208,6 +208,13 @@ export class HDSegwitBech32Wallet extends AbstractHDWallet {
// OR some tx for address is unconfirmed
// OR some tx has < 7 confirmations
// fetching transactions in batch: first, getting batch history for all addresses,
// then batch fetching all involved txids
// finally, batch fetching txids of all inputs (needed to see amounts & addresses of those inputs)
// then we combine it all together
let addresses2fetch = [];
for (let c = 0; c < this.next_free_address_index + this.gap_limit; c++) {
// external addresses first
let hasUnconfirmed = false;
@ -215,7 +222,7 @@ export class HDSegwitBech32Wallet extends AbstractHDWallet {
for (let tx of this._txs_by_external_index[c]) hasUnconfirmed = hasUnconfirmed || !tx.confirmations || tx.confirmations < 7;
if (hasUnconfirmed || this._txs_by_external_index[c].length === 0 || this._balances_by_external_index[c].u !== 0) {
this._txs_by_external_index[c] = await BlueElectrum.getTransactionsFullByAddress(this._getExternalAddressByIndex(c));
addresses2fetch.push(this._getExternalAddressByIndex(c));
}
}
@ -226,7 +233,105 @@ export class HDSegwitBech32Wallet extends AbstractHDWallet {
for (let tx of this._txs_by_internal_index[c]) hasUnconfirmed = hasUnconfirmed || !tx.confirmations || tx.confirmations < 7;
if (hasUnconfirmed || this._txs_by_internal_index[c].length === 0 || this._balances_by_internal_index[c].u !== 0) {
this._txs_by_internal_index[c] = await BlueElectrum.getTransactionsFullByAddress(this._getInternalAddressByIndex(c));
addresses2fetch.push(this._getInternalAddressByIndex(c));
}
}
// first: batch fetch for all addresses histories
let histories = await BlueElectrum.multiGetHistoryByAddress(addresses2fetch);
let txs = {};
for (let history of Object.values(histories)) {
for (let tx of history) {
txs[tx.tx_hash] = tx;
}
}
// next, batch fetching each txid we got
let txdatas = await BlueElectrum.multiGetTransactionByTxid(Object.keys(txs));
// now, tricky part. we collect all transactions from inputs (vin), and batch fetch them too.
// then we combine all this data (we need inputs to see source addresses and amounts)
let vinTxids = [];
for (let txdata of Object.values(txdatas)) {
for (let vin of txdata.vin) {
vinTxids.push(vin.txid);
}
}
let vintxdatas = await BlueElectrum.multiGetTransactionByTxid(vinTxids);
// fetched all transactions from our inputs. now we need to combine it.
// iterating all _our_ transactions:
for (let txid of Object.keys(txdatas)) {
// iterating all inputs our our single transaction:
for (let inpNum = 0; inpNum < txdatas[txid].vin.length; inpNum++) {
let inpTxid = txdatas[txid].vin[inpNum].txid;
let inpVout = txdatas[txid].vin[inpNum].vout;
// got txid and output number of _previous_ transaction we shoud look into
if (vintxdatas[inpTxid] && vintxdatas[inpTxid].vout[inpVout]) {
// extracting amount & addresses from previous output and adding it to _our_ input:
txdatas[txid].vin[inpNum].addresses = vintxdatas[inpTxid].vout[inpVout].scriptPubKey.addresses;
txdatas[txid].vin[inpNum].value = vintxdatas[inpTxid].vout[inpVout].value;
}
}
}
// now, we need to put transactions in all relevant `cells` of internal hashmaps: this._txs_by_internal_index && this._txs_by_external_index
for (let c = 0; c < this.next_free_address_index + this.gap_limit; c++) {
for (let tx of Object.values(txdatas)) {
for (let vin of tx.vin) {
if (vin.addresses.indexOf(this._getExternalAddressByIndex(c)) !== -1) {
// this TX is related to our address
this._txs_by_external_index[c] = this._txs_by_external_index[c] || {};
let clonedTx = Object.assign({}, tx);
clonedTx.inputs = tx.vin.slice(0);
clonedTx.outputs = tx.vout.slice(0);
delete clonedTx.vin;
delete clonedTx.vout;
this._txs_by_external_index[c].push(clonedTx);
}
}
for (let vout of tx.vout) {
if (vout.scriptPubKey.addresses.indexOf(this._getExternalAddressByIndex(c)) !== -1) {
// this TX is related to our address
this._txs_by_external_index[c] = this._txs_by_external_index[c] || {};
let clonedTx = Object.assign({}, tx);
clonedTx.inputs = tx.vin.slice(0);
clonedTx.outputs = tx.vout.slice(0);
delete clonedTx.vin;
delete clonedTx.vout;
this._txs_by_external_index[c].push(clonedTx);
}
}
}
}
for (let c = 0; c < this.next_free_change_address_index + this.gap_limit; c++) {
for (let tx of Object.values(txdatas)) {
for (let vin of tx.vin) {
if (vin.addresses.indexOf(this._getInternalAddressByIndex(c)) !== -1) {
// this TX is related to our address
this._txs_by_internal_index[c] = this._txs_by_internal_index[c] || [];
let clonedTx = Object.assign({}, tx);
clonedTx.inputs = tx.vin.slice(0);
clonedTx.outputs = tx.vout.slice(0);
delete clonedTx.vin;
delete clonedTx.vout;
this._txs_by_internal_index[c].push(clonedTx);
}
}
for (let vout of tx.vout) {
if (vout.scriptPubKey.addresses.indexOf(this._getInternalAddressByIndex(c)) !== -1) {
// this TX is related to our address
this._txs_by_internal_index[c] = this._txs_by_internal_index[c] || [];
let clonedTx = Object.assign({}, tx);
clonedTx.inputs = tx.vin.slice(0);
clonedTx.outputs = tx.vout.slice(0);
delete clonedTx.vin;
delete clonedTx.vout;
this._txs_by_internal_index[c].push(clonedTx);
}
}
}
}
@ -260,7 +365,7 @@ export class HDSegwitBech32Wallet extends AbstractHDWallet {
for (let vout of tx.outputs) {
// when output goes to our address - this means we are gaining!
if (vout.addresses && vout.addresses[0] && this.weOwnAddress(vout.scriptPubKey.addresses[0])) {
if (vout.scriptPubKey.addresses && vout.scriptPubKey.addresses[0] && this.weOwnAddress(vout.scriptPubKey.addresses[0])) {
tx.value += new BigNumber(vout.value).multipliedBy(100000000).toNumber();
}
}

50
tests/integration/Electrum.test.js
View file

@ -134,4 +134,54 @@ describe('Electrum', () => {
assert.strictEqual(utxos['bc1qt4t9xl2gmjvxgmp5gev6m8e6s9c85979ta7jeh'][0].value, 50000);
assert.strictEqual(utxos['bc1qt4t9xl2gmjvxgmp5gev6m8e6s9c85979ta7jeh'][0].address, 'bc1qt4t9xl2gmjvxgmp5gev6m8e6s9c85979ta7jeh');
});
it('ElectrumClient can do multiGetHistoryByAddress()', async () => {
let histories = await BlueElectrum.multiGetHistoryByAddress(
[
'bc1qt4t9xl2gmjvxgmp5gev6m8e6s9c85979ta7jeh',
'bc1qvd6w54sydc08z3802svkxr7297ez7cusd6266p',
'bc1qwp58x4c9e5cplsnw5096qzdkae036ug7a34x3r',
'bc1qcg6e26vtzja0h8up5w2m7utex0fsu4v0e0e7uy',
'bc1qcg6e26vtzja0h8up5w2m7utex0fsu4v0e0e7uy', // duplicate intended
],
3,
);
assert.ok(
histories['bc1qt4t9xl2gmjvxgmp5gev6m8e6s9c85979ta7jeh'][0]['tx_hash'] ===
'ad00a92409d8982a1d7f877056dbed0c4337d2ebab70b30463e2802279fb936d',
);
assert.ok(
histories['bc1qcg6e26vtzja0h8up5w2m7utex0fsu4v0e0e7uy'][0]['tx_hash'] ===
'5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df',
);
assert.ok(Object.keys(histories).length === 4);
});
it('ElectrumClient can do multiGetHistoryByAddress()', async () => {
let txdatas = await BlueElectrum.multiGetTransactionByTxid(
[
'ad00a92409d8982a1d7f877056dbed0c4337d2ebab70b30463e2802279fb936d',
'042c9e276c2d06b0b84899771a7f218af90dd60436947c49a844a05d7c104b26',
'2cf439be65e7cc7c6e4db721b1c8fcb1cd95ff07cde79a52a73b3d15a12b2eb6',
'5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df',
'5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df', // duplicate intended
],
3,
);
assert.ok(
txdatas['ad00a92409d8982a1d7f877056dbed0c4337d2ebab70b30463e2802279fb936d'].txid ===
'ad00a92409d8982a1d7f877056dbed0c4337d2ebab70b30463e2802279fb936d',
);
assert.ok(
txdatas['5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df'].txid ===
'5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df',
);
assert.ok(txdatas['5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df'].size);
assert.ok(txdatas['5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df'].vin);
assert.ok(txdatas['5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df'].vout);
assert.ok(txdatas['5e2fa84148a7389537434b3ad12fcae71ed43ce5fb0f016a7f154a9b99a973df'].blocktime);
assert.ok(Object.keys(txdatas).length === 4);
});
});

13
tests/integration/HDBech32Wallet.test.js
View file

@ -203,7 +203,7 @@ describe('Bech32 Segwit HD (BIP84)', () => {
assert.strictEqual(hd.getTransactions().length, oldTransactions.length);
});
it('can create transactions', async () => {
it('can fetchBalance, fetchTransactions, fetchUtxo and create transactions', async () => {
if (!process.env.HD_MNEMONIC_BIP84) {
console.error('process.env.HD_MNEMONIC_BIP84 not set, skipped');
return;
@ -223,6 +223,17 @@ describe('Bech32 Segwit HD (BIP84)', () => {
end = +new Date();
end - start > 15000 && console.warn('fetchTransactions took', (end - start) / 1000, 'sec');
start = +new Date();
await hd.fetchBalance();
end = +new Date();
end - start > 2000 && console.warn('warm fetchBalance took', (end - start) / 1000, 'sec');
global.debug = true;
start = +new Date();
await hd.fetchTransactions();
end = +new Date();
end - start > 2000 && console.warn('warm fetchTransactions took', (end - start) / 1000, 'sec');
let txFound = 0;
for (let tx of hd.getTransactions()) {
if (tx.hash === 'e9ef58baf4cff3ad55913a360c2fa1fd124309c59dcd720cdb172ce46582097b') {