diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 2e7e09316..9b55905f4 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -1,15 +1,27 @@ { "MEMPOOL": { "NETWORK": "mainnet", + "BACKEND": "electrum", "HTTP_PORT": 8999, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "REST_API_URL": "http://127.0.0.1:3000", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "HOST": "127.0.0.1", + "PORT": 8332, + "USERNAME": "mempool", + "PASSWORD": "mempool" + }, + "ELECTRUM": { + "HOST": "127.0.0.1", + "PORT": 50002, + "TLS_ENABLED": true, + "TX_LOOKUPS": false + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:3000" + }, "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", diff --git a/backend/package.json b/backend/package.json index 369308e71..0d6576fd3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,7 +27,11 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "dependencies": { + "@mempool/bitcoin": "^3.0.2", + "@mempool/electrum-client": "^1.1.7", "axios": "^0.21.0", + "bitcoinjs-lib": "^5.2.0", + "crypto-js": "^4.0.0", "express": "^4.17.1", "locutus": "^2.0.12", "mysql2": "^1.6.1", diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 8c4d89ec6..7824d30ff 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import axios from 'axios'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { Common } from '../common'; -import { Block } from '../../interfaces'; +import { BlockExtended } from '../../mempool.interfaces'; import { StaticPool } from 'node-worker-threads-pool'; import logger from '../../logger'; @@ -42,7 +42,7 @@ class Bisq { this.startSubDirectoryWatcher(); } - handleNewBitcoinBlock(block: Block): void { + handleNewBitcoinBlock(block: BlockExtended): void { if (block.height - 2 > this.latestBlockHeight && this.latestBlockHeight !== 0) { logger.warn(`Bitcoin block height (#${block.height}) has diverged from the latest Bisq block height (#${this.latestBlockHeight}). Restarting watchers...`); this.startTopDirectoryWatcher(); diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts new file mode 100644 index 000000000..951d9576c --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -0,0 +1,14 @@ +import { IEsploraApi } from './esplora-api.interface'; + +export interface AbstractBitcoinApi { + $getRawMempool(): Promise; + $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise; + $getRawTransactionBitcoind(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise; + $getBlockHeightTip(): Promise; + $getTxIdsForBlock(hash: string): Promise; + $getBlockHash(height: number): Promise; + $getBlock(hash: string): Promise; + $getAddress(address: string): Promise; + $getAddressTransactions(address: string, lastSeenTxId: string): Promise; + $getAddressPrefix(prefix: string): string[]; +} diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts new file mode 100644 index 000000000..a16521a0e --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts @@ -0,0 +1,19 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import EsploraApi from './esplora-api'; +import BitcoinApi from './bitcoin-api'; +import ElectrumApi from './electrum-api'; + +function bitcoinApiFactory(): AbstractBitcoinApi { + switch (config.MEMPOOL.BACKEND) { + case 'esplora': + return new EsploraApi(); + case 'electrum': + return new ElectrumApi(); + case 'none': + default: + return new BitcoinApi(); + } +} + +export default bitcoinApiFactory(); diff --git a/backend/src/api/bitcoin/bitcoin-api.interface.ts b/backend/src/api/bitcoin/bitcoin-api.interface.ts new file mode 100644 index 000000000..194beebee --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api.interface.ts @@ -0,0 +1,116 @@ +export namespace IBitcoinApi { + export interface MempoolInfo { + loaded: boolean; // (boolean) True if the mempool is fully loaded + size: number; // (numeric) Current tx count + bytes: number; // (numeric) Sum of all virtual transaction sizes as defined in BIP 141. + usage: number; // (numeric) Total memory usage for the mempool + maxmempool: number; // (numeric) Maximum memory usage for the mempool + mempoolminfee: number; // (numeric) Minimum fee rate in BTC/kB for tx to be accepted. + minrelaytxfee: number; // (numeric) Current minimum relay fee for transactions + } + + export interface RawMempool { [txId: string]: MempoolEntry; } + + export interface MempoolEntry { + vsize: number; // (numeric) virtual transaction size as defined in BIP 141. + weight: number; // (numeric) transaction weight as defined in BIP 141. + time: number; // (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT + height: number; // (numeric) block height when transaction entered pool + descendantcount: number; // (numeric) number of in-mempool descendant transactions (including this one) + descendantsize: number; // (numeric) virtual transaction size of in-mempool descendants (including this one) + ancestorcount: number; // (numeric) number of in-mempool ancestor transactions (including this one) + ancestorsize: number; // (numeric) virtual transaction size of in-mempool ancestors (including this one) + wtxid: string; // (string) hash of serialized transactionumber; including witness data + fees: { + base: number; // (numeric) transaction fee in BTC + modified: number; // (numeric) transaction fee with fee deltas used for mining priority in BTC + ancestor: number; // (numeric) modified fees (see above) of in-mempool ancestors (including this one) in BTC + descendant: number; // (numeric) modified fees (see above) of in-mempool descendants (including this one) in BTC + }; + depends: string[]; // (string) parent transaction id + spentby: string[]; // (array) unconfirmed transactions spending outputs from this transaction + 'bip125-replaceable': boolean; // (boolean) Whether this transaction could be replaced due to BIP125 (replace-by-fee) + } + + export interface Block { + hash: string; // (string) the block hash (same as provided) + confirmations: number; // (numeric) The number of confirmations, or -1 if the block is not on the main chain + size: number; // (numeric) The block size + strippedsize: number; // (numeric) The block size excluding witness data + weight: number; // (numeric) The block weight as defined in BIP 141 + height: number; // (numeric) The block height or index + version: number; // (numeric) The block version + versionHex: string; // (string) The block version formatted in hexadecimal + merkleroot: string; // (string) The merkle root + tx: Transaction[]; + time: number; // (numeric) The block time expressed in UNIX epoch time + mediantime: number; // (numeric) The median block time expressed in UNIX epoch time + nonce: number; // (numeric) The nonce + bits: string; // (string) The bits + difficulty: number; // (numeric) The difficulty + chainwork: string; // (string) Expected number of hashes required to produce the chain up to this block (in hex) + nTx: number; // (numeric) The number of transactions in the block + previousblockhash: string; // (string) The hash of the previous block + nextblockhash: string; // (string) The hash of the next block + } + + export interface Transaction { + in_active_chain: boolean; // (boolean) Whether specified block is in the active chain or not + hex: string; // (string) The serialized, hex-encoded data for 'txid' + txid: string; // (string) The transaction id (same as provided) + hash: string; // (string) The transaction hash (differs from txid for witness transactions) + size: number; // (numeric) The serialized transaction size + vsize: number; // (numeric) The virtual transaction size (differs from size for witness transactions) + weight: number; // (numeric) The transaction's weight (between vsize*4-3 and vsize*4) + version: number; // (numeric) The version + locktime: number; // (numeric) The lock time + vin: Vin[]; + vout: Vout[]; + blockhash: string; // (string) the block hash + confirmations: number; // (numeric) The confirmations + blocktime: number; // (numeric) The block time expressed in UNIX epoch time + time: number; // (numeric) Same as blocktime + } + + interface Vin { + txid?: string; // (string) The transaction id + vout?: number; // (string) + scriptSig?: { // (json object) The script + asm: string; // (string) asm + hex: string; // (string) hex + }; + sequence: number; // (numeric) The script sequence number + txinwitness?: string[]; // (string) hex-encoded witness data + coinbase?: string; + } + + interface Vout { + value: number; // (numeric) The value in BTC + n: number; // (numeric) index + scriptPubKey: { // (json object) + asm: string; // (string) the asm + hex: string; // (string) the hex + reqSigs: number; // (numeric) The required sigs + type: string; // (string) The type, eg 'pubkeyhash' + addresses: string[] // (string) bitcoin address + }; + } + + export interface AddressInformation { + isvalid: boolean; // (boolean) If the address is valid or not. If not, this is the only property returned. + address: string; // (string) The bitcoin address validated + scriptPubKey: string; // (string) The hex-encoded scriptPubKey generated by the address + isscript: boolean; // (boolean) If the key is a script + iswitness: boolean; // (boolean) If the address is a witness + witness_version?: boolean; // (numeric, optional) The version number of the witness program + witness_program: string; // (string, optional) The hex value of the witness program + } + + export interface ChainTips { + height: number; // (numeric) height of the chain tip + hash: string; // (string) block hash of the tip + branchlen: number; // (numeric) zero for main chain, otherwise length of branch connecting the tip to the main chain + status: 'invalid' | 'headers-only' | 'valid-headers' | 'valid-fork' | 'active'; + } + +} diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts new file mode 100644 index 000000000..c40ab031c --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -0,0 +1,298 @@ +import config from '../../config'; +import * as bitcoin from '@mempool/bitcoin'; +import * as bitcoinjs from 'bitcoinjs-lib'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { IBitcoinApi } from './bitcoin-api.interface'; +import { IEsploraApi } from './esplora-api.interface'; +import blocks from '../blocks'; +import bitcoinBaseApi from './bitcoin-base.api'; +import mempool from '../mempool'; +import { TransactionExtended } from '../../mempool.interfaces'; + +class BitcoinApi implements AbstractBitcoinApi { + private rawMempoolCache: IBitcoinApi.RawMempool | null = null; + private bitcoindClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, + timeout: 60000, + }); + } + + $getRawTransactionBitcoind(txId: string, skipConversion = false, addPrevout = false): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: IBitcoinApi.Transaction) => { + if (skipConversion) { + return transaction; + } + return this.$convertTransaction(transaction, addPrevout); + }); + } + + $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && addPrevout) { + return this.$addPrevouts(txInMempool); + } + + // Special case to fetch the Coinbase transaction + if (txId === '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b') { + return this.$returnCoinbaseTransaction(); + } + + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: IBitcoinApi.Transaction) => { + if (skipConversion) { + return transaction; + } + return this.$convertTransaction(transaction, addPrevout); + }); + } + + $getBlockHeightTip(): Promise { + return this.bitcoindClient.getChainTips() + .then((result: IBitcoinApi.ChainTips[]) => result[0].height); + } + + $getTxIdsForBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash, 1) + .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx); + } + + $getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height); + } + + async $getBlock(hash: string): Promise { + const foundBlock = blocks.getBlocks().find((block) => block.id === hash); + if (foundBlock) { + return foundBlock; + } + + return this.bitcoindClient.getBlock(hash) + .then((block: IBitcoinApi.Block) => this.convertBlock(block)); + } + + $getAddress(address: string): Promise { + throw new Error('Method getAddress not supported by the Bitcoin RPC API.'); + } + + $getAddressTransactions(address: string, lastSeenTxId: string): Promise { + throw new Error('Method getAddressTransactions not supported by the Bitcoin RPC API.'); + } + + $getRawMempool(): Promise { + return this.bitcoindClient.getRawMemPool(); + } + + $getAddressPrefix(prefix: string): string[] { + const found: string[] = []; + const mp = mempool.getMempool(); + for (const tx in mp) { + for (const vout of mp[tx].vout) { + if (vout.scriptpubkey_address.indexOf(prefix) === 0) { + found.push(vout.scriptpubkey_address); + if (found.length >= 10) { + return found; + } + } + } + } + return found; + } + + protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise { + let esploraTransaction: IEsploraApi.Transaction = { + txid: transaction.txid, + version: transaction.version, + locktime: transaction.locktime, + size: transaction.size, + weight: transaction.weight, + fee: 0, + vin: [], + vout: [], + status: { confirmed: false }, + }; + + esploraTransaction.vout = transaction.vout.map((vout) => { + return { + value: vout.value * 100000000, + scriptpubkey: vout.scriptPubKey.hex, + scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '', + scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '', + scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), + }; + }); + + esploraTransaction.vin = transaction.vin.map((vin) => { + return { + is_coinbase: !!vin.coinbase, + prevout: null, + scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '', + scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '', + sequence: vin.sequence, + txid: vin.txid || '', + vout: vin.vout || 0, + witness: vin.txinwitness, + }; + }); + + if (transaction.confirmations) { + esploraTransaction.status = { + confirmed: true, + block_height: blocks.getCurrentBlockHeight() - transaction.confirmations + 1, + block_hash: transaction.blockhash, + block_time: transaction.blocktime, + }; + } + + if (transaction.confirmations) { + esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout); + } else { + esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction); + } + + return esploraTransaction; + } + + private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block { + return { + id: block.hash, + height: block.height, + version: block.version, + timestamp: block.time, + bits: parseInt(block.bits, 16), + nonce: block.nonce, + difficulty: block.difficulty, + merkle_root: block.merkleroot, + tx_count: block.nTx, + size: block.size, + weight: block.weight, + previousblockhash: block.previousblockhash, + }; + } + + private translateScriptPubKeyType(outputType: string): string { + const map = { + 'pubkey': 'p2pk', + 'pubkeyhash': 'p2pkh', + 'scripthash': 'p2sh', + 'witness_v0_keyhash': 'v0_p2wpkh', + 'witness_v0_scripthash': 'v0_p2wsh', + 'witness_v1_taproot': 'v1_p2tr', + 'nonstandard': 'nonstandard', + 'nulldata': 'op_return' + }; + + if (map[outputType]) { + return map[outputType]; + } else { + return ''; + } + } + + private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise { + if (transaction.fee) { + return transaction; + } + let mempoolEntry: IBitcoinApi.MempoolEntry; + if (!mempool.isInSync() && !this.rawMempoolCache) { + this.rawMempoolCache = await bitcoinBaseApi.$getRawMempoolVerbose(); + } + if (this.rawMempoolCache && this.rawMempoolCache[transaction.txid]) { + mempoolEntry = this.rawMempoolCache[transaction.txid]; + } else { + mempoolEntry = await bitcoinBaseApi.$getMempoolEntry(transaction.txid); + } + transaction.fee = mempoolEntry.fees.base * 100000000; + return transaction; + } + + protected async $addPrevouts(transaction: TransactionExtended): Promise { + for (const vin of transaction.vin) { + if (vin.prevout) { + continue; + } + const innerTx = await this.$getRawTransaction(vin.txid, false); + vin.prevout = innerTx.vout[vin.vout]; + this.addInnerScriptsToVin(vin); + } + return transaction; + } + + protected $returnCoinbaseTransaction(): Promise { + return this.bitcoindClient.getBlock('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', 2) + .then((block: IBitcoinApi.Block) => { + return this.$convertTransaction(Object.assign(block.tx[0], { + confirmations: blocks.getCurrentBlockHeight() + 1, + blocktime: 1231006505 }), false); + }); + } + + protected $validateAddress(address: string): Promise { + return this.bitcoindClient.validateAddress(address); + } + + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { + if (transaction.vin[0].is_coinbase) { + transaction.fee = 0; + return transaction; + } + let totalIn = 0; + for (const vin of transaction.vin) { + const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout); + if (addPrevout) { + vin.prevout = innerTx.vout[vin.vout]; + this.addInnerScriptsToVin(vin); + } + totalIn += innerTx.vout[vin.vout].value; + } + const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0); + transaction.fee = parseFloat((totalIn - totalOut).toFixed(8)); + return transaction; + } + + private convertScriptSigAsm(str: string): string { + const a = str.split(' '); + const b: string[] = []; + a.forEach((chunk) => { + if (chunk.substr(0, 3) === 'OP_') { + chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1'); + chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV'); + b.push(chunk); + } else { + chunk = chunk.replace('[ALL]', '01'); + if (chunk === '0') { + b.push('OP_0'); + } else { + b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk); + } + } + }); + return b.join(' '); + } + + private addInnerScriptsToVin(vin: IEsploraApi.Vin): void { + if (!vin.prevout) { + return; + } + + if (vin.prevout.scriptpubkey_type === 'p2sh') { + const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0]; + vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex'))); + } + + if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) { + const witnessScript = vin.witness[vin.witness.length - 1]; + vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex'))); + } + } + +} + +export default BitcoinApi; diff --git a/backend/src/api/bitcoin/bitcoin-base.api.ts b/backend/src/api/bitcoin/bitcoin-base.api.ts new file mode 100644 index 000000000..71e7c9093 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-base.api.ts @@ -0,0 +1,36 @@ +import config from '../../config'; +import * as bitcoin from '@mempool/bitcoin'; +import { IBitcoinApi } from './bitcoin-api.interface'; + +class BitcoinBaseApi { + bitcoindClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, + timeout: 60000, + }); + } + + $getMempoolInfo(): Promise { + return this.bitcoindClient.getMempoolInfo(); + } + + $getRawTransaction(txId: string): Promise { + return this.bitcoindClient.getRawTransaction(txId, true); + } + + $getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid); + } + + $getRawMempoolVerbose(): Promise { + return this.bitcoindClient.getRawMemPool(true); + } + +} + +export default new BitcoinBaseApi(); diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts deleted file mode 100644 index 0530f0cfa..000000000 --- a/backend/src/api/bitcoin/electrs-api.ts +++ /dev/null @@ -1,56 +0,0 @@ -import config from '../../config'; -import { Transaction, Block, MempoolInfo } from '../../interfaces'; -import axios from 'axios'; - -class ElectrsApi { - - constructor() { - } - - getMempoolInfo(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/mempool', { timeout: 10000 }) - .then((response) => { - return { - size: response.data.count, - bytes: response.data.vsize, - }; - }); - } - - getRawMempool(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/mempool/txids') - .then((response) => response.data); - } - - getRawTransaction(txId: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/tx/' + txId) - .then((response) => response.data); - } - - getBlockHeightTip(): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/blocks/tip/height') - .then((response) => response.data); - } - - getTxIdsForBlock(hash: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids') - .then((response) => response.data); - } - - getBlockHash(height: number): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block-height/' + height) - .then((response) => response.data); - } - - getBlocksFromHeight(height: number): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/blocks/' + height) - .then((response) => response.data); - } - - getBlock(hash: string): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) - .then((response) => response.data); - } -} - -export default new ElectrsApi(); diff --git a/backend/src/api/bitcoin/electrum-api.interface.ts b/backend/src/api/bitcoin/electrum-api.interface.ts new file mode 100644 index 000000000..633de3cbc --- /dev/null +++ b/backend/src/api/bitcoin/electrum-api.interface.ts @@ -0,0 +1,12 @@ +export namespace IElectrumApi { + export interface ScriptHashBalance { + confirmed: number; + unconfirmed: number; + } + + export interface ScriptHashHistory { + height: number; + tx_hash: string; + fee?: number; + } +} diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts new file mode 100644 index 000000000..6c275356b --- /dev/null +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -0,0 +1,179 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { IBitcoinApi } from './bitcoin-api.interface'; +import { IEsploraApi } from './esplora-api.interface'; +import { IElectrumApi } from './electrum-api.interface'; +import BitcoinApi from './bitcoin-api'; +import mempool from '../mempool'; +import logger from '../../logger'; +import * as ElectrumClient from '@mempool/electrum-client'; +import * as sha256 from 'crypto-js/sha256'; +import * as hexEnc from 'crypto-js/enc-hex'; +import loadingIndicators from '../loading-indicators'; +import memoryCache from '../memory-cache'; + +class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { + private electrumClient: any; + + constructor() { + super(); + + const electrumConfig = { client: 'mempool-v2', version: '1.4' }; + const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null }; + + const electrumCallbacks = { + onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); }, + onClose: (client) => { logger.info(`Disconnected from Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`); }, + onError: (err) => { logger.err(`Electrum error: ${JSON.stringify(err)}`); }, + onLog: (str) => { logger.debug(str); }, + }; + + this.electrumClient = new ElectrumClient( + config.ELECTRUM.PORT, + config.ELECTRUM.HOST, + config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp', + null, + electrumCallbacks + ); + + this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy) + .then(() => {}) + .catch((err) => { + logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`); + }); + } + + async $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + if (!config.ELECTRUM.TX_LOOKUPS) { + return super.$getRawTransaction(txId, skipConversion, addPrevout); + } + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && addPrevout) { + return this.$addPrevouts(txInMempool); + } + const transaction: IBitcoinApi.Transaction = await this.electrumClient.blockchainTransaction_get(txId, true); + if (!transaction) { + throw new Error('Unable to get transaction: ' + txId); + } + if (skipConversion) { + // @ts-ignore + return transaction; + } + return this.$convertTransaction(transaction, addPrevout); + } + + async $getAddress(address: string): Promise { + const addressInfo = await this.$validateAddress(address); + if (!addressInfo || !addressInfo.isvalid) { + return ({ + 'address': address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': 0, + 'spent_txo_count': 0, + 'spent_txo_sum': 0, + 'tx_count': 0 + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': 0, + 'spent_txo_count': 0, + 'spent_txo_sum': 0, + 'tx_count': 0 + } + }); + } + + try { + const balance = await this.$getScriptHashBalance(addressInfo.scriptPubKey); + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + + const unconfirmed = history.filter((h) => h.fee).length; + + return { + 'address': addressInfo.address, + 'chain_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.confirmed ? balance.confirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0, + 'tx_count': history.length - unconfirmed, + }, + 'mempool_stats': { + 'funded_txo_count': 0, + 'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0, + 'spent_txo_count': 0, + 'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0, + 'tx_count': unconfirmed, + } + }; + } catch (e) { + if (e === 'failed to get confirmed status') { + e = 'The number of transactions on this address exceeds the Electrum server limit'; + } + throw new Error(e); + } + } + + async $getAddressTransactions(address: string, lastSeenTxId: string): Promise { + const addressInfo = await this.$validateAddress(address); + if (!addressInfo || !addressInfo.isvalid) { + return []; + } + + try { + loadingIndicators.setProgress('address-' + address, 0); + + const transactions: IEsploraApi.Transaction[] = []; + const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); + history.sort((a, b) => (b.height || 9999999) - (a.height || 9999999)); + + let startingIndex = 0; + if (lastSeenTxId) { + const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId); + if (pos) { + startingIndex = pos + 1; + } + } + const endIndex = Math.min(startingIndex + 10, history.length); + + for (let i = startingIndex; i < endIndex; i++) { + const tx = await this.$getRawTransaction(history[i].tx_hash, false, true); + transactions.push(tx); + loadingIndicators.setProgress('address-' + address, (i + 1) / endIndex * 100); + } + + return transactions; + } catch (e) { + loadingIndicators.setProgress('address-' + address, 100); + if (e === 'failed to get confirmed status') { + e = 'The number of transactions on this address exceeds the Electrum server limit'; + } + throw new Error(e); + } + } + + private $getScriptHashBalance(scriptHash: string): Promise { + return this.electrumClient.blockchainScripthash_getBalance(this.encodeScriptHash(scriptHash)); + } + + private $getScriptHashHistory(scriptHash: string): Promise { + const fromCache = memoryCache.get('Scripthash_getHistory', scriptHash); + if (fromCache) { + return Promise.resolve(fromCache); + } + return this.electrumClient.blockchainScripthash_getHistory(this.encodeScriptHash(scriptHash)) + .then((history) => { + memoryCache.set('Scripthash_getHistory', scriptHash, history, 2); + return history; + }); + } + + private encodeScriptHash(scriptPubKey: string): string { + const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey))); + return addrScripthash.match(/.{2}/g).reverse().join(''); + } + +} + +export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts new file mode 100644 index 000000000..cfabe40bc --- /dev/null +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -0,0 +1,168 @@ +export namespace IEsploraApi { + export interface Transaction { + txid: string; + version: number; + locktime: number; + size: number; + weight: number; + fee: number; + vin: Vin[]; + vout: Vout[]; + status: Status; + } + + export interface Recent { + txid: string; + fee: number; + vsize: number; + value: number; + } + + export interface Vin { + txid: string; + vout: number; + is_coinbase: boolean; + scriptsig: string; + scriptsig_asm: string; + inner_redeemscript_asm?: string; + inner_witnessscript_asm?: string; + sequence: any; + witness?: string[]; + prevout: Vout | null; + // Elements + is_pegin?: boolean; + issuance?: Issuance; + } + + interface Issuance { + asset_id: string; + is_reissuance: string; + asset_blinding_nonce: string; + asset_entropy: string; + contract_hash: string; + assetamount?: number; + assetamountcommitment?: string; + tokenamount?: number; + tokenamountcommitment?: string; + } + + export interface Vout { + scriptpubkey: string; + scriptpubkey_asm: string; + scriptpubkey_type: string; + scriptpubkey_address: string; + value: number; + // Elements + valuecommitment?: number; + asset?: string; + pegout?: Pegout; + } + + interface Pegout { + genesis_hash: string; + scriptpubkey: string; + scriptpubkey_asm: string; + scriptpubkey_address: string; + } + + export interface Status { + confirmed: boolean; + block_height?: number; + block_hash?: string; + block_time?: number; + } + + export interface Block { + id: string; + height: number; + version: number; + timestamp: number; + bits: number; + nonce: number; + difficulty: number; + merkle_root: string; + tx_count: number; + size: number; + weight: number; + previousblockhash: string; + } + + export interface Address { + address: string; + chain_stats: ChainStats; + mempool_stats: MempoolStats; + } + + export interface ChainStats { + funded_txo_count: number; + funded_txo_sum: number; + spent_txo_count: number; + spent_txo_sum: number; + tx_count: number; + } + + export interface MempoolStats { + funded_txo_count: number; + funded_txo_sum: number; + spent_txo_count: number; + spent_txo_sum: number; + tx_count: number; + } + + export interface Outspend { + spent: boolean; + txid: string; + vin: number; + status: Status; + } + + export interface Asset { + asset_id: string; + issuance_txin: IssuanceTxin; + issuance_prevout: IssuancePrevout; + reissuance_token: string; + contract_hash: string; + status: Status; + chain_stats: AssetStats; + mempool_stats: AssetStats; + } + + export interface AssetExtended extends Asset { + name: string; + ticker: string; + precision: number; + entity: Entity; + version: number; + issuer_pubkey: string; + } + + export interface Entity { + domain: string; + } + + interface IssuanceTxin { + txid: string; + vin: number; + } + + interface IssuancePrevout { + txid: string; + vout: number; + } + + interface AssetStats { + tx_count: number; + issuance_count: number; + issued_amount: number; + burned_amount: number; + has_blinded_issuances: boolean; + reissuance_tokens: number; + burned_reissuance_tokens: number; + peg_in_count: number; + peg_in_amount: number; + peg_out_count: number; + peg_out_amount: number; + burn_count: number; + } + +} diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts new file mode 100644 index 000000000..24220da3e --- /dev/null +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -0,0 +1,58 @@ +import config from '../../config'; +import axios from 'axios'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { IEsploraApi } from './esplora-api.interface'; + +class ElectrsApi implements AbstractBitcoinApi { + + constructor() { } + + $getRawMempool(): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/mempool/txids') + .then((response) => response.data); + } + + $getRawTransaction(txId: string): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId) + .then((response) => response.data); + } + + $getBlockHeightTip(): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/blocks/tip/height') + .then((response) => response.data); + } + + $getTxIdsForBlock(hash: string): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids') + .then((response) => response.data); + } + + $getBlockHash(height: number): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/block-height/' + height) + .then((response) => response.data); + } + + $getBlock(hash: string): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash) + .then((response) => response.data); + } + + $getAddress(address: string): Promise { + throw new Error('Method getAddress not implemented.'); + } + + $getAddressTransactions(address: string, txId?: string): Promise { + throw new Error('Method getAddressTransactions not implemented.'); + } + + $getRawTransactionBitcoind(txId: string): Promise { + return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId) + .then((response) => response.data); + } + + $getAddressPrefix(prefix: string): string[] { + throw new Error('Method not implemented.'); + } +} + +export default ElectrsApi; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index de1d702aa..4258b5260 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -1,49 +1,51 @@ -import bitcoinApi from './bitcoin/electrs-api'; +import config from '../config'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'; +import { BlockExtended, TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; +import transactionUtils from './transaction-utils'; class Blocks { - private static KEEP_BLOCK_AMOUNT = 8; - private blocks: Block[] = []; + private static INITIAL_BLOCK_AMOUNT = 8; + private blocks: BlockExtended[] = []; private currentBlockHeight = 0; private lastDifficultyAdjustmentTime = 0; - private newBlockCallbacks: ((block: Block, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; + private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; constructor() { } - public getBlocks(): Block[] { + public getBlocks(): BlockExtended[] { return this.blocks; } - public setBlocks(blocks: Block[]) { + public setBlocks(blocks: BlockExtended[]) { this.blocks = blocks; } - public setNewBlockCallback(fn: (block: Block, txIds: string[], transactions: TransactionExtended[]) => void) { + public setNewBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void) { this.newBlockCallbacks.push(fn); } public async $updateBlocks() { - const blockHeightTip = await bitcoinApi.getBlockHeightTip(); + const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); if (this.blocks.length === 0) { - this.currentBlockHeight = blockHeightTip - Blocks.KEEP_BLOCK_AMOUNT; + this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT; } else { this.currentBlockHeight = this.blocks[this.blocks.length - 1].height; } - if (blockHeightTip - this.currentBlockHeight > Blocks.KEEP_BLOCK_AMOUNT * 2) { - logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${Blocks.KEEP_BLOCK_AMOUNT} recent blocks`); - this.currentBlockHeight = blockHeightTip - Blocks.KEEP_BLOCK_AMOUNT; + if (blockHeightTip - this.currentBlockHeight > Blocks.INITIAL_BLOCK_AMOUNT * 2) { + logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${Blocks.INITIAL_BLOCK_AMOUNT} recent blocks`); + this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT; } if (!this.lastDifficultyAdjustmentTime) { const heightDiff = blockHeightTip % 2016; - const blockHash = await bitcoinApi.getBlockHash(blockHeightTip - heightDiff); - const block = await bitcoinApi.getBlock(blockHash); + const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); + const block = await bitcoinApi.$getBlock(blockHash); this.lastDifficultyAdjustmentTime = block.timestamp; } @@ -55,49 +57,64 @@ class Blocks { logger.debug(`New block found (#${this.currentBlockHeight})!`); } - const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight); - const block = await bitcoinApi.getBlock(blockHash); - const txIds = await bitcoinApi.getTxIdsForBlock(blockHash); - - const mempool = memPool.getMempool(); - let found = 0; - let notFound = 0; - const transactions: TransactionExtended[] = []; + const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); + const block = await bitcoinApi.$getBlock(blockHash); + const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); + + const mempool = memPool.getMempool(); + let transactionsFound = 0; + for (let i = 0; i < txIds.length; i++) { + // When using bitcoind, just fetch the coinbase tx for now + if (config.MEMPOOL.BACKEND !== 'esplora' && i === 0) { + let txFound = false; + let findCoinbaseTxTries = 0; + // It takes Electrum Server a few seconds to index the transaction after a block is found + while (findCoinbaseTxTries < 5 && !txFound) { + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); + if (tx) { + txFound = true; + transactions.push(tx); + } else { + await Common.sleep(1000); + findCoinbaseTxTries++; + } + } + } if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); - found++; - } else { + transactionsFound++; + } else if (config.MEMPOOL.BACKEND === 'esplora') { logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await memPool.getTransactionExtended(txIds[i]); + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); if (tx) { transactions.push(tx); } - notFound++; } } - logger.debug(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`); + logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`); - block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); + const blockExtended: BlockExtended = Object.assign({}, block); + blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); transactions.sort((a, b) => b.feePerVsize - a.feePerVsize); - block.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; - block.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; + blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; + blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; if (block.height % 2016 === 0) { this.lastDifficultyAdjustmentTime = block.timestamp; } - this.blocks.push(block); - if (this.blocks.length > Blocks.KEEP_BLOCK_AMOUNT) { - this.blocks = this.blocks.slice(-Blocks.KEEP_BLOCK_AMOUNT); + this.blocks.push(blockExtended); + if (this.blocks.length > Blocks.INITIAL_BLOCK_AMOUNT * 4) { + this.blocks = this.blocks.slice(-Blocks.INITIAL_BLOCK_AMOUNT * 4); } if (this.newBlockCallbacks.length) { - this.newBlockCallbacks.forEach((cb) => cb(block, txIds, transactions)); + this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions)); } diskCache.$saveCacheToDisk(); } @@ -107,15 +124,8 @@ class Blocks { return this.lastDifficultyAdjustmentTime; } - private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { - return { - vin: [{ - scriptsig: tx.vin[0].scriptsig - }], - vout: tx.vout - .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, value: vout.value })) - .filter((vout) => vout.value) - }; + public getCurrentBlockHeight(): number { + return this.currentBlockHeight; } } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 63256be9b..3d6fb7161 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { Transaction, TransactionExtended, TransactionStripped } from '../interfaces'; +import { TransactionExtended, TransactionStripped } from '../mempool.interfaces'; export class Common { static median(numbers: number[]) { @@ -53,7 +53,15 @@ export class Common { txid: tx.txid, fee: tx.fee, weight: tx.weight, - value: tx.vin.reduce((acc, vin) => acc + (vin.prevout ? vin.prevout.value : 0), 0), + value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), }; } + + static sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); + }); + } } diff --git a/backend/src/api/fee-api.ts b/backend/src/api/fee-api.ts index 551501240..8293520bf 100644 --- a/backend/src/api/fee-api.ts +++ b/backend/src/api/fee-api.ts @@ -1,5 +1,5 @@ import config from '../config'; -import { MempoolBlock } from '../interfaces'; +import { MempoolBlock } from '../mempool.interfaces'; import projectedBlocks from './mempool-blocks'; class FeeApi { diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 89916c91d..8de9f10bb 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -1,13 +1,19 @@ import logger from '../logger'; import axios from 'axios'; +import { IConversionRates } from '../mempool.interfaces'; class FiatConversion { - private conversionRates = { + private conversionRates: IConversionRates = { 'USD': 0 }; + private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined; constructor() { } + public setProgressChangedCallback(fn: (rates: IConversionRates) => void) { + this.ratesChangedCallback = fn; + } + public startService() { logger.info('Starting currency rates service'); setInterval(this.updateCurrency.bind(this), 1000 * 60 * 60); @@ -25,6 +31,9 @@ class FiatConversion { this.conversionRates = { 'USD': usd.price, }; + if (this.ratesChangedCallback) { + this.ratesChangedCallback(this.conversionRates); + } } catch (e) { logger.err('Error updating fiat conversion rates: ' + e); } diff --git a/backend/src/api/loading-indicators.ts b/backend/src/api/loading-indicators.ts new file mode 100644 index 000000000..c2d682d1c --- /dev/null +++ b/backend/src/api/loading-indicators.ts @@ -0,0 +1,32 @@ +import { ILoadingIndicators } from '../mempool.interfaces'; + +class LoadingIndicators { + private loadingIndicators: ILoadingIndicators = { + 'mempool': 0, + }; + private progressChangedCallback: ((loadingIndicators: ILoadingIndicators) => void) | undefined; + + constructor() { } + + public setProgressChangedCallback(fn: (loadingIndicators: ILoadingIndicators) => void) { + this.progressChangedCallback = fn; + } + + public setProgress(name: string, progressPercent: number) { + const newProgress = Math.round(progressPercent); + if (newProgress >= 100) { + delete this.loadingIndicators[name]; + } else { + this.loadingIndicators[name] = newProgress; + } + if (this.progressChangedCallback) { + this.progressChangedCallback(this.loadingIndicators); + } + } + + public getLoadingIndicators() { + return this.loadingIndicators; + } +} + +export default new LoadingIndicators(); diff --git a/backend/src/api/memory-cache.ts b/backend/src/api/memory-cache.ts new file mode 100644 index 000000000..fe4162420 --- /dev/null +++ b/backend/src/api/memory-cache.ts @@ -0,0 +1,38 @@ +interface ICache { + type: string; + id: string; + expires: Date; + data: any; +} + +class MemoryCache { + private cache: ICache[] = []; + constructor() { + setInterval(this.cleanup.bind(this), 1000); + } + + public set(type: string, id: string, data: any, secondsExpiry: number) { + const expiry = new Date(); + expiry.setSeconds(expiry.getSeconds() + secondsExpiry); + this.cache.push({ + type: type, + id: id, + data: data, + expires: expiry, + }); + } + + public get(type: string, id: string): T | null { + const found = this.cache.find((cache) => cache.type === type && cache.id === id); + if (found) { + return found.data; + } + return null; + } + + private cleanup() { + this.cache = this.cache.filter((cache) => cache.expires < (new Date())); + } +} + +export default new MemoryCache(); diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 681c4f26e..d3dbfcb16 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../interfaces'; +import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; import { Common } from './common'; class MempoolBlocks { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 2a509aa3f..c00d5c10c 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,14 +1,20 @@ import config from '../config'; -import bitcoinApi from './bitcoin/electrs-api'; -import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import { TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; +import transactionUtils from './transaction-utils'; +import { IBitcoinApi } from './bitcoin/bitcoin-api.interface'; +import bitcoinBaseApi from './bitcoin/bitcoin-base.api'; +import loadingIndicators from './loading-indicators'; class Mempool { + private static WEBSOCKET_REFRESH_RATE_MS = 10000; private inSync: boolean = false; private mempoolCache: { [txId: string]: TransactionExtended } = {}; - private mempoolInfo: MempoolInfo = { size: 0, bytes: 0 }; - private mempoolChangedCallback: ((newMempool: { [txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], + private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, + maxmempool: 0, mempoolminfee: 0, minrelaytxfee: 0 }; + private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) | undefined; private txPerSecondArray: number[] = []; @@ -48,10 +54,10 @@ class Mempool { } public async $updateMemPoolInfo() { - this.mempoolInfo = await bitcoinApi.getMempoolInfo(); + this.mempoolInfo = await bitcoinBaseApi.$getMempoolInfo(); } - public getMempoolInfo(): MempoolInfo | undefined { + public getMempoolInfo(): IBitcoinApi.MempoolInfo | undefined { return this.mempoolInfo; } @@ -66,8 +72,9 @@ class Mempool { public getFirstSeenForTransactions(txIds: string[]): number[] { const txTimes: number[] = []; txIds.forEach((txId: string) => { - if (this.mempoolCache[txId]) { - txTimes.push(this.mempoolCache[txId].firstSeen); + const tx = this.mempoolCache[txId]; + if (tx && tx.firstSeen) { + txTimes.push(tx.firstSeen); } else { txTimes.push(0); } @@ -75,33 +82,23 @@ class Mempool { return txTimes; } - public async getTransactionExtended(txId: string): Promise { - try { - const transaction: Transaction = await bitcoinApi.getRawTransaction(txId); - return Object.assign({ - vsize: transaction.weight / 4, - feePerVsize: (transaction.fee || 0) / (transaction.weight / 4), - firstSeen: Math.round((new Date().getTime() / 1000)), - }, transaction); - } catch (e) { - logger.debug(txId + ' not found'); - return false; - } - } - public async $updateMempool() { logger.debug('Updating mempool'); const start = new Date().getTime(); let hasChange: boolean = false; const currentMempoolSize = Object.keys(this.mempoolCache).length; let txCount = 0; - const transactions = await bitcoinApi.getRawMempool(); + const transactions = await bitcoinApi.$getRawMempool(); const diff = transactions.length - currentMempoolSize; const newTransactions: TransactionExtended[] = []; + if (!this.inSync) { + loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100); + } + for (const txid of transactions) { if (!this.mempoolCache[txid]) { - const transaction = await this.getTransactionExtended(txid); + const transaction = await transactionUtils.$getTransactionExtended(txid, true); if (transaction) { this.mempoolCache[txid] = transaction; txCount++; @@ -124,13 +121,14 @@ class Mempool { } } - if ((new Date().getTime()) - start > config.MEMPOOL.WEBSOCKET_REFRESH_RATE_MS * 10) { + if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) { break; } } // Prevent mempool from clear on bitcoind restart by delaying the deletion if (this.mempoolProtection === 0 + && config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize > 20000 && transactions.length / currentMempoolSize <= 0.80 ) { @@ -170,6 +168,7 @@ class Mempool { if (!this.inSync && transactions.length === Object.keys(newMempool).length) { this.inSync = true; logger.info('The mempool is now in sync!'); + loadingIndicators.setProgress('mempool', 100); } if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics.ts index ebdd85a9b..619331b77 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics.ts @@ -2,7 +2,7 @@ import memPool from './mempool'; import { DB } from '../database'; import logger from '../logger'; -import { Statistic, TransactionExtended, OptimizedStatistic } from '../interfaces'; +import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces'; class Statistics { protected intervalTimer: NodeJS.Timer | undefined; @@ -25,15 +25,15 @@ class Statistics { setTimeout(() => { this.runStatistics(); this.intervalTimer = setInterval(() => { - if (!memPool.isInSync()) { - return; - } this.runStatistics(); }, 1 * 60 * 1000); }, difference); } private async runStatistics(): Promise { + if (!memPool.isInSync()) { + return; + } const currentMempool = memPool.getMempool(); const txPerSecond = memPool.getTxPerSecond(); const vBytesPerSecond = memPool.getVBytesPerSecond(); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts new file mode 100644 index 000000000..1b7fda068 --- /dev/null +++ b/backend/src/api/transaction-utils.ts @@ -0,0 +1,51 @@ +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import logger from '../logger'; +import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; + +class TransactionUtils { + constructor() { } + + public stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { + return { + vin: [{ + scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase'] + }], + vout: tx.vout + .map((vout) => ({ + scriptpubkey_address: vout.scriptpubkey_address, + value: vout.value + })) + .filter((vout) => vout.value) + }; + } + + public async $getTransactionExtended(txId: string, forceBitcoind = false, addPrevouts = false): Promise { + try { + let transaction: IEsploraApi.Transaction; + if (forceBitcoind) { + transaction = await bitcoinApi.$getRawTransactionBitcoind(txId, false, addPrevouts); + } else { + transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts); + } + return this.extendTransaction(transaction); + } catch (e) { + logger.debug('getTransactionExtended error: ' + (e.message || e)); + logger.debug(JSON.stringify(e)); + return null; + } + } + + private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { + const transactionExtended: TransactionExtended = Object.assign({ + vsize: Math.round(transaction.weight / 4), + feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), + }, transaction); + if (!transaction.status.confirmed) { + transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); + } + return transactionExtended; + } +} + +export default new TransactionUtils(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 76fb91583..35d11c37c 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,12 +1,16 @@ import logger from '../logger'; import * as WebSocket from 'ws'; -import { Block, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../interfaces'; +import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, + OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; import mempoolBlocks from './mempool-blocks'; import fiatConversion from './fiat-conversion'; import { Common } from './common'; +import loadingIndicators from './loading-indicators'; +import config from '../config'; +import transactionUtils from './transaction-utils'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -77,7 +81,7 @@ class WebsocketHandler { } if (parsedMessage.action === 'init') { - const _blocks = blocks.getBlocks(); + const _blocks = blocks.getBlocks().slice(-8); if (!_blocks) { return; } @@ -117,9 +121,35 @@ class WebsocketHandler { }); } - getInitData(_blocks?: Block[]) { + handleLoadingChanged(indicators: ILoadingIndicators) { + if (!this.wss) { + throw new Error('WebSocket.Server is not set'); + } + + this.wss.clients.forEach((client: WebSocket) => { + if (client.readyState !== WebSocket.OPEN) { + return; + } + client.send(JSON.stringify({ loadingIndicators: indicators })); + }); + } + + handleNewConversionRates(conversionRates: IConversionRates) { + if (!this.wss) { + throw new Error('WebSocket.Server is not set'); + } + + this.wss.clients.forEach((client: WebSocket) => { + if (client.readyState !== WebSocket.OPEN) { + return; + } + client.send(JSON.stringify({ conversions: conversionRates })); + }); + } + + getInitData(_blocks?: BlockExtended[]) { if (!_blocks) { - _blocks = blocks.getBlocks(); + _blocks = blocks.getBlocks().slice(-8); } return { 'mempoolInfo': memPool.getMempoolInfo(), @@ -131,6 +161,7 @@ class WebsocketHandler { 'transactions': memPool.getLatestTransactions(), 'git-commit': backendInfo.gitCommitHash, 'hostname': backendInfo.hostname, + 'loadingIndicators': loadingIndicators.getLoadingIndicators(), ...this.extraInitProperties }; } @@ -167,7 +198,7 @@ class WebsocketHandler { const vBytesPerSecond = memPool.getVBytesPerSecond(); const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions); - this.wss.clients.forEach((client: WebSocket) => { + this.wss.clients.forEach(async (client: WebSocket) => { if (client.readyState !== WebSocket.OPEN) { return; } @@ -187,7 +218,14 @@ class WebsocketHandler { if (client['track-mempool-tx']) { const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']); if (tx) { - response['tx'] = tx; + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, false, true); + if (fullTx) { + response['tx'] = fullTx; + } + } else { + response['tx'] = tx; + } client['track-mempool-tx'] = null; } } @@ -195,17 +233,31 @@ class WebsocketHandler { if (client['track-address']) { const foundTransactions: TransactionExtended[] = []; - newTransactions.forEach((tx) => { + for (const tx of newTransactions) { const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']); if (someVin) { - foundTransactions.push(tx); + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, false, true); + if (fullTx) { + foundTransactions.push(fullTx); + } + } else { + foundTransactions.push(tx); + } return; } const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']); if (someVout) { - foundTransactions.push(tx); + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, false, true); + if (fullTx) { + foundTransactions.push(fullTx); + } + } else { + foundTransactions.push(tx); + } } - }); + } if (foundTransactions.length) { response['address-transactions'] = foundTransactions; @@ -244,7 +296,15 @@ class WebsocketHandler { if (client['track-tx'] && rbfTransactions[client['track-tx']]) { for (const rbfTransaction in rbfTransactions) { if (client['track-tx'] === rbfTransaction) { - response['rbfTransaction'] = rbfTransactions[rbfTransaction]; + const rbfTx = rbfTransactions[rbfTransaction]; + if (config.MEMPOOL.BACKEND !== 'esplora') { + const fullTx = await transactionUtils.$getTransactionExtended(rbfTransaction, false, true); + if (fullTx) { + response['rbfTransaction'] = fullTx; + } + } else { + response['rbfTransaction'] = rbfTx; + } break; } } @@ -256,7 +316,7 @@ class WebsocketHandler { }); } - handleNewBlock(block: Block, txIds: string[], transactions: TransactionExtended[]) { + handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } diff --git a/backend/src/config.ts b/backend/src/config.ts index 7aa1d2a84..2b7367507 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -3,15 +3,27 @@ const configFile = require('../mempool-config.json'); interface IConfig { MEMPOOL: { NETWORK: 'mainnet' | 'testnet' | 'liquid'; + BACKEND: 'esplora' | 'electrum' | 'none'; HTTP_PORT: number; SPAWN_CLUSTER_PROCS: number; API_URL_PREFIX: string; - WEBSOCKET_REFRESH_RATE_MS: number; - }; - ELECTRS: { - REST_API_URL: string; POLL_RATE_MS: number; }; + ESPLORA: { + REST_API_URL: string; + }; + ELECTRUM: { + HOST: string; + PORT: number; + TLS_ENABLED: boolean; + TX_LOOKUPS: boolean; + }; + CORE_RPC: { + HOST: string; + PORT: number; + USERNAME: string; + PASSWORD: string; + }; DATABASE: { ENABLED: boolean; HOST: string, @@ -44,15 +56,27 @@ interface IConfig { const defaults: IConfig = { 'MEMPOOL': { 'NETWORK': 'mainnet', + 'BACKEND': 'none', 'HTTP_PORT': 8999, 'SPAWN_CLUSTER_PROCS': 0, 'API_URL_PREFIX': '/api/v1/', - 'WEBSOCKET_REFRESH_RATE_MS': 2000 - }, - 'ELECTRS': { - 'REST_API_URL': 'http://127.0.0.1:3000', 'POLL_RATE_MS': 2000 }, + 'ESPLORA': { + 'REST_API_URL': 'http://127.0.0.1:3000', + }, + 'ELECTRUM': { + 'HOST': '127.0.0.1', + 'PORT': 3306, + 'TLS_ENABLED': true, + 'TX_LOOKUPS': false + }, + 'CORE_RPC': { + 'HOST': '127.0.0.1', + 'PORT': 8332, + 'USERNAME': 'mempool', + 'PASSWORD': 'mempool' + }, 'DATABASE': { 'ENABLED': true, 'HOST': 'localhost', @@ -84,7 +108,9 @@ const defaults: IConfig = { class Config implements IConfig { MEMPOOL: IConfig['MEMPOOL']; - ELECTRS: IConfig['ELECTRS']; + ESPLORA: IConfig['ESPLORA']; + ELECTRUM: IConfig['ELECTRUM']; + CORE_RPC: IConfig['CORE_RPC']; DATABASE: IConfig['DATABASE']; STATISTICS: IConfig['STATISTICS']; BISQ_BLOCKS: IConfig['BISQ_BLOCKS']; @@ -94,7 +120,9 @@ class Config implements IConfig { constructor() { const configs = this.merge(configFile, defaults); this.MEMPOOL = configs.MEMPOOL; - this.ELECTRS = configs.ELECTRS; + this.ESPLORA = configs.ESPLORA; + this.ELECTRUM = configs.ELECTRUM; + this.CORE_RPC = configs.CORE_RPC; this.DATABASE = configs.DATABASE; this.STATISTICS = configs.STATISTICS; this.BISQ_BLOCKS = configs.BISQ_BLOCKS; diff --git a/backend/src/index.ts b/backend/src/index.ts index ab50afc36..d9e8722d1 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -20,12 +20,13 @@ import bisqMarkets from './api/bisq/markets'; import donations from './api/donations'; import logger from './logger'; import backendInfo from './api/backend-info'; +import loadingIndicators from './api/loading-indicators'; class Server { private wss: WebSocket.Server | undefined; private server: https.Server | http.Server | undefined; private app: Express; - private retryOnElectrsErrorAfterSeconds = 5; + private currentBackendRetryInterval = 5; constructor() { this.app = express(); @@ -110,18 +111,19 @@ class Server { await memPool.$updateMemPoolInfo(); await blocks.$updateBlocks(); await memPool.$updateMempool(); - setTimeout(this.runMainUpdateLoop.bind(this), config.ELECTRS.POLL_RATE_MS); - this.retryOnElectrsErrorAfterSeconds = 5; + setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); + this.currentBackendRetryInterval = 5; } catch (e) { - const loggerMsg = `runMainLoop error: ${(e.message || e)}. Retrying in ${this.retryOnElectrsErrorAfterSeconds} sec.`; - if (this.retryOnElectrsErrorAfterSeconds > 5) { + const loggerMsg = `runMainLoop error: ${(e.message || e)}. Retrying in ${this.currentBackendRetryInterval} sec.`; + if (this.currentBackendRetryInterval > 5) { logger.warn(loggerMsg); } else { logger.debug(loggerMsg); } - setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.retryOnElectrsErrorAfterSeconds); - this.retryOnElectrsErrorAfterSeconds *= 2; - this.retryOnElectrsErrorAfterSeconds = Math.min(this.retryOnElectrsErrorAfterSeconds, 60); + logger.debug(JSON.stringify(e)); + setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval); + this.currentBackendRetryInterval *= 2; + this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60); } } @@ -134,6 +136,8 @@ class Server { blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); donations.setNotfyDonationStatusCallback(websocketHandler.handleNewDonation.bind(websocketHandler)); + fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); + loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); } setUpHttpApiRoutes() { @@ -208,6 +212,22 @@ class Server { } }); } + + if (config.MEMPOOL.BACKEND !== 'esplora') { + this.app + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix) + ; + } } } diff --git a/backend/src/interfaces.ts b/backend/src/mempool.interfaces.ts similarity index 54% rename from backend/src/interfaces.ts rename to backend/src/mempool.interfaces.ts index 159ad868d..30227ba08 100644 --- a/backend/src/interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,11 +1,4 @@ -export interface MempoolInfo { - size: number; - bytes: number; - usage?: number; - maxmempool?: number; - mempoolminfee?: number; - minrelaytxfee?: number; -} +import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; export interface MempoolBlock { blockSize: number; @@ -20,23 +13,6 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { transactionIds: string[]; } -export interface Transaction { - txid: string; - version: number; - locktime: number; - fee: number; - size: number; - weight: number; - vin: Vin[]; - vout: Vout[]; - status: Status; -} - -export interface TransactionMinerInfo { - vin: VinStrippedToScriptsig[]; - vout: VoutStrippedToScriptPubkey[]; -} - interface VinStrippedToScriptsig { scriptsig: string; } @@ -46,10 +22,10 @@ interface VoutStrippedToScriptPubkey { value: number; } -export interface TransactionExtended extends Transaction { +export interface TransactionExtended extends IEsploraApi.Transaction { vsize: number; feePerVsize: number; - firstSeen: number; + firstSeen?: number; } export interface TransactionStripped { @@ -58,96 +34,17 @@ export interface TransactionStripped { weight: number; value: number; } - -export interface Vin { - txid: string; - vout: number; - is_coinbase: boolean; - scriptsig: string; - scriptsig_asm: string; - inner_redeemscript_asm?: string; - inner_witnessscript_asm?: string; - sequence: any; - witness?: string[]; - prevout: Vout; - // Elements - is_pegin?: boolean; - issuance?: Issuance; -} - -interface Issuance { - asset_id: string; - is_reissuance: string; - asset_blinding_nonce: string; - asset_entropy: string; - contract_hash: string; - assetamount?: number; - assetamountcommitment?: string; - tokenamount?: number; - tokenamountcommitment?: string; -} - -export interface Vout { - scriptpubkey: string; - scriptpubkey_asm: string; - scriptpubkey_type: string; - scriptpubkey_address: string; - value: number; - // Elements - valuecommitment?: number; - asset?: string; - pegout?: Pegout; -} - -interface Pegout { - genesis_hash: string; - scriptpubkey: string; - scriptpubkey_asm: string; - scriptpubkey_address: string; -} - -export interface Status { - confirmed: boolean; - block_height?: number; - block_hash?: string; - block_time?: number; -} - -export interface Block { - id: string; - height: number; - version: number; - timestamp: number; - bits: number; - nounce: number; - difficulty: number; - merkle_root: string; - tx_count: number; - size: number; - weight: number; - previousblockhash: string; - - // Custom properties +export interface BlockExtended extends IEsploraApi.Block { medianFee?: number; feeRange?: number[]; reward?: number; coinbaseTx?: TransactionMinerInfo; - matchRate: number; - stage: number; + matchRate?: number; } -export interface Address { - address: string; - chain_stats: ChainStats; - mempool_stats: MempoolStats; -} - -export interface ChainStats { - funded_txo_count: number; - funded_txo_sum: number; - spent_txo_count: number; - spent_txo_sum: number; - tx_count: number; +export interface TransactionMinerInfo { + vin: VinStrippedToScriptsig[]; + vout: VoutStrippedToScriptPubkey[]; } export interface MempoolStats { @@ -219,12 +116,6 @@ export interface OptimizedStatistic { vsizes: number[]; } -export interface Outspend { - spent: boolean; - txid: string; - vin: number; - status: Status; -} export interface WebsocketResponse { action: string; data: string[]; @@ -244,3 +135,6 @@ interface RequiredParams { required: boolean; types: ('@string' | '@number' | '@boolean' | string)[]; } + +export interface ILoadingIndicators { [name: string]: number; } +export interface IConversionRates { [currency: string]: number; } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 3d60f7df2..e35164ab0 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -8,10 +8,15 @@ import mempool from './api/mempool'; import bisq from './api/bisq/bisq'; import websocketHandler from './api/websocket-handler'; import bisqMarket from './api/bisq/markets-api'; -import { OptimizedStatistic, RequiredSpec } from './interfaces'; +import { OptimizedStatistic, RequiredSpec, TransactionExtended } from './mempool.interfaces'; import { MarketsApiError } from './api/bisq/interfaces'; +import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; import donations from './api/donations'; import logger from './logger'; +import bitcoinApi from './api/bitcoin/bitcoin-api-factory'; +import transactionUtils from './api/transaction-utils'; +import blocks from './api/blocks'; +import loadingIndicators from './api/loading-indicators'; class Routes { private cache: { [date: string]: OptimizedStatistic[] } = { @@ -524,6 +529,148 @@ class Routes { }; } + public async getTransaction(req: Request, res: Response) { + try { + const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, false, true); + + if (transaction) { + res.json(transaction); + } else { + res.status(500).send('Error fetching transaction.'); + } + } catch (e) { + res.status(500).send(e.message || e); + } + } + + public async getBlock(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getBlock(req.params.hash); + res.json(result); + } catch (e) { + res.status(500).send(e.message || e); + } + } + + public async getBlocks(req: Request, res: Response) { + try { + loadingIndicators.setProgress('blocks', 0); + + const returnBlocks: IEsploraApi.Block[] = []; + const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight(); + + // Check if block height exist in local cache to skip the hash lookup + const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight); + let startFromHash: string | null = null; + if (blockByHeight) { + startFromHash = blockByHeight.id; + } else { + startFromHash = await bitcoinApi.$getBlockHash(fromHeight); + } + + let nextHash = startFromHash; + for (let i = 0; i < 10; i++) { + const localBlock = blocks.getBlocks().find((b) => b.id === nextHash); + if (localBlock) { + returnBlocks.push(localBlock); + nextHash = localBlock.previousblockhash; + } else { + const block = await bitcoinApi.$getBlock(nextHash); + returnBlocks.push(block); + nextHash = block.previousblockhash; + } + loadingIndicators.setProgress('blocks', i / 10 * 100); + } + + res.json(returnBlocks); + } catch (e) { + loadingIndicators.setProgress('blocks', 100); + res.status(500).send(e.message || e); + } + } + + public async getBlockTransactions(req: Request, res: Response) { + try { + loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0); + + const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); + const transactions: TransactionExtended[] = []; + const startingIndex = Math.max(0, parseInt(req.params.index, 10)); + + const endIndex = Math.min(startingIndex + 10, txIds.length); + for (let i = startingIndex; i < endIndex; i++) { + const transaction = await transactionUtils.$getTransactionExtended(txIds[i], false, true); + if (transaction) { + transactions.push(transaction); + loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i + 1) / endIndex * 100); + } + } + res.json(transactions); + } catch (e) { + loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100); + res.status(500).send(e.message || e); + } + } + + public async getBlockHeight(req: Request, res: Response) { + try { + const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10)); + res.send(blockHash); + } catch (e) { + res.status(500).send(e.message || e); + } + } + + public async getAddress(req: Request, res: Response) { + if (config.MEMPOOL.BACKEND === 'none') { + res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); + return; + } + + try { + const addressData = await bitcoinApi.$getAddress(req.params.address); + res.json(addressData); + } catch (e) { + if (e.message && e.message.indexOf('exceeds') > 0) { + return res.status(413).send(e.message); + } + res.status(500).send(e.message || e); + } + } + + public async getAddressTransactions(req: Request, res: Response) { + if (config.MEMPOOL.BACKEND === 'none') { + res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); + return; + } + + try { + const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId); + res.json(transactions); + } catch (e) { + if (e.message && e.message.indexOf('exceeds') > 0) { + return res.status(413).send(e.message); + } + res.status(500).send(e.message || e); + } + } + + public async getAdressTxChain(req: Request, res: Response) { + res.status(501).send('Not implemented'); + } + + public async getAddressPrefix(req: Request, res: Response) { + try { + const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix); + res.send(blockHash); + } catch (e) { + res.status(500).send(e.message || e); + } + } + + public getTransactionOutspends(req: Request, res: Response) { + res.status(501).send('Not implemented'); + } } export default new Routes(); diff --git a/backend/tslint.json b/backend/tslint.json index 65ac58f4b..945512322 100644 --- a/backend/tslint.json +++ b/backend/tslint.json @@ -12,7 +12,7 @@ "severity": "warn" }, "eofline": true, - "forin": true, + "forin": false, "import-blacklist": [ true, "rxjs", diff --git a/frontend/proxy.conf.json b/frontend/proxy.conf.json index c02eb5e1e..5886ff104 100644 --- a/frontend/proxy.conf.json +++ b/frontend/proxy.conf.json @@ -9,10 +9,10 @@ "ws": true }, "/api/": { - "target": "http://localhost:50001/", + "target": "http://localhost:8999/", "secure": false, "pathRewrite": { - "^/api/": "" + "^/api/": "/api/v1/" } }, "/testnet/api/v1": { diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index 41728c083..6c06aec2d 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -67,6 +67,14 @@ + + +
+
+
+
+ + @@ -105,6 +113,14 @@ Error loading address data.
{{ error.error }} + +

+ Consider view this address on the official Mempool website instead: +
+ https://mempool.space/address/{{ addressString }} +
+ http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/address/{{ addressString }} + diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index f1d21d3b9..601d422d6 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -1,13 +1,13 @@ import { Component, OnInit, OnDestroy } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; -import { switchMap, filter, catchError } from 'rxjs/operators'; +import { switchMap, filter, catchError, map, tap } from 'rxjs/operators'; import { Address, Transaction } from '../../interfaces/electrs.interface'; import { WebsocketService } from 'src/app/services/websocket.service'; import { StateService } from 'src/app/services/state.service'; import { AudioService } from 'src/app/services/audio.service'; import { ApiService } from 'src/app/services/api.service'; -import { of, merge, Subscription } from 'rxjs'; +import { of, merge, Subscription, Observable } from 'rxjs'; import { SeoService } from 'src/app/services/seo.service'; @Component({ @@ -25,6 +25,7 @@ export class AddressComponent implements OnInit, OnDestroy { isLoadingTransactions = true; error: any; mainSubscription: Subscription; + addressLoadingStatus$: Observable; totalConfirmedTxCount = 0; loadedConfirmedTxCount = 0; @@ -48,7 +49,13 @@ export class AddressComponent implements OnInit, OnDestroy { ngOnInit() { this.stateService.networkChanged$.subscribe((network) => this.network = network); - this.websocketService.want(['blocks', 'mempool-blocks']); + this.websocketService.want(['blocks']); + + this.addressLoadingStatus$ = this.route.paramMap + .pipe( + switchMap(() => this.stateService.loadingIndicators$), + map((indicators) => indicators['address-' + this.addressString] !== undefined ? indicators['address-' + this.addressString] : 0) + ); this.mainSubscription = this.route.paramMap .pipe( diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 8b0f852c9..85daa10bd 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -112,6 +112,13 @@ + + +
+
+
+
+ diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 3c9b3bf08..99b66de77 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -2,9 +2,9 @@ import { Component, OnInit, OnDestroy } from '@angular/core'; import { Location } from '@angular/common'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; -import { switchMap, tap, debounceTime, catchError } from 'rxjs/operators'; +import { switchMap, tap, debounceTime, catchError, map } from 'rxjs/operators'; import { Block, Transaction, Vout } from '../../interfaces/electrs.interface'; -import { of, Subscription } from 'rxjs'; +import { Observable, of, Subscription } from 'rxjs'; import { StateService } from '../../services/state.service'; import { SeoService } from 'src/app/services/seo.service'; import { WebsocketService } from 'src/app/services/websocket.service'; @@ -31,6 +31,7 @@ export class BlockComponent implements OnInit, OnDestroy { coinbaseTx: Transaction; page = 1; itemsPerPage: number; + txsLoadingStatus$: Observable; constructor( private route: ActivatedRoute, @@ -48,6 +49,12 @@ export class BlockComponent implements OnInit, OnDestroy { this.network = this.stateService.network; this.itemsPerPage = this.stateService.env.ELECTRS_ITEMS_PER_PAGE; + this.txsLoadingStatus$ = this.route.paramMap + .pipe( + switchMap(() => this.stateService.loadingIndicators$), + map((indicators) => indicators['blocktxs-' + this.blockHash] !== undefined ? indicators['blocktxs-' + this.blockHash] : 0) + ); + this.subscription = this.route.paramMap .pipe( switchMap((params: ParamMap) => { diff --git a/frontend/src/app/components/latest-blocks/latest-blocks.component.html b/frontend/src/app/components/latest-blocks/latest-blocks.component.html index 5c8853158..ce9496657 100644 --- a/frontend/src/app/components/latest-blocks/latest-blocks.component.html +++ b/frontend/src/app/components/latest-blocks/latest-blocks.component.html @@ -33,8 +33,25 @@ - + + + +
+
+
+ + + + + +
+ Error loading blocks +
+ {{ error.error }} +
+ + diff --git a/frontend/src/app/components/latest-blocks/latest-blocks.component.ts b/frontend/src/app/components/latest-blocks/latest-blocks.component.ts index ccc898355..15b2843bd 100644 --- a/frontend/src/app/components/latest-blocks/latest-blocks.component.ts +++ b/frontend/src/app/components/latest-blocks/latest-blocks.component.ts @@ -5,6 +5,7 @@ import { Block } from '../../interfaces/electrs.interface'; import { Subscription, Observable, merge, of } from 'rxjs'; import { SeoService } from '../../services/seo.service'; import { WebsocketService } from 'src/app/services/websocket.service'; +import { map } from 'rxjs/operators'; @Component({ selector: 'app-latest-blocks', @@ -14,11 +15,12 @@ import { WebsocketService } from 'src/app/services/websocket.service'; }) export class LatestBlocksComponent implements OnInit, OnDestroy { network$: Observable; - + error: any; blocks: any[] = []; blockSubscription: Subscription; isLoading = true; interval: any; + blocksLoadingStatus$: Observable; latestBlockHeight: number; @@ -39,6 +41,11 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { this.network$ = merge(of(''), this.stateService.networkChanged$); + this.blocksLoadingStatus$ = this.stateService.loadingIndicators$ + .pipe( + map((indicators) => indicators['blocks'] !== undefined ? indicators['blocks'] : 0) + ); + this.blockSubscription = this.stateService.blocks$ .subscribe(([block]) => { if (block === null || !this.blocks.length) { @@ -79,6 +86,7 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { .subscribe((blocks) => { this.blocks = blocks; this.isLoading = false; + this.error = undefined; this.latestBlockHeight = blocks[0].height; @@ -88,6 +96,12 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { this.loadMore(chunks); } this.cd.markForCheck(); + }, + (error) => { + console.log(error); + this.error = error; + this.isLoading = false; + this.cd.markForCheck(); }); } @@ -100,12 +114,19 @@ export class LatestBlocksComponent implements OnInit, OnDestroy { .subscribe((blocks) => { this.blocks = this.blocks.concat(blocks); this.isLoading = false; + this.error = undefined; const chunksLeft = chunks - 1; if (chunksLeft > 0) { this.loadMore(chunksLeft); } this.cd.markForCheck(); + }, + (error) => { + console.log(error); + this.error = error; + this.isLoading = false; + this.cd.markForCheck(); }); } diff --git a/frontend/src/app/dashboard/dashboard.component.html b/frontend/src/app/dashboard/dashboard.component.html index dcc7df9dd..a78092376 100644 --- a/frontend/src/app/dashboard/dashboard.component.html +++ b/frontend/src/app/dashboard/dashboard.component.html @@ -179,8 +179,8 @@
Incoming transactions
- -  Backend is synchronizing + +  Backend is synchronizing ({{ mempoolLoadingStatus$ | async }}%)
diff --git a/frontend/src/app/dashboard/dashboard.component.ts b/frontend/src/app/dashboard/dashboard.component.ts index 6e419712e..6f573aac2 100644 --- a/frontend/src/app/dashboard/dashboard.component.ts +++ b/frontend/src/app/dashboard/dashboard.component.ts @@ -50,6 +50,7 @@ export class DashboardComponent implements OnInit { mempoolBlocksData$: Observable; mempoolInfoData$: Observable; difficultyEpoch$: Observable; + mempoolLoadingStatus$: Observable; vBytesPerSecondLimit = 1667; blocks$: Observable; transactions$: Observable; @@ -77,6 +78,9 @@ export class DashboardComponent implements OnInit { this.websocketService.want(['blocks', 'stats', 'mempool-blocks', 'live-2h-chart']); this.network$ = merge(of(''), this.stateService.networkChanged$); this.collapseLevel = this.storageService.getValue('dashboard-collapsed') || 'one'; + this.mempoolLoadingStatus$ = this.stateService.loadingIndicators$.pipe( + map((indicators) => indicators.mempool !== undefined ? indicators.mempool : 100) + ); this.languageForm = this.formBuilder.group({ language: [''] diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 893dc1f99..e6d21267b 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -1,3 +1,4 @@ +import { ILoadingIndicators } from '../services/state.service'; import { Block, Transaction } from './electrs.interface'; export interface WebsocketResponse { @@ -15,6 +16,7 @@ export interface WebsocketResponse { rbfTransaction?: Transaction; transactions?: TransactionStripped[]; donationConfirmed?: boolean; + loadingIndicators?: ILoadingIndicators; 'track-tx'?: string; 'track-address'?: string; 'track-asset'?: string; diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 29b501f1e..289a2caa0 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -13,6 +13,8 @@ interface MarkBlockState { txFeePerVSize?: number; } +export interface ILoadingIndicators { [name: string]: number; } + export interface Env { TESTNET_ENABLED: boolean; LIQUID_ENABLED: boolean; @@ -63,6 +65,7 @@ export class StateService { lastDifficultyAdjustment$ = new ReplaySubject(1); gitCommit$ = new ReplaySubject(1); donationConfirmed$ = new Subject(); + loadingIndicators$ = new ReplaySubject(1); live2Chart$ = new Subject(); diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 71bbe5263..66bb70b54 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -270,6 +270,10 @@ export class WebsocketService { this.stateService.live2Chart$.next(response['live-2h-chart']); } + if (response.loadingIndicators) { + this.stateService.loadingIndicators$.next(response.loadingIndicators); + } + if (response.mempoolInfo) { this.stateService.mempoolInfo$.next(response.mempoolInfo); } diff --git a/production/mempool-config.bisq.json b/production/mempool-config.bisq.json index bd00bd0bf..de1c32104 100644 --- a/production/mempool-config.bisq.json +++ b/production/mempool-config.bisq.json @@ -1,17 +1,20 @@ { "MEMPOOL": { "NETWORK": "bisq", + "BACKEND": "esplora", "HTTP_PORT": 8996, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 4, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3000", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3000" + }, "DATABASE": { "ENABLED": false, "HOST": "localhost", diff --git a/production/mempool-config.liquid.json b/production/mempool-config.liquid.json index af2ce910a..5902dee68 100644 --- a/production/mempool-config.liquid.json +++ b/production/mempool-config.liquid.json @@ -1,16 +1,20 @@ { "MEMPOOL": { "NETWORK": "liquid", + "BACKEND": "esplora", "HTTP_PORT": 8998, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", "WEBSOCKET_REFRESH_RATE_MS": 2000 }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3001", - "POLL_RATE_MS": 2000 + "CORE_RPC": { + "PORT": 7041, + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3001" }, "DATABASE": { "ENABLED": true, diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index aef3838fc..bb6f7d2e1 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -1,17 +1,20 @@ { "MEMPOOL": { "NETWORK": "mainnet", + "BACKEND": "esplora", "HTTP_PORT": 8999, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3000", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3000" + }, "DATABASE": { "ENABLED": true, "HOST": "localhost", diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index 563c0b9a2..8e0c013ed 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -1,17 +1,21 @@ { "MEMPOOL": { "NETWORK": "testnet", + "BACKEND": "esplora", "HTTP_PORT": 8997, "MINED_BLOCKS_CACHE": 144, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "WEBSOCKET_REFRESH_RATE_MS": 2000 - }, - "ELECTRS": { - "ENABLED": true, - "REST_API_URL": "http://[::1]:3002", "POLL_RATE_MS": 2000 }, + "CORE_RPC": { + "PORT": 18332, + "USERNAME": "foo", + "PASSWORD": "bar" + }, + "ESPLORA": { + "REST_API_URL": "http://[::1]:3002" + }, "DATABASE": { "ENABLED": true, "HOST": "localhost",