Basic bitcoind/romanz-electrum support to sync the mempool and blocks.

backend/mempool-config.sample.json

@@ -1,6 +1,7 @@
 {
   "MEMPOOL": {
     "NETWORK": "mainnet",
+    "BACKEND": "electrs",
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
@@ -8,7 +9,15 @@
   },
   "ELECTRS": {
     "REST_API_URL": "http://127.0.0.1:3000",
-    "POLL_RATE_MS": 2000
+    "POLL_RATE_MS": 2000,
+    "HOST": "127.0.0.1",
+    "PORT": 50002
+  },
+  "BITCOIND": {
+    "HOST": "127.0.0.1",
+    "PORT": 3306,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
   },
   "DATABASE": {
     "ENABLED": true,

backend/package.json

@@ -27,6 +27,8 @@
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "dependencies": {
+    "@codewarriorr/electrum-client-js": "^0.1.1",
+    "@mempool/bitcoin": "^3.0.2",
     "axios": "^0.21.0",
     "express": "^4.17.1",
     "locutus": "^2.0.12",

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -0,0 +1,16 @@
+import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry } from '../../interfaces';
+
+export interface AbstractBitcoinApi {
+  getMempoolInfo(): Promise<MempoolInfo>;
+  getRawMempool(): Promise<Transaction['txid'][]>;
+  getRawTransaction(txId: string): Promise<Transaction>;
+  getBlockHeightTip(): Promise<number>;
+  getTxIdsForBlock(hash: string): Promise<string[]>;
+  getBlockHash(height: number): Promise<string>;
+  getBlock(hash: string): Promise<Block>;
+  getMempoolEntry(txid: string): Promise<MempoolEntry>;
+
+  // Custom
+  getRawMempoolVerbose(): Promise<MempoolEntries>;
+  getRawTransactionBitcond(txId: string): Promise<Transaction>;
+}

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -0,0 +1,19 @@
+import config from '../../config';
+import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import BitcoindElectrsApi from './bitcoind-electrs-api';
+import BitcoindApi from './bitcoind-api';
+import ElectrsApi from './electrs-api';
+
+function bitcoinApiFactory(): AbstractBitcoinApi {
+  switch (config.MEMPOOL.BACKEND) {
+    case 'electrs':
+      return new ElectrsApi();
+    case 'bitcoind-electrs':
+      return new BitcoindElectrsApi();
+    case 'bitcoind':
+    default:
+      return new BitcoindApi();
+  }
+}
+
+export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoind-api.ts

@@ -0,0 +1,83 @@
+import config from '../../config';
+import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces';
+import * as bitcoin from '@mempool/bitcoin';
+
+class BitcoindApi {
+  bitcoindClient: any;
+
+  constructor() {
+    this.bitcoindClient = new bitcoin.Client({
+      host: config.BITCOIND.HOST,
+      port: config.BITCOIND.PORT,
+      user: config.BITCOIND.USERNAME,
+      pass: config.BITCOIND.PASSWORD,
+      timeout: 60000,
+    });
+  }
+
+  getMempoolInfo(): Promise<MempoolInfo> {
+    return this.bitcoindClient.getMempoolInfo();
+  }
+
+  getRawMempool(): Promise<Transaction['txid'][]> {
+    return this.bitcoindClient.getRawMemPool();
+  }
+
+  getRawMempoolVerbose(): Promise<MempoolEntries> {
+    return this.bitcoindClient.getRawMemPool(true);
+  }
+
+  getMempoolEntry(txid: string): Promise<MempoolEntry> {
+    return this.bitcoindClient.getMempoolEntry(txid,);
+  }
+
+  getRawTransaction(txId: string): Promise<Transaction> {
+    return this.bitcoindClient.getRawTransaction(txId, true)
+      .then((transaction: Transaction) => {
+        transaction.vout.forEach((vout) => vout.value = vout.value * 100000000);
+        return transaction;
+      });
+  }
+
+  getBlockHeightTip(): Promise<number> {
+    return this.bitcoindClient.getChainTips()
+      .then((result) => result[0].height);
+  }
+
+  getTxIdsForBlock(hash: string): Promise<string[]> {
+    return this.bitcoindClient.getBlock(hash, 1)
+      .then((rpcBlock: RpcBlock) => {
+        return rpcBlock.tx;
+      });
+  }
+
+  getBlockHash(height: number): Promise<string> {
+    return this.bitcoindClient.getBlockHash(height)
+  }
+
+  getBlock(hash: string): Promise<Block> {
+    return this.bitcoindClient.getBlock(hash)
+      .then((rpcBlock: RpcBlock) => {
+        return {
+          id: rpcBlock.hash,
+          height: rpcBlock.height,
+          version: rpcBlock.version,
+          timestamp: rpcBlock.time,
+          bits: rpcBlock.bits,
+          nonce: rpcBlock.nonce,
+          difficulty: rpcBlock.difficulty,
+          merkle_root: rpcBlock.merkleroot,
+          tx_count: rpcBlock.nTx,
+          size: rpcBlock.size,
+          weight: rpcBlock.weight,
+          previousblockhash: rpcBlock.previousblockhash,
+        };
+      });
+  }
+
+  getRawTransactionBitcond(txId: string): Promise<Transaction> {
+    throw new Error('Method not implemented.');
+  }
+}
+
+export default BitcoindApi;

backend/src/api/bitcoin/bitcoind-electrs-api.ts

@@ -0,0 +1,108 @@
+import config from '../../config';
+import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces';
+import * as bitcoin from '@mempool/bitcoin';
+import * as ElectrumClient from '@codewarriorr/electrum-client-js';
+import logger from '../../logger';
+
+class BitcoindElectrsApi implements AbstractBitcoinApi {
+  bitcoindClient: any;
+  electrumClient: any;
+
+  constructor() {
+    this.bitcoindClient = new bitcoin.Client({
+      host: config.BITCOIND.HOST,
+      port: config.BITCOIND.PORT,
+      user: config.BITCOIND.USERNAME,
+      pass: config.BITCOIND.PASSWORD,
+      timeout: 60000,
+    });
+
+    this.electrumClient = new ElectrumClient(
+      config.ELECTRS.HOST,
+      config.ELECTRS.PORT,
+      'ssl'
+    );
+
+    this.electrumClient.connect(
+      'electrum-client-js',
+      '1.4'
+    )
+  }
+
+  getMempoolInfo(): Promise<MempoolInfo> {
+    return this.bitcoindClient.getMempoolInfo();
+  }
+
+  getRawMempool(): Promise<Transaction['txid'][]> {
+    return this.bitcoindClient.getRawMemPool();
+  }
+
+  getRawMempoolVerbose(): Promise<MempoolEntries> {
+    return this.bitcoindClient.getRawMemPool(true);
+  }
+
+  getMempoolEntry(txid: string): Promise<MempoolEntry> {
+    return this.bitcoindClient.getMempoolEntry(txid,);
+  }
+
+  async getRawTransaction(txId: string): Promise<Transaction> {
+    try {
+      const transaction: Transaction = await this.electrumClient.blockchain_transaction_get(txId, true);
+      if (!transaction) {
+        throw new Error('not found');
+      }
+      transaction.vout.forEach((vout) => vout.value = vout.value * 100000000);
+      return transaction;
+    } catch (e) {
+      logger.debug('getRawTransaction error: ' + (e.message || e)); 
+      throw new Error(e);
+    }
+  }
+
+  getRawTransactionBitcond(txId: string): Promise<Transaction> {
+    return this.bitcoindClient.getRawTransaction(txId, true)
+      .then((transaction: Transaction) => {
+        transaction.vout.forEach((vout) => vout.value = vout.value * 100000000);
+        return transaction;
+      });
+  }
+
+  getBlockHeightTip(): Promise<number> {
+    return this.bitcoindClient.getChainTips()
+      .then((result) => result[0].height);
+  }
+
+  getTxIdsForBlock(hash: string): Promise<string[]> {
+    return this.bitcoindClient.getBlock(hash, 1)
+      .then((rpcBlock: RpcBlock) => {
+        return rpcBlock.tx;
+      });
+  }
+
+  getBlockHash(height: number): Promise<string> {
+    return this.bitcoindClient.getBlockHash(height)
+  }
+
+  getBlock(hash: string): Promise<Block> {
+    return this.bitcoindClient.getBlock(hash)
+      .then((rpcBlock: RpcBlock) => {
+        return {
+          id: rpcBlock.hash,
+          height: rpcBlock.height,
+          version: rpcBlock.version,
+          timestamp: rpcBlock.time,
+          bits: rpcBlock.bits,
+          nonce: rpcBlock.nonce,
+          difficulty: rpcBlock.difficulty,
+          merkle_root: rpcBlock.merkleroot,
+          tx_count: rpcBlock.nTx,
+          size: rpcBlock.size,
+          weight: rpcBlock.weight,
+          previousblockhash: rpcBlock.previousblockhash,
+        };
+      });
+  }
+}
+
+export default BitcoindElectrsApi;

backend/src/api/bitcoin/electrs-api.ts

@@ -1,8 +1,9 @@
 import config from '../../config';
-import { Transaction, Block, MempoolInfo } from '../../interfaces';
+import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries } from '../../interfaces';
 import axios from 'axios';
 
-class ElectrsApi {
+class ElectrsApi implements AbstractBitcoinApi {
 
   constructor() {
   }
@@ -42,15 +43,22 @@ class ElectrsApi {
       .then((response) => response.data);
   }
 
-  getBlocksFromHeight(height: number): Promise<string> {
-    return axios.get<string>(config.ELECTRS.REST_API_URL + '/blocks/' + height)
-      .then((response) => response.data);
-  }
-
   getBlock(hash: string): Promise<Block> {
     return axios.get<Block>(config.ELECTRS.REST_API_URL + '/block/' + hash)
       .then((response) => response.data);
   }
+
+  getRawMempoolVerbose(): Promise<MempoolEntries> {
+    throw new Error('Method not implemented.');
+  }
+
+  getMempoolEntry(): Promise<MempoolEntry> {
+    throw new Error('Method not implemented.');
+  }
+
+  getRawTransactionBitcond(txId: string): Promise<Transaction> {
+    throw new Error('Method not implemented.');
+  }
 }
 
-export default new ElectrsApi();
+export default ElectrsApi;

backend/src/api/blocks.ts

@@ -1,7 +1,8 @@
-import bitcoinApi from './bitcoin/electrs-api';
+import config from '../config';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces';
+import { Block, Transaction, TransactionExtended, TransactionMinerInfo } from '../interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 
@@ -55,31 +56,38 @@ class Blocks {
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
       }
 
+      let transactions: TransactionExtended[] = [];
+
       const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight);
       const block = await bitcoinApi.getBlock(blockHash);
-      const txIds = await bitcoinApi.getTxIdsForBlock(blockHash);
+      let txIds: string[] = await bitcoinApi.getTxIdsForBlock(blockHash);
 
       const mempool = memPool.getMempool();
       let found = 0;
-      let notFound = 0;
-
-      const transactions: TransactionExtended[] = [];
 
       for (let i = 0; i < txIds.length; i++) {
         if (mempool[txIds[i]]) {
           transactions.push(mempool[txIds[i]]);
           found++;
         } else {
+          if (config.MEMPOOL.BACKEND === 'electrs') {
             logger.debug(`Fetching block tx ${i} of ${txIds.length}`);
             const tx = await memPool.getTransactionExtended(txIds[i]);
             if (tx) {
               transactions.push(tx);
             }
-          notFound++;
+          } else { // When using bitcoind, just skip parsing past block tx's for now
+            if (i === 0) {
+              const tx = await memPool.getTransactionExtended(txIds[i], true);
+              if (tx) {
+                transactions.push(tx);
+              }
+            }
+          }
         }
       }
 
-      logger.debug(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`);
+      logger.debug(`${found} of ${txIds.length} found in mempool. ${txIds.length - found} not found.`);
 
       block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
       block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]);
@@ -110,10 +118,13 @@ class Blocks {
   private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo {
     return {
       vin: [{
-        scriptsig: tx.vin[0].scriptsig
+        scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase']
       }],
       vout: tx.vout
-        .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, value: vout.value }))
+        .map((vout) => ({
+          scriptpubkey_address: vout.scriptpubkey_address || (vout['scriptPubKey']['addresses'] && vout['scriptPubKey']['addresses'][0]) || null,
+          value: vout.value
+        }))
         .filter((vout) => vout.value)
     };
   }

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { Transaction, TransactionExtended, TransactionStripped } from '../interfaces';
+import { TransactionExtended, TransactionStripped } from '../interfaces';
 
 export class Common {
   static median(numbers: number[]) {
@@ -53,7 +53,7 @@ export class Common {
       txid: tx.txid,
       fee: tx.fee,
       weight: tx.weight,
-      value: tx.vin.reduce((acc, vin) => acc + (vin.prevout ? vin.prevout.value : 0), 0),
+      value: tx.vout ? tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) : 0,
     };
   }
 }

backend/src/api/mempool.ts

@@ -1,6 +1,6 @@
 import config from '../config';
-import bitcoinApi from './bitcoin/electrs-api';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces';
+import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond, MempoolEntry, MempoolEntries } from '../interfaces';
 import logger from '../logger';
 import { Common } from './common';
 
@@ -18,6 +18,7 @@ class Mempool {
   private vBytesPerSecond: number = 0;
   private mempoolProtection = 0;
   private latestTransactions: any[] = [];
+  private mempoolEntriesCache: MempoolEntries | null = null;
 
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
@@ -75,20 +76,47 @@ class Mempool {
     return txTimes;
   }
 
-  public async getTransactionExtended(txId: string): Promise<TransactionExtended | false> {
+  public async getTransactionExtended(txId: string, isCoinbase = false): Promise<TransactionExtended | false> {
     try {
-      const transaction: Transaction = await bitcoinApi.getRawTransaction(txId);
+      let transaction: Transaction;
-      return Object.assign({
+      if (!isCoinbase && config.MEMPOOL.BACKEND === 'bitcoind-electrs') {
-        vsize: transaction.weight / 4,
+        transaction = await bitcoinApi.getRawTransactionBitcond(txId);
-        feePerVsize: (transaction.fee || 0) / (transaction.weight / 4),
+      } else {
-        firstSeen: Math.round((new Date().getTime() / 1000)),
+        transaction = await bitcoinApi.getRawTransaction(txId);
-      }, transaction);
+      }
+      if (config.MEMPOOL.BACKEND !== 'electrs' && !isCoinbase) {
+        transaction = await this.$appendFeeData(transaction);
+      }
+      return this.extendTransaction(transaction);
     } catch (e) {
       logger.debug(txId + ' not found');
       return false;
     }
   }
 
+  private async $appendFeeData(transaction: Transaction): Promise<Transaction> {
+    let mempoolEntry: MempoolEntry;
+    if (!this.inSync && !this.mempoolEntriesCache) {
+      this.mempoolEntriesCache = await bitcoinApi.getRawMempoolVerbose();
+    }
+    if (this.mempoolEntriesCache && this.mempoolEntriesCache[transaction.txid]) {
+      mempoolEntry = this.mempoolEntriesCache[transaction.txid];
+    } else {
+      mempoolEntry = await bitcoinApi.getMempoolEntry(transaction.txid);
+    }
+    transaction.fee = mempoolEntry.fees.base * 100000000;
+    return transaction;
+  }
+
+  private extendTransaction(transaction: Transaction | MempoolEntry): TransactionExtended {
+    // @ts-ignore
+    return Object.assign({
+      vsize: Math.round(transaction.weight / 4),
+      feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)),
+      firstSeen: Math.round((new Date().getTime() / 1000)),
+    }, transaction);
+  }
+
   public async $updateMempool() {
     logger.debug('Updating mempool');
     const start = new Date().getTime();
@@ -169,6 +197,7 @@ class Mempool {
 
     if (!this.inSync && transactions.length === Object.keys(newMempool).length) {
       this.inSync = true;
+      this.mempoolEntriesCache = null;
       logger.info('The mempool is now in sync!');
     }
 

backend/src/config.ts

@@ -3,6 +3,7 @@ const configFile = require('../mempool-config.json');
 interface IConfig {
   MEMPOOL: {
     NETWORK: 'mainnet' | 'testnet' | 'liquid';
+    BACKEND: 'electrs' | 'bitcoind' | 'bitcoind-electrs';
     HTTP_PORT: number;
     SPAWN_CLUSTER_PROCS: number;
     API_URL_PREFIX: string;
@@ -11,7 +12,15 @@ interface IConfig {
   ELECTRS: {
     REST_API_URL: string;
     POLL_RATE_MS: number;
+    HOST: string;
+    PORT: number;
   };
+  BITCOIND: {
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  },
   DATABASE: {
     ENABLED: boolean;
     HOST: string,
@@ -44,6 +53,7 @@ interface IConfig {
 const defaults: IConfig = {
   'MEMPOOL': {
     'NETWORK': 'mainnet',
+    'BACKEND': 'electrs',
     'HTTP_PORT': 8999,
     'SPAWN_CLUSTER_PROCS': 0,
     'API_URL_PREFIX': '/api/v1/',
@@ -51,7 +61,15 @@ const defaults: IConfig = {
   },
   'ELECTRS': {
     'REST_API_URL': 'http://127.0.0.1:3000',
-    'POLL_RATE_MS': 2000
+    'POLL_RATE_MS': 2000,
+    'HOST': '127.0.0.1',
+    'PORT': 3306
+  },
+  'BITCOIND': {
+    'HOST': "127.0.0.1",
+    'PORT': 8332,
+    'USERNAME': "mempoo",
+    'PASSWORD': "mempool"
   },
   'DATABASE': {
     'ENABLED': true,
@@ -85,6 +103,7 @@ const defaults: IConfig = {
 class Config implements IConfig {
   MEMPOOL: IConfig['MEMPOOL'];
   ELECTRS: IConfig['ELECTRS'];
+  BITCOIND: IConfig['BITCOIND'];
   DATABASE: IConfig['DATABASE'];
   STATISTICS: IConfig['STATISTICS'];
   BISQ_BLOCKS: IConfig['BISQ_BLOCKS'];
@@ -95,6 +114,7 @@ class Config implements IConfig {
     const configs = this.merge(configFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ELECTRS = configs.ELECTRS;
+    this.BITCOIND = configs.BITCOIND;
     this.DATABASE = configs.DATABASE;
     this.STATISTICS = configs.STATISTICS;
     this.BISQ_BLOCKS = configs.BISQ_BLOCKS;

backend/src/interfaces.ts

@@ -119,7 +119,7 @@ export interface Block {
   version: number;
   timestamp: number;
   bits: number;
-  nounce: number;
+  nonce: number;
   difficulty: number;
   merkle_root: string;
   tx_count: number;
@@ -132,8 +132,58 @@ export interface Block {
   feeRange?: number[];
   reward?: number;
   coinbaseTx?: TransactionMinerInfo;
-  matchRate: number;
+  matchRate?: number;
-  stage: number;
+}
+
+export interface RpcBlock {
+  hash: string;
+  confirmations: number;
+  size: number;
+  strippedsize: number;
+  weight: number;
+  height: number;
+  version: number,
+  versionHex: string;
+  merkleroot: string;
+  tx: Transaction[];
+  time: number;
+  mediantime: number;
+  nonce: number;
+  bits: number;
+  difficulty: number;
+  chainwork: string;
+  nTx: number,
+  previousblockhash: string;
+  nextblockhash: string;
+}
+
+export interface MempoolEntries { [txId: string]: MempoolEntry };
+
+export interface MempoolEntry {
+  fees: Fees
+  vsize: number
+  weight: number
+  fee: number
+  modifiedfee: number
+  time: number
+  height: number
+  descendantcount: number
+  descendantsize: number
+  descendantfees: number
+  ancestorcount: number
+  ancestorsize: number
+  ancestorfees: number
+  wtxid: string
+  depends: any[]
+  spentby: any[]
+  'bip125-replaceable': boolean
+}
+
+export interface Fees {
+  base: number
+  modified: number
+  ancestor: number
+  descendant: number
 }
 
 export interface Address {