Refactor blocks.ts and index 10k block headers at launch

backend/src/api/blocks.ts

@@ -2,11 +2,15 @@ import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, TransactionExtended } from '../mempool.interfaces';
+import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
 import bitcoinClient from './bitcoin/bitcoin-client';
+import { DB } from '../database';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import poolsRepository from '../repositories/PoolsRepository';
+import blocksRepository from '../repositories/BlocksRepository';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -30,6 +34,137 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
+  /**
+   * Return the list of transaction for a block
+   * @param blockHash
+   * @param blockHeight
+   * @param onlyCoinbase - Set to true if you only need the coinbase transaction
+   * @returns Promise<TransactionExtended[]>
+   */
+  private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean) : Promise<TransactionExtended[]> {
+    const transactions: TransactionExtended[] = [];
+    const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+
+    const mempool = memPool.getMempool();
+    let transactionsFound = 0;
+    let transactionsFetched = 0;
+
+    for (let i = 0; i < txIds.length; i++) {
+      if (mempool[txIds[i]]) {
+        // We update blocks before the mempool (index.ts), therefore we can
+        // optimize here by directly fetching txs in the "outdated" mempool
+        transactions.push(mempool[txIds[i]]);
+        transactionsFound++;
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
+        // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
+        if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
+          logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
+        }
+        try {
+          const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
+          transactions.push(tx);
+          transactionsFetched++;
+        } catch (e) {
+          logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
+          if (i === 0) {
+            throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
+          }
+        }
+      }
+
+      if (onlyCoinbase === true) {
+        break; // Fetch the first transaction and exit
+      }
+    }
+
+    transactions.forEach((tx) => {
+      if (!tx.cpfpChecked) {
+        Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent
+      }
+    });
+
+    logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+
+    return transactions;
+  }
+
+  /**
+   * Return a block with additional data (reward, coinbase, fees...)
+   * @param block
+   * @param transactions
+   * @returns BlockExtended
+   */
+  private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]) : BlockExtended {
+    const blockExtended: BlockExtended = Object.assign({}, block);
+    blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+
+    const transactionsTmp = [...transactions];
+    transactionsTmp.shift();
+    transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+    blockExtended.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
+    blockExtended.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
+
+    return blockExtended;
+  }
+
+  /**
+   * Try to find which miner found the block
+   * @param txMinerInfo
+   * @returns
+   */
+  private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined) : Promise<PoolTag> {
+    if (txMinerInfo === undefined) {
+      return poolsRepository.getUnknownPool();
+    }
+
+    const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
+    const address = txMinerInfo.vout[0].scriptpubkey_address;
+
+    const pools: PoolTag[] = await poolsRepository.$getPools();
+    for (let i = 0; i < pools.length; ++i) {
+      if (address !== undefined) {
+        let addresses: string[] = JSON.parse(pools[i].addresses);
+        if (addresses.indexOf(address) !== -1) {
+          return pools[i];
+        }
+      }
+
+      let regexes: string[] = JSON.parse(pools[i].regexes);
+      for (let y = 0; y < regexes.length; ++y) {
+        let match = asciiScriptSig.match(regexes[y]);
+        if (match !== null) {
+          return pools[i];
+        }
+      }
+    }
+
+    return poolsRepository.getUnknownPool();
+  }
+
+  /**
+   * Index all blocks metadata for the mining dashboard
+   */
+  public async $generateBlockDatabase() {
+    let currentBlockHeight = await bitcoinApi.$getBlockHeightTip();
+    let maxBlocks = 100; // tmp
+
+    while (currentBlockHeight-- > 0 && maxBlocks-- > 0) {
+      if (await blocksRepository.$isBlockAlreadyIndexed(currentBlockHeight)) {
+        // logger.debug(`Block #${currentBlockHeight} already indexed, skipping`);
+        continue;
+      }
+      logger.debug(`Indexing block #${currentBlockHeight}`);
+      const blockHash = await bitcoinApi.$getBlockHash(currentBlockHeight);
+      const block = await bitcoinApi.$getBlock(blockHash);
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+      const blockExtended = this.getBlockExtended(block, transactions);
+      const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
+      const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
+      await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+    }
+  }
+
   public async $updateBlocks() {
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
 
@@ -70,48 +205,14 @@ class Blocks {
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
       }
 
-      const transactions: TransactionExtended[] = [];
-
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
       const block = await bitcoinApi.$getBlock(blockHash);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
-
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
-      const mempool = memPool.getMempool();
+      const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
-      let transactionsFound = 0;
+      const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
-
+      const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
-      for (let i = 0; i < txIds.length; i++) {
+      await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
-        if (mempool[txIds[i]]) {
-          transactions.push(mempool[txIds[i]]);
-          transactionsFound++;
-        } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
-          logger.debug(`Fetching block tx ${i} of ${txIds.length}`);
-          try {
-            const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
-            transactions.push(tx);
-          } catch (e) {
-            logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
-            if (i === 0) {
-              throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
-            }
-          }
-        }
-      }
-
-      transactions.forEach((tx) => {
-        if (!tx.cpfpChecked) {
-          Common.setRelativesAndGetCpfpInfo(tx, mempool);
-        }
-      });
-
-      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`);
-
-      const blockExtended: BlockExtended = Object.assign({}, block);
-      blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-      blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-      transactions.shift();
-      transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
-      blockExtended.medianFee = transactions.length > 0 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0;
-      blockExtended.feeRange = transactions.length > 0 ? Common.getFeesInRange(transactions, 8) : [0, 0];
 
       if (block.height % 2016 === 0) {
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
@@ -130,6 +231,8 @@ class Blocks {
       if (memPool.isInSync()) {
         diskCache.$saveCacheToDisk();
       }
+
+      return;
     }
   }
 

backend/src/api/mempool.ts

@@ -118,11 +118,11 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
+          // if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
+          //   logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
+          // } else {
-            logger.debug('Fetched transaction ' + txCount);
+          //   logger.debug('Fetched transaction ' + txCount);
-          }
+          // }
           newTransactions.push(transaction);
         } catch (e) {
           logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));

backend/src/api/transaction-utils.ts

@@ -44,6 +44,14 @@ class TransactionUtils {
     }
     return transactionExtended;
   }
+
+  public hex2ascii(hex: string) {
+    let str = '';
+    for (let i = 0; i < hex.length; i += 2) {
+      str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
+    }
+    return str;
+  }
 }
 
 export default new TransactionUtils();

backend/src/index.ts

@@ -25,7 +25,6 @@ import databaseMigration from './api/database-migration';
 import poolsParser from './api/pools-parser';
 import syncAssets from './sync-assets';
 import icons from './api/liquid/icons';
-import poolsParser from './api/pools-parser';
 import { Common } from './api/common';
 
 class Server {
@@ -33,6 +32,7 @@ class Server {
   private server: http.Server | undefined;
   private app: Express;
   private currentBackendRetryInterval = 5;
+  private blockIndexingStarted = false;
 
   constructor() {
     this.app = express();
@@ -90,7 +90,6 @@ class Server {
       await checkDbConnection();
       try {
         await databaseMigration.$initializeOrMigrateDatabase();
-        await poolsParser.migratePoolsJson();
       } catch (e) {
         throw new Error(e instanceof Error ? e.message : 'Error');
       }
@@ -139,6 +138,13 @@ class Server {
       }
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
+
+      if (this.blockIndexingStarted === false/* && memPool.isInSync()*/) {
+        blocks.$generateBlockDatabase();
+        this.blockIndexingStarted = true;
+        logger.info("START OLDER BLOCK INDEXING");
+      }
+
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
     } catch (e) {

backend/src/mempool.interfaces.ts

@@ -1,5 +1,13 @@
+import { RowDataPacket } from 'mysql2';
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
 
+export interface PoolTag extends RowDataPacket {
+  name: string,
+  link: string,
+  regexes: string,
+  addresses: string,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;

backend/src/repositories/BlocksRepository.ts

@@ -0,0 +1,73 @@
+import { IEsploraApi } from "../api/bitcoin/esplora-api.interface";
+import { BlockExtended, PoolTag } from "../mempool.interfaces";
+import { DB } from "../database";
+import logger from "../logger";
+import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
+
+class BlocksRepository {
+  /**
+   * Save indexed block data in the database
+   * @param block 
+   * @param blockHash 
+   * @param coinbaseTxid 
+   * @param poolTag 
+   */
+  public async $saveBlockInDatabase(
+    block: BlockExtended,
+    blockHash: string,
+    coinbaseHex: string | undefined,
+    poolTag: PoolTag
+  ) {
+    const connection = await DB.pool.getConnection();
+
+    try {
+      const query = `INSERT INTO blocks(
+        height,  hash,     timestamp,    size,
+        weight,  tx_count, coinbase_raw, difficulty,
+        pool_id, fees,     fee_span,     median_fee
+      ) VALUE (
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?
+      )`;
+
+      const params: any[] = [
+        block.height, blockHash, block.timestamp, block.size,
+        block.weight, block.tx_count, coinbaseHex ? coinbaseHex : "", block.difficulty,
+        poolTag.id, 0, "[]", block.medianFee,
+      ];
+
+      await connection.query(query, params);
+    } catch (e) {
+      console.log(e);
+      logger.err('$updateBlocksDatabase() error' + (e instanceof Error ? e.message : e));
+    }
+
+    connection.release();
+  }
+
+  /**
+   * Check if a block has already been indexed in the database. Query the databse directly.
+   * This can be cached/optimized if required later on to avoid too many db queries.
+   * @param blockHeight
+   * @returns
+   */
+  public async $isBlockAlreadyIndexed(blockHeight: number) {
+    const connection = await DB.pool.getConnection();
+    let exists = false;
+
+    try {
+      const query = `SELECT height from blocks where blocks.height = ${blockHeight}`;
+      const [rows]: any[] = await connection.query(query);
+      exists = rows.length === 1;
+    } catch (e) {
+      console.log(e);
+      logger.err('$isBlockAlreadyIndexed() error' + (e instanceof Error ? e.message : e));
+    }
+    connection.release();
+
+    return exists;
+  }
+}
+
+export default new BlocksRepository();

backend/src/repositories/PoolsRepository.ts

@@ -0,0 +1,30 @@
+import { FieldPacket } from "mysql2";
+import { DB } from "../database";
+import { PoolTag } from "../mempool.interfaces"
+
+class PoolsRepository {
+  /**
+   * Get all pools tagging info
+   */
+  public async $getPools() : Promise<PoolTag[]> {
+    const connection = await DB.pool.getConnection();
+    const [rows]: [PoolTag[], FieldPacket[]] = await connection.query("SELECT * FROM pools;");        
+    connection.release();
+    return rows;
+  }
+
+  /**
+   * Get unknown pool tagging info
+   */
+  public getUnknownPool(): PoolTag {
+    return <PoolTag>{
+      id: null,
+      name: 'Unknown',
+      link: 'rickroll?',
+      regexes: "[]",
+      addresses: "[]",
+    };
+  } 
+}
+
+export default new PoolsRepository();