From c44896f53e46a7725ced8e9239d7cce9580bd337 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Thu, 16 Feb 2023 15:36:16 +0900
Subject: [PATCH 01/23] Get blocks data set by bulk (non indexed)

---
 backend/src/api/bitcoin/bitcoin.routes.ts | 28 +++++++++++
 backend/src/api/blocks.ts                 | 58 ++++++++++++++++++++++-
 backend/src/rpc-api/commands.ts           |  3 +-
 3 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts
index ea8154206..0ff30376c 100644
--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -95,6 +95,8 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -402,6 +404,32 @@ class BitcoinRoutes {
     }
   }
 
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`Not implemented`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!from) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (!to) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksByBulk(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];
diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index d110186f5..5c8884c71 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -688,7 +688,6 @@ class Blocks {
   }
 
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-
     let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
     if (currentHeight > this.currentBlockHeight) {
       limit -= currentHeight - this.currentBlockHeight;
@@ -728,6 +727,63 @@ class Blocks {
     return returnBlocks;
   }
 
+  public async $getBlocksByBulk(start: number, end: number) {
+    start = Math.max(1, start);
+
+    const blocks: any[] = [];
+    for (let i = end; i >= start; --i) {
+      const blockHash = await bitcoinApi.$getBlockHash(i);
+      const coreBlock = await bitcoinClient.getBlock(blockHash);
+      const electrsBlock = await bitcoinApi.$getBlock(blockHash);
+      const txs = await this.$getTransactionsExtended(blockHash, i, true);
+      const stats = await bitcoinClient.getBlockStats(blockHash);
+      const header = await bitcoinClient.getBlockHeader(blockHash, false);
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', i);
+
+      const formatted = {
+        blockhash: coreBlock.id,
+        blockheight: coreBlock.height,
+        prev_blockhash: coreBlock.previousblockhash,
+        timestamp: coreBlock.timestamp,
+        median_timestamp: coreBlock.mediantime,
+        // @ts-ignore
+        blocktime: coreBlock.time,
+        orphaned: null,
+        header: header,
+        version: coreBlock.version,
+        difficulty: coreBlock.difficulty,
+        merkle_root: coreBlock.merkle_root,
+        bits: coreBlock.bits,
+        nonce: coreBlock.nonce,
+        coinbase_scriptsig: txs[0].vin[0].scriptsig,
+        coinbase_address: txs[0].vout[0].scriptpubkey_address,
+        coinbase_signature: txs[0].vout[0].scriptpubkey_asm,
+        size: coreBlock.size,
+        virtual_size: coreBlock.weight / 4.0,
+        weight: coreBlock.weight,
+        utxoset_size: txoutset.txouts,
+        utxoset_change: stats.utxo_increase,
+        total_txs: coreBlock.tx_count,
+        avg_tx_size: Math.round(stats.total_size / stats.txs * 100) * 0.01,
+        total_inputs: stats.ins,
+        total_outputs: stats.outs,
+        total_input_amt: Math.round(txoutset.block_info.prevout_spent * 100000000),
+        total_output_amt: stats.total_out,
+        block_subsidy: txs[0].vout.reduce((acc, curr) => acc + curr.value, 0),
+        total_fee: stats.totalfee,
+        avg_feerate: stats.avgfeerate,
+        feerate_percentiles: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
+        avg_fee: stats.avgfee,
+        fee_percentiles: null,
+        segwit_total_txs: stats.swtxs,
+        segwit_total_size: stats.swtotal_size,
+        segwit_total_weight: stats.swtotal_weight,
+      };
+      blocks.push(formatted);
+    }
+    return blocks;
+  }
+
   public async $getBlockAuditSummary(hash: string): Promise<any> {
     let summary;
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
diff --git a/backend/src/rpc-api/commands.ts b/backend/src/rpc-api/commands.ts
index ea9bd7bf0..5905a2bb6 100644
--- a/backend/src/rpc-api/commands.ts
+++ b/backend/src/rpc-api/commands.ts
@@ -88,5 +88,6 @@ module.exports = {
   verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
+  walletPassphraseChange: 'walletpassphrasechange',
+  getTxoutSetinfo: 'gettxoutsetinfo'
 }

From 73f76474dd7fa0d8f95e8fd02af8c923b6b7fa86 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 17 Feb 2023 21:21:21 +0900
Subject: [PATCH 02/23] Implemented coinstatsindex indexing

---
 backend/src/api/bitcoin/bitcoin.routes.ts     |   7 +-
 .../src/api/bitcoin/esplora-api.interface.ts  |   1 +
 backend/src/api/blocks.ts                     | 158 ++++++++++--------
 backend/src/api/database-migration.ts         |  29 +++-
 backend/src/api/mining/mining.ts              |  41 ++++-
 backend/src/api/transaction-utils.ts          |   1 +
 backend/src/index.ts                          |   1 +
 backend/src/indexer.ts                        |  76 ++++++++-
 backend/src/mempool.interfaces.ts             |  21 +++
 backend/src/repositories/BlocksRepository.ts  | 107 +++++++++---
 backend/src/rpc-api/commands.ts               |   5 +-
 11 files changed, 330 insertions(+), 117 deletions(-)

diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts
index 0ff30376c..6d145e854 100644
--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -407,7 +407,10 @@ class BitcoinRoutes {
   private async getBlocksByBulk(req: Request, res: Response) {
     try {
       if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
-        return res.status(404).send(`Not implemented`);
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
       }
 
       const from = parseInt(req.params.from, 10);
@@ -423,7 +426,7 @@ class BitcoinRoutes {
       }
 
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(await blocks.$getBlocksByBulk(from, to));
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
 
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts
index 39f8cfd6f..eaf6476f4 100644
--- a/backend/src/api/bitcoin/esplora-api.interface.ts
+++ b/backend/src/api/bitcoin/esplora-api.interface.ts
@@ -88,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    medianTime?: number;
   }
 
   export interface Address {
diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 5c8884c71..d950a9bd3 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -165,33 +165,75 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = priceUpdater.latestPrices.USD;
+    const blk: BlockExtended = Object.assign({ extras: {} }, block);
+    blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
+    blk.extras.usd = priceUpdater.latestPrices.USD;
 
     if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      blk.extras.medianFee = 0; // 50th percentiles
+      blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blk.extras.totalFees = 0;
+      blk.extras.avgFee = 0;
+      blk.extras.avgFeeRate = 0;
+      blk.extras.utxoSetChange = 0;
+      blk.extras.avgTxSize = 0;
+      blk.extras.totalInputs = 0;
+      blk.extras.totalOutputs = 1;
+      blk.extras.totalOutputAmt = 0;
+      blk.extras.segwitTotalTxs = 0;
+      blk.extras.segwitTotalSize = 0;
+      blk.extras.segwitTotalWeight = 0;
     } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      const stats = await bitcoinClient.getBlockStats(block.id);
+      blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blk.extras.totalFees = stats.totalfee;
+      blk.extras.avgFee = stats.avgfee;
+      blk.extras.avgFeeRate = stats.avgfeerate;
+      blk.extras.utxoSetChange = stats.utxo_increase;
+      blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      blk.extras.totalInputs = stats.ins;
+      blk.extras.totalOutputs = stats.outs;
+      blk.extras.totalOutputAmt = stats.total_out;
+      blk.extras.segwitTotalTxs = stats.swtxs;
+      blk.extras.segwitTotalSize = stats.swtotal_size;
+      blk.extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    blk.extras.feePercentiles = [], // TODO
+    blk.extras.medianFeeAmt = 0; // TODO
+    blk.extras.medianTimestamp = block.medianTime; // TODO
+    blk.extras.blockTime = 0; // TODO
+    blk.extras.orphaned = false; // TODO
+  
+    blk.extras.virtualSize = block.weight / 4.0;
+    if (blk.extras.coinbaseTx.vout.length > 0) {
+      blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+    } else {
+      blk.extras.coinbaseAddress = null;
+      blk.extras.coinbaseSignature = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    blk.extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      blk.extras.utxoSetSize = txoutset.txouts,
+      blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      blk.extras.utxoSetSize = null;
+      blk.extras.totalInputAmt = null;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      if (blk.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -201,10 +243,10 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
       } else {
-        blockExtended.extras.pool = {
+        blk.extras.pool = {
           id: pool.id,
           name: pool.name,
           slug: pool.slug,
@@ -214,12 +256,12 @@ class Blocks {
       if (config.MEMPOOL.AUDIT) {
         const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
         if (auditScore != null) {
-          blockExtended.extras.matchRate = auditScore.matchRate;
+          blk.extras.matchRate = auditScore.matchRate;
         }
       }
     }
 
-    return blockExtended;
+    return blk;
   }
 
   /**
@@ -727,60 +769,28 @@ class Blocks {
     return returnBlocks;
   }
 
-  public async $getBlocksByBulk(start: number, end: number) {
-    start = Math.max(1, start);
+  /**
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
 
     const blocks: any[] = [];
-    for (let i = end; i >= start; --i) {
-      const blockHash = await bitcoinApi.$getBlockHash(i);
-      const coreBlock = await bitcoinClient.getBlock(blockHash);
-      const electrsBlock = await bitcoinApi.$getBlock(blockHash);
-      const txs = await this.$getTransactionsExtended(blockHash, i, true);
-      const stats = await bitcoinClient.getBlockStats(blockHash);
-      const header = await bitcoinClient.getBlockHeader(blockHash, false);
-      const txoutset = await bitcoinClient.getTxoutSetinfo('none', i);
 
-      const formatted = {
-        blockhash: coreBlock.id,
-        blockheight: coreBlock.height,
-        prev_blockhash: coreBlock.previousblockhash,
-        timestamp: coreBlock.timestamp,
-        median_timestamp: coreBlock.mediantime,
-        // @ts-ignore
-        blocktime: coreBlock.time,
-        orphaned: null,
-        header: header,
-        version: coreBlock.version,
-        difficulty: coreBlock.difficulty,
-        merkle_root: coreBlock.merkle_root,
-        bits: coreBlock.bits,
-        nonce: coreBlock.nonce,
-        coinbase_scriptsig: txs[0].vin[0].scriptsig,
-        coinbase_address: txs[0].vout[0].scriptpubkey_address,
-        coinbase_signature: txs[0].vout[0].scriptpubkey_asm,
-        size: coreBlock.size,
-        virtual_size: coreBlock.weight / 4.0,
-        weight: coreBlock.weight,
-        utxoset_size: txoutset.txouts,
-        utxoset_change: stats.utxo_increase,
-        total_txs: coreBlock.tx_count,
-        avg_tx_size: Math.round(stats.total_size / stats.txs * 100) * 0.01,
-        total_inputs: stats.ins,
-        total_outputs: stats.outs,
-        total_input_amt: Math.round(txoutset.block_info.prevout_spent * 100000000),
-        total_output_amt: stats.total_out,
-        block_subsidy: txs[0].vout.reduce((acc, curr) => acc + curr.value, 0),
-        total_fee: stats.totalfee,
-        avg_feerate: stats.avgfeerate,
-        feerate_percentiles: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
-        avg_fee: stats.avgfee,
-        fee_percentiles: null,
-        segwit_total_txs: stats.swtxs,
-        segwit_total_size: stats.swtotal_size,
-        segwit_total_weight: stats.swtotal_weight,
-      };
-      blocks.push(formatted);
+    while (fromHeight <= toHeight) {
+      let block = await blocksRepository.$getBlockByHeight(fromHeight);
+      if (!block) {
+        block = await this.$indexBlock(fromHeight);
+      }
+      blocks.push(block);
+      fromHeight++;
     }
+
     return blocks;
   }
 
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 6e4221857..2c6adfd1b 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 54;
+  private static currentVersion = 55;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -483,6 +483,11 @@ class DatabaseMigration {
       }
       await this.updateToSchemaVersion(54);
     }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      await this.updateToSchemaVersion(55);
+    }
   }
 
   /**
@@ -756,6 +761,28 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD block_time int unsigned NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NOT NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,
diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts
index edcb5b2e5..f33a68dcb 100644
--- a/backend/src/api/mining/mining.ts
+++ b/backend/src/api/mining/mining.ts
@@ -172,7 +172,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
@@ -279,7 +279,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
@@ -459,7 +459,7 @@ class Mining {
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -520,6 +520,41 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);
diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts
index fb5aeea42..fb69419fc 100644
--- a/backend/src/api/transaction-utils.ts
+++ b/backend/src/api/transaction-utils.ts
@@ -14,6 +14,7 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)
diff --git a/backend/src/index.ts b/backend/src/index.ts
index 919c039c3..d8d46fc9f 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
+import mining from './api/mining/mining';
 import { AxiosError } from 'axios';
 
 class Server {
diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts
index 22f3ce319..41c8024e0 100644
--- a/backend/src/indexer.ts
+++ b/backend/src/indexer.ts
@@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
   tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
 
-  public reindex() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async runSingleTask(task: 'blocksPrices') {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
     if (!Common.indexingEnabled()) {
       return;
     }
@@ -28,20 +77,27 @@ class Indexer {
       this.tasksRunning.push(task);
       const lastestPriceId = await PricesRepository.$getLatestPriceId();
       if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
         setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
           this.runSingleTask('blocksPrices');
         }, 10000);
       } else {
-        logger.debug(`Blocks prices indexer will run now`)
+        logger.debug(`Blocks prices indexer will run now`);
         await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
       }
     }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
   }
 
-  public async $run() {
+  public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -57,7 +113,9 @@ class Indexer {
     this.runIndexer = false;
     this.indexerRunning = true;
 
-    logger.debug(`Running mining indexer`);
+    logger.info(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();
 
     try {
       await priceUpdater.$run();
@@ -93,7 +151,7 @@ class Indexer {
     setTimeout(() => this.reindex(), runEvery);
   }
 
-  async $resetHashratesIndexingState() {
+  async $resetHashratesIndexingState(): Promise<void> {
     try {
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index 6b258c173..a1a9e1687 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -64,6 +64,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -160,6 +161,26 @@ export interface BlockExtension {
   avgFeeRate?: number;
   coinbaseRaw?: string;
   usd?: number | null;
+  medianTimestamp?: number;
+  blockTime?: number;
+  orphaned?: boolean;
+  coinbaseAddress?: string | null;
+  coinbaseSignature?: string | null;
+  virtualSize?: number;
+  avgTxSize?: number;
+  totalInputs?: number;
+  totalOutputs?: number;
+  totalOutputAmt?: number;
+  medianFeeAmt?: number;
+  feePercentiles?: number[],
+  segwitTotalTxs?: number;
+  segwitTotalSize?: number;
+  segwitTotalWeight?: number;
+  header?: string;
+  utxoSetChange?: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize?: number | null;
+  totalInputAmt?: number | null;
 }
 
 export interface BlockExtended extends IEsploraApi.Block {
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index df98719b9..baaea38d9 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -18,17 +18,27 @@ class BlocksRepository {
   public async $saveBlockInDatabase(block: BlockExtended) {
     try {
       const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,             hash,                blockTimestamp,    size,
+        weight,             tx_count,            coinbase_raw,      difficulty,
+        pool_id,            fees,                fee_span,          median_fee,
+        reward,             version,             bits,              nonce,
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   block_time,          header,            coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?
       )`;
 
       const params: any[] = [
@@ -52,6 +62,23 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
+        block.extras.medianTimestamp,
+        block.extras.blockTime,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        block.extras.coinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        JSON.stringify(block.extras.feePercentiles),
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
       ];
 
       await DB.query(query, params);
@@ -65,6 +92,33 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -310,32 +364,16 @@ class BlocksRepository {
   public async $getBlockByHeight(height: number): Promise<object | null> {
     try {
       const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
-        hash,
+        blocks.*,
         hash as id,
         UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
         pools.id as pool_id,
         pools.name as pool_name,
         pools.link as pool_link,
         pools.slug as pool_slug,
         pools.addresses as pool_addresses,
         pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
+        previous_block_hash as previousblockhash
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE blocks.height = ${height}
@@ -694,7 +732,6 @@ class BlocksRepository {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
-    return [];
   }
 
   /**
@@ -741,7 +778,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -754,6 +791,24 @@ class BlocksRepository {
       }
     }
   }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksRepository();
diff --git a/backend/src/rpc-api/commands.ts b/backend/src/rpc-api/commands.ts
index 5905a2bb6..78f5e12f4 100644
--- a/backend/src/rpc-api/commands.ts
+++ b/backend/src/rpc-api/commands.ts
@@ -89,5 +89,6 @@ module.exports = {
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
   walletPassphraseChange: 'walletpassphrasechange',
-  getTxoutSetinfo: 'gettxoutsetinfo'
-}
+  getTxoutSetinfo: 'gettxoutsetinfo',
+  getIndexInfo: 'getindexinfo',
+};

From 8612dd2d73cd173d02e8d01d640ddfb8c89f74c1 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 18 Feb 2023 08:28:59 +0900
Subject: [PATCH 03/23] Remove unescessary data from the blocks-bulk API

---
 backend/src/api/blocks.ts                    | 10 +++++++++-
 backend/src/repositories/BlocksRepository.ts |  1 +
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index d950a9bd3..9c1c1d05b 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -783,10 +783,18 @@ class Blocks {
     const blocks: any[] = [];
 
     while (fromHeight <= toHeight) {
-      let block = await blocksRepository.$getBlockByHeight(fromHeight);
+      let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
       if (!block) {
         block = await this.$indexBlock(fromHeight);
       }
+
+      delete(block.hash);
+      delete(block.previous_block_hash);
+      delete(block.pool_name);
+      delete(block.pool_link);
+      delete(block.pool_addresses);
+      delete(block.pool_regexes);
+
       blocks.push(block);
       fromHeight++;
     }
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index baaea38d9..cc6fdeb08 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -384,6 +384,7 @@ class BlocksRepository {
       }
 
       rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
       return rows[0];
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));

From 8f716a1d8c7e5eeb4370e36119cc2446b37355b5 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 18 Feb 2023 08:42:38 +0900
Subject: [PATCH 04/23] Fix median timestamp field - Fix reponse format when
 block is indexed on the fly

---
 backend/src/api/blocks.ts                    | 9 +++++++--
 backend/src/repositories/BlocksRepository.ts | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 9c1c1d05b..006d5f055 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -170,6 +170,7 @@ class Blocks {
     blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
     blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
     blk.extras.usd = priceUpdater.latestPrices.USD;
+    blk.extras.medianTimestamp = block.medianTime;
 
     if (block.height === 0) {
       blk.extras.medianFee = 0; // 50th percentiles
@@ -204,7 +205,6 @@ class Blocks {
 
     blk.extras.feePercentiles = [], // TODO
     blk.extras.medianFeeAmt = 0; // TODO
-    blk.extras.medianTimestamp = block.medianTime; // TODO
     blk.extras.blockTime = 0; // TODO
     blk.extras.orphaned = false; // TODO
   
@@ -785,7 +785,11 @@ class Blocks {
     while (fromHeight <= toHeight) {
       let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
       if (!block) {
-        block = await this.$indexBlock(fromHeight);
+        await this.$indexBlock(fromHeight);
+        block = await blocksRepository.$getBlockByHeight(fromHeight);
+        if (!block) {
+          continue;
+        }
       }
 
       delete(block.hash);
@@ -794,6 +798,7 @@ class Blocks {
       delete(block.pool_link);
       delete(block.pool_addresses);
       delete(block.pool_regexes);
+      delete(block.median_timestamp);
 
       blocks.push(block);
       fromHeight++;
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index cc6fdeb08..d7811f601 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -367,6 +367,7 @@ class BlocksRepository {
         blocks.*,
         hash as id,
         UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+        UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
         pools.id as pool_id,
         pools.name as pool_name,
         pools.link as pool_link,

From 458f24c9f21a84269c4fd6e67ec57eef84ed4845 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 18 Feb 2023 11:26:13 +0900
Subject: [PATCH 05/23] Compute median fee and fee percentiles in sats

---
 backend/src/api/blocks.ts                     | 18 ++++++--
 backend/src/api/database-migration.ts         |  2 +-
 backend/src/mempool.interfaces.ts             |  4 +-
 backend/src/repositories/BlocksRepository.ts  | 19 +++++++++
 .../repositories/BlocksSummariesRepository.ts | 42 +++++++++++++++++++
 5 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 006d5f055..ccf7bd2f4 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -203,10 +203,13 @@ class Blocks {
       blk.extras.segwitTotalWeight = stats.swtotal_weight;
     }
 
-    blk.extras.feePercentiles = [], // TODO
-    blk.extras.medianFeeAmt = 0; // TODO
     blk.extras.blockTime = 0; // TODO
     blk.extras.orphaned = false; // TODO
+
+    blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+    if (blk.extras.feePercentiles !== null) {
+      blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
+    }
   
     blk.extras.virtualSize = block.weight / 4.0;
     if (blk.extras.coinbaseTx.vout.length > 0) {
@@ -791,7 +794,6 @@ class Blocks {
           continue;
         }
       }
-
       delete(block.hash);
       delete(block.previous_block_hash);
       delete(block.pool_name);
@@ -800,6 +802,16 @@ class Blocks {
       delete(block.pool_regexes);
       delete(block.median_timestamp);
 
+      // This requires `blocks_summaries` to be available. It takes a very long
+      // time to index this table so we just try to serve the data the best we can
+      if (block.fee_percentiles === null) {
+        block.fee_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); 
+        if (block.fee_percentiles !== null) {
+          block.median_fee_amt = block.fee_percentiles[3];
+          await blocksRepository.$saveFeePercentilesForBlockId(block.id, block.fee_percentiles);
+        }
+      }
+
       blocks.push(block);
       fromHeight++;
     }
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 2c6adfd1b..352abfbfe 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -772,7 +772,7 @@ class DatabaseMigration {
       ADD total_outputs int unsigned NOT NULL,
       ADD total_output_amt bigint unsigned NOT NULL,
       ADD fee_percentiles longtext NULL,
-      ADD median_fee_amt int unsigned NOT NULL,
+      ADD median_fee_amt int unsigned NULL,
       ADD segwit_total_txs int unsigned NOT NULL,
       ADD segwit_total_size int unsigned NOT NULL,
       ADD segwit_total_weight int unsigned NOT NULL,
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index a1a9e1687..a7e7c4ec6 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -171,8 +171,8 @@ export interface BlockExtension {
   totalInputs?: number;
   totalOutputs?: number;
   totalOutputAmt?: number;
-  medianFeeAmt?: number;
-  feePercentiles?: number[],
+  medianFeeAmt?: number | null;
+  feePercentiles?: number[] | null,
   segwitTotalTxs?: number;
   segwitTotalSize?: number;
   segwitTotalWeight?: number;
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index d7811f601..cc0b43fe9 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -811,6 +811,25 @@ class BlocksRepository {
       throw e;
     }
   }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksRepository();
diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts
index 1406a1d07..ebc83b7dd 100644
--- a/backend/src/repositories/BlocksSummariesRepository.ts
+++ b/backend/src/repositories/BlocksSummariesRepository.ts
@@ -80,6 +80,48 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0], // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)], // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)], // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)], // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)], // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)], // 90th
+        fees[fees.length - 1], // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

From 281899f5514417224be4275873b4804a3323dd7e Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 18 Feb 2023 14:10:07 +0900
Subject: [PATCH 06/23] List orphaned blocks in the new blocks-bulk API

---
 backend/src/api/blocks.ts         |  8 ++++-
 backend/src/api/chain-tips.ts     | 53 +++++++++++++++++++++++++++++++
 backend/src/index.ts              |  2 ++
 backend/src/mempool.interfaces.ts |  3 +-
 4 files changed, 64 insertions(+), 2 deletions(-)
 create mode 100644 backend/src/api/chain-tips.ts

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index ccf7bd2f4..25c199de9 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -25,6 +25,7 @@ import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
+import chainTips from './chain-tips';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -171,6 +172,7 @@ class Blocks {
     blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
     blk.extras.usd = priceUpdater.latestPrices.USD;
     blk.extras.medianTimestamp = block.medianTime;
+    blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
 
     if (block.height === 0) {
       blk.extras.medianFee = 0; // 50th percentiles
@@ -204,7 +206,6 @@ class Blocks {
     }
 
     blk.extras.blockTime = 0; // TODO
-    blk.extras.orphaned = false; // TODO
 
     blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
     if (blk.extras.feePercentiles !== null) {
@@ -545,6 +546,7 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        await chainTips.updateOrphanedBlocks();
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -812,6 +814,10 @@ class Blocks {
         }
       }
 
+      // Re-org can happen after indexing so we need to always get the
+      // latest state from core
+      block.orphans = chainTips.getOrphanedBlocksAtHeight(block.height);
+
       blocks.push(block);
       fromHeight++;
     }
diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts
new file mode 100644
index 000000000..5b0aa8a5c
--- /dev/null
+++ b/backend/src/api/chain-tips.ts
@@ -0,0 +1,53 @@
+import logger from "../logger";
+import bitcoinClient from "./bitcoin/bitcoin-client";
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    this.chainTips = await bitcoinClient.getChainTips();
+    this.orphanedBlocks = [];
+
+    for (const chain of this.chainTips) {
+      if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') {
+        let block = await bitcoinClient.getBlock(chain.hash);
+        while (block && block.confirmations === -1) {
+          this.orphanedBlocks.push({
+            height: block.height,
+            hash: block.hash,
+            status: chain.status
+          });
+          block = await bitcoinClient.getBlock(block.previousblockhash);
+        }
+      }
+    }
+
+    logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+  }
+
+  public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();
\ No newline at end of file
diff --git a/backend/src/index.ts b/backend/src/index.ts
index d8d46fc9f..6ea3ddc43 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -37,6 +37,7 @@ import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
 import mining from './api/mining/mining';
+import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';
 
 class Server {
@@ -134,6 +135,7 @@ class Server {
     }
 
     priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
 
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index a7e7c4ec6..e139bde8f 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -1,4 +1,5 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { OrphanedBlock } from './api/chain-tips';
 import { HeapNode } from "./utils/pairing-heap";
 
 export interface PoolTag {
@@ -163,7 +164,7 @@ export interface BlockExtension {
   usd?: number | null;
   medianTimestamp?: number;
   blockTime?: number;
-  orphaned?: boolean;
+  orphans?: OrphanedBlock[] | null;
   coinbaseAddress?: string | null;
   coinbaseSignature?: string | null;
   virtualSize?: number;

From e2fe39f241432ed6ec387952f5570071f7b81658 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 18 Feb 2023 14:53:21 +0900
Subject: [PATCH 07/23] Wrap orphaned blocks updater into try/catch

---
 backend/src/api/chain-tips.ts | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts
index 5b0aa8a5c..92f148c8e 100644
--- a/backend/src/api/chain-tips.ts
+++ b/backend/src/api/chain-tips.ts
@@ -19,24 +19,28 @@ class ChainTips {
   private orphanedBlocks: OrphanedBlock[] = [];
 
   public async updateOrphanedBlocks(): Promise<void> {
-    this.chainTips = await bitcoinClient.getChainTips();
-    this.orphanedBlocks = [];
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
 
-    for (const chain of this.chainTips) {
-      if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') {
-        let block = await bitcoinClient.getBlock(chain.hash);
-        while (block && block.confirmations === -1) {
-          this.orphanedBlocks.push({
-            height: block.height,
-            hash: block.hash,
-            status: chain.status
-          });
-          block = await bitcoinClient.getBlock(block.previousblockhash);
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
         }
       }
-    }
 
-    logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
   }
 
   public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {

From 6965c8f41ba3d2a358f17f27294f12fd0798bca8 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sun, 19 Feb 2023 18:38:23 +0900
Subject: [PATCH 08/23] Fix median time indexing

---
 backend/src/api/bitcoin/bitcoin-api.ts       | 1 +
 backend/src/repositories/BlocksRepository.ts | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts
index cad11aeda..117245ef8 100644
--- a/backend/src/api/bitcoin/bitcoin-api.ts
+++ b/backend/src/api/bitcoin/bitcoin-api.ts
@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       size: block.size,
       weight: block.weight,
       previousblockhash: block.previousblockhash,
+      medianTime: block.mediantime,
     };
   }
 
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index cc0b43fe9..20331897c 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -34,7 +34,7 @@ class BlocksRepository {
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,

From eceedf0bdfa6fd806596d2f707bc3f437de94c9d Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sun, 19 Feb 2023 19:17:51 +0900
Subject: [PATCH 09/23] Dont compute fee percentile / median fee when indexing
 is disabled because we need summaries

---
 backend/src/api/blocks.ts | 8 +++++---
 backend/src/database.ts   | 3 ++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 25c199de9..8a11bccc5 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -207,9 +207,11 @@ class Blocks {
 
     blk.extras.blockTime = 0; // TODO
 
-    blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
-    if (blk.extras.feePercentiles !== null) {
-      blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
+    if (Common.indexingEnabled()) {
+      blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+      if (blk.extras.feePercentiles !== null) {
+        blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
+      }
     }
   
     blk.extras.virtualSize = block.weight / 4.0;
diff --git a/backend/src/database.ts b/backend/src/database.ts
index c2fb0980b..a504eb0fa 100644
--- a/backend/src/database.ts
+++ b/backend/src/database.ts
@@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
 
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
     }
   }
 

From b2eaa7efb1636ddcfb37fc3ab1c7671c82be91d1 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Mon, 20 Feb 2023 11:59:38 +0900
Subject: [PATCH 10/23] Fix fee percentiles indexing

---
 backend/src/repositories/BlocksRepository.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index 20331897c..1f244d7cd 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -74,7 +74,7 @@ class BlocksRepository {
         block.extras.totalOutputs,
         block.extras.totalInputAmt,
         block.extras.totalOutputAmt,
-        JSON.stringify(block.extras.feePercentiles),
+        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
         block.extras.segwitTotalTxs,
         block.extras.segwitTotalSize,
         block.extras.segwitTotalWeight,

From 75a99568bfff64d5192df21f9093bc3e412b07fd Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Thu, 23 Feb 2023 08:50:30 +0900
Subject: [PATCH 11/23] Index coinbase signature in ascii

---
 backend/src/api/blocks.ts                    | 2 ++
 backend/src/api/database-migration.ts        | 1 +
 backend/src/mempool.interfaces.ts            | 1 +
 backend/src/repositories/BlocksRepository.ts | 5 +++--
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 8a11bccc5..3d33642ce 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -218,9 +218,11 @@ class Blocks {
     if (blk.extras.coinbaseTx.vout.length > 0) {
       blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
       blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+      blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
     } else {
       blk.extras.coinbaseAddress = null;
       blk.extras.coinbaseSignature = null;
+      blk.extras.coinbaseSignatureAscii = null;
     }
 
     const header = await bitcoinClient.getBlockHeader(block.id, false);
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 352abfbfe..c965ef420 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -767,6 +767,7 @@ class DatabaseMigration {
       ADD block_time int unsigned NOT NULL,
       ADD coinbase_address varchar(100) NULL,
       ADD coinbase_signature varchar(500) NULL,
+      ADD coinbase_signature_ascii varchar(500) NULL,
       ADD avg_tx_size double unsigned NOT NULL,
       ADD total_inputs int unsigned NOT NULL,
       ADD total_outputs int unsigned NOT NULL,
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index e139bde8f..cb95be98a 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -167,6 +167,7 @@ export interface BlockExtension {
   orphans?: OrphanedBlock[] | null;
   coinbaseAddress?: string | null;
   coinbaseSignature?: string | null;
+  coinbaseSignatureAscii?: string | null;
   virtualSize?: number;
   avgTxSize?: number;
   totalInputs?: number;
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index 1f244d7cd..e2362b67d 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -27,7 +27,7 @@ class BlocksRepository {
         coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
         total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
         fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
-        median_fee_amt
+        median_fee_amt,     coinbase_signature_ascii
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
@@ -38,7 +38,7 @@ class BlocksRepository {
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?
+        ?, ?
       )`;
 
       const params: any[] = [
@@ -79,6 +79,7 @@ class BlocksRepository {
         block.extras.segwitTotalSize,
         block.extras.segwitTotalWeight,
         block.extras.medianFeeAmt,
+        block.extras.coinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);

From 086ee68b520156e8b1fe6b63dc2cbe78ea4e2e5f Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 10:29:58 +0900
Subject: [PATCH 12/23] Remove `block_time` from indexed fields

---
 backend/src/api/blocks.ts             | 2 --
 backend/src/api/database-migration.ts | 1 -
 2 files changed, 3 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 3d33642ce..ba3927ab7 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -205,8 +205,6 @@ class Blocks {
       blk.extras.segwitTotalWeight = stats.swtotal_weight;
     }
 
-    blk.extras.blockTime = 0; // TODO
-
     if (Common.indexingEnabled()) {
       blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
       if (blk.extras.feePercentiles !== null) {
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index c965ef420..6e6e6855f 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -764,7 +764,6 @@ class DatabaseMigration {
   private getAdditionalBlocksDataQuery(): string {
     return `ALTER TABLE blocks
       ADD median_timestamp timestamp NOT NULL,
-      ADD block_time int unsigned NOT NULL,
       ADD coinbase_address varchar(100) NULL,
       ADD coinbase_signature varchar(500) NULL,
       ADD coinbase_signature_ascii varchar(500) NULL,

From a0488dba7664acc26e7517e3b019b8dd28d43324 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 11:33:36 +0900
Subject: [PATCH 13/23] Cleanup block before sending response in /blocks-bulk
 API Remove block_time Index summaries on the fly

---
 backend/src/api/blocks.ts                    | 70 +++++++++++++++-----
 backend/src/repositories/BlocksRepository.ts |  5 +-
 2 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index ba3927ab7..459b3903e 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -205,7 +205,7 @@ class Blocks {
       blk.extras.segwitTotalWeight = stats.swtotal_weight;
     }
 
-    if (Common.indexingEnabled()) {
+    if (Common.blocksSummariesIndexingEnabled()) {
       blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
       if (blk.extras.feePercentiles !== null) {
         blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
@@ -798,29 +798,65 @@ class Blocks {
           continue;
         }
       }
-      delete(block.hash);
-      delete(block.previous_block_hash);
-      delete(block.pool_name);
-      delete(block.pool_link);
-      delete(block.pool_addresses);
-      delete(block.pool_regexes);
-      delete(block.median_timestamp);
 
-      // This requires `blocks_summaries` to be available. It takes a very long
-      // time to index this table so we just try to serve the data the best we can
-      if (block.fee_percentiles === null) {
-        block.fee_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); 
-        if (block.fee_percentiles !== null) {
-          block.median_fee_amt = block.fee_percentiles[3];
-          await blocksRepository.$saveFeePercentilesForBlockId(block.id, block.fee_percentiles);
+      // Cleanup fields before sending the response
+      const cleanBlock: any = {
+        height: block.height ?? null,
+        hash: block.id ?? null,
+        timestamp: block.blockTimestamp ?? null,
+        median_timestamp: block.medianTime ?? null,
+        previousblockhash: block.previousblockhash ?? null,
+        difficulty: block.difficulty ?? null,
+        header: block.header ?? null,
+        version: block.version ?? null,
+        bits: block.bits ?? null,
+        nonce: block.nonce ?? null,
+        size: block.size ?? null,
+        weight: block.weight ?? null,
+        tx_count: block.tx_count ?? null,
+        merkle_root: block.merkle_root ?? null,
+        reward: block.reward ?? null,
+        total_fee_amt: block.fees ?? null,
+        avg_fee_amt: block.avg_fee ?? null,
+        median_fee_amt: block.median_fee_amt ?? null,
+        fee_amt_percentiles: block.fee_percentiles ?? null,
+        avg_fee_rate: block.avg_fee_rate ?? null,
+        median_fee_rate: block.median_fee ?? null,
+        fee_rate_percentiles: block.fee_span ?? null,
+        total_inputs: block.total_inputs ?? null,
+        total_input_amt: block.total_input_amt ?? null,
+        total_outputs: block.total_outputs ?? null,
+        total_output_amt: block.total_output_amt ?? null,
+        segwit_total_txs: block.segwit_total_txs ?? null,
+        segwit_total_size: block.segwit_total_size ?? null,
+        segwit_total_weight: block.segwit_total_weight ?? null,
+        avg_tx_size: block.avg_tx_size ?? null,
+        utxoset_change: block.utxoset_change ?? null,
+        utxoset_size: block.utxoset_size ?? null,
+        coinbase_raw: block.coinbase_raw ?? null,
+        coinbase_address: block.coinbase_address ?? null,
+        coinbase_signature: block.coinbase_signature ?? null,
+        pool_slug: block.pool_slug ?? null,
+      };
+
+      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
+        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        if (cleanBlock.fee_amt_percentiles === null) {
+          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+          const summary = this.summarizeBlock(block);
+          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        }
+        if (cleanBlock.fee_amt_percentiles !== null) {
+          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
         }
       }
 
       // Re-org can happen after indexing so we need to always get the
       // latest state from core
-      block.orphans = chainTips.getOrphanedBlocksAtHeight(block.height);
+      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
 
-      blocks.push(block);
+      blocks.push(cleanBlock);
       fromHeight++;
     }
 
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index e2362b67d..86dc006ff 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -23,7 +23,7 @@ class BlocksRepository {
         pool_id,            fees,                fee_span,          median_fee,
         reward,             version,             bits,              nonce,
         merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
-        median_timestamp,   block_time,          header,            coinbase_address,
+        median_timestamp,   header,              coinbase_address,
         coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
         total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
         fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
@@ -34,7 +34,7 @@ class BlocksRepository {
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        FROM_UNIXTIME(?), ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
@@ -63,7 +63,6 @@ class BlocksRepository {
         block.extras.avgFee,
         block.extras.avgFeeRate,
         block.extras.medianTimestamp,
-        block.extras.blockTime,
         block.extras.header,
         block.extras.coinbaseAddress,
         block.extras.coinbaseSignature,

From 0bf4d5218326373cdafb195ae3c08235419f76fd Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 11:41:54 +0900
Subject: [PATCH 14/23] Return zeroed out `fee_amt_percentiles` if there is no
 transaction

---
 .../src/repositories/BlocksSummariesRepository.ts  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts
index ebc83b7dd..2724ddcf5 100644
--- a/backend/src/repositories/BlocksSummariesRepository.ts
+++ b/backend/src/repositories/BlocksSummariesRepository.ts
@@ -108,13 +108,13 @@ class BlocksSummariesRepository {
       const fees = transactions.map((t: any) => t.fee);
 
       return [
-        fees[0], // min
-        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)], // 10th
-        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)], // 25th
-        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)], // median
-        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)], // 75th
-        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)], // 90th
-        fees[fees.length - 1], // max
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
       ];
 
     } catch (e) {

From aa1114926c2296d79b9429a6e0ca47cb7b117c62 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 11:43:38 +0900
Subject: [PATCH 15/23] `previousblockhash` -> `previous_block_hash`

---
 backend/src/api/blocks.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 459b3903e..2a026a303 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -805,7 +805,7 @@ class Blocks {
         hash: block.id ?? null,
         timestamp: block.blockTimestamp ?? null,
         median_timestamp: block.medianTime ?? null,
-        previousblockhash: block.previousblockhash ?? null,
+        previous_block_hash: block.previousblockhash ?? null,
         difficulty: block.difficulty ?? null,
         header: block.header ?? null,
         version: block.version ?? null,

From e19db4ae35e8d4f3a81c1a5b7c1ba5802d24bcd2 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 11:46:47 +0900
Subject: [PATCH 16/23] Add missing `coinbase_signature_ascii`

---
 backend/src/api/blocks.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 2a026a303..fb38e0d7e 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -836,6 +836,7 @@ class Blocks {
         coinbase_raw: block.coinbase_raw ?? null,
         coinbase_address: block.coinbase_address ?? null,
         coinbase_signature: block.coinbase_signature ?? null,
+        coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
         pool_slug: block.pool_slug ?? null,
       };
 

From ed8cf89fee51bf54a39f60421255711d0d981509 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 12:48:55 +0900
Subject: [PATCH 17/23] Format percentiles in a more verbose way

---
 backend/src/api/blocks.ts | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index fb38e0d7e..204419496 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -853,6 +853,25 @@ class Blocks {
         }
       }
 
+      cleanBlock.fee_amt_percentiles = {
+        'min': cleanBlock.fee_amt_percentiles[0],
+        'perc_10': cleanBlock.fee_amt_percentiles[1],
+        'perc_25': cleanBlock.fee_amt_percentiles[2],
+        'perc_50': cleanBlock.fee_amt_percentiles[3],
+        'perc_75': cleanBlock.fee_amt_percentiles[4],
+        'perc_90': cleanBlock.fee_amt_percentiles[5],
+        'max': cleanBlock.fee_amt_percentiles[6],
+      };
+      cleanBlock.fee_rate_percentiles = {
+        'min': cleanBlock.fee_rate_percentiles[0],
+        'perc_10': cleanBlock.fee_rate_percentiles[1],
+        'perc_25': cleanBlock.fee_rate_percentiles[2],
+        'perc_50': cleanBlock.fee_rate_percentiles[3],
+        'perc_75': cleanBlock.fee_rate_percentiles[4],
+        'perc_90': cleanBlock.fee_rate_percentiles[5],
+        'max': cleanBlock.fee_rate_percentiles[6],
+      };
+
       // Re-org can happen after indexing so we need to always get the
       // latest state from core
       cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);

From 6c3a273e7588d53a495abfd3dccceaf0b9995ce7 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 14:24:47 +0900
Subject: [PATCH 18/23] Enabled coinstatsindex=1

---
 production/bitcoin.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/production/bitcoin.conf b/production/bitcoin.conf
index 46ab41b20..501f49f50 100644
--- a/production/bitcoin.conf
+++ b/production/bitcoin.conf
@@ -1,6 +1,7 @@
 datadir=/bitcoin
 server=1
 txindex=1
+coinstatsindex=1
 listen=1
 discover=1
 par=16

From 822362c10584cc2eb4bc376685bd19e50be3b4cb Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 14:30:59 +0900
Subject: [PATCH 19/23] Increase cache schema version

---
 backend/src/api/disk-cache.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts
index cf40d6952..a75fd43cc 100644
--- a/backend/src/api/disk-cache.ts
+++ b/backend/src/api/disk-cache.ts
@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 1;
+  private cacheSchemaVersion = 2;
 
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';

From ad4cbd60d5623dde7b8940065775dd3e24be79bc Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 14:40:28 +0900
Subject: [PATCH 20/23] Do not download orphaned block if `headers-only`

---
 backend/src/api/chain-tips.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts
index 92f148c8e..3384ebb19 100644
--- a/backend/src/api/chain-tips.ts
+++ b/backend/src/api/chain-tips.ts
@@ -24,7 +24,7 @@ class ChainTips {
       this.orphanedBlocks = [];
 
       for (const chain of this.chainTips) {
-        if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
           let block = await bitcoinClient.getBlock(chain.hash);
           while (block && block.confirmations === -1) {
             this.orphanedBlocks.push({

From 5d7c9f93153c501fe9d10080290510a877a146d7 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 15:06:47 +0900
Subject: [PATCH 21/23] Add config.MEMPOOOL.MAX_BLOCKS_BULK_QUERY parameter
 (default to 0, API disable)

---
 backend/src/__fixtures__/mempool-config.template.json |  3 ++-
 backend/src/__tests__/config.test.ts                  |  1 +
 backend/src/api/bitcoin/bitcoin.routes.ts             | 10 ++++++++--
 backend/src/config.ts                                 |  2 ++
 docker/README.md                                      |  2 ++
 docker/backend/mempool-config.json                    |  3 ++-
 docker/backend/start.sh                               |  2 ++
 7 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json
index 9d8a7e900..fa7ea7d21 100644
--- a/backend/src/__fixtures__/mempool-config.template.json
+++ b/backend/src/__fixtures__/mempool-config.template.json
@@ -28,7 +28,8 @@
     "AUDIT": "__MEMPOOL_AUDIT__",
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts
index 8b011d833..1e4c05ae3 100644
--- a/backend/src/__tests__/config.test.ts
+++ b/backend/src/__tests__/config.test.ts
@@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts
index 6d145e854..78d027663 100644
--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -409,21 +409,27 @@ class BitcoinRoutes {
       if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
         return res.status(404).send(`This API is only available for Bitcoin networks`);
       }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
       if (!Common.indexingEnabled()) {
         return res.status(404).send(`Indexing is required for this API`);
       }
 
       const from = parseInt(req.params.from, 10);
-      if (!from) {
+      if (!req.params.from || from < 0) {
         return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
       }
       const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
-      if (!to) {
+      if (to < 0) {
         return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
       }
       if (from > to) {
         return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
       }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
 
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(await blocks.$getBlocksBetweenHeight(from, to));
diff --git a/backend/src/config.ts b/backend/src/config.ts
index 2cda8d85b..ecd5c80aa 100644
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -32,6 +32,7 @@ interface IConfig {
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -153,6 +154,7 @@ const defaults: IConfig = {
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
diff --git a/docker/README.md b/docker/README.md
index 69bb96030..168d4b1fa 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -111,6 +111,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
+    "MAX_BLOCKS_BULK_QUERY": 0,
   },
 ```
 
@@ -141,6 +142,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_AUDIT: ""
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
+      MAX_BLOCKS_BULK_QUERY: ""
       ...
 ```
 
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json
index 904370f3e..d2aa75c69 100644
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@@ -25,7 +25,8 @@
     "AUDIT": __MEMPOOL_AUDIT__,
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
-    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__
+    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
diff --git a/docker/backend/start.sh b/docker/backend/start.sh
index 58b19898a..3ee542892 100755
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@@ -30,6 +30,7 @@ __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
+__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -142,6 +143,7 @@ sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

From 8d9568016ed63608719d720dcb66ee69da514599 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Fri, 24 Feb 2023 15:14:35 +0900
Subject: [PATCH 22/23] Remove duplicated entry in
 backend/src/__fixtures__/mempool-config.template.json

---
 backend/src/__fixtures__/mempool-config.template.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json
index fa7ea7d21..9890654a5 100644
--- a/backend/src/__fixtures__/mempool-config.template.json
+++ b/backend/src/__fixtures__/mempool-config.template.json
@@ -3,7 +3,6 @@
     "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
-    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,

From 210f939e653767a53e11ab4025ed088427da00f7 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 25 Feb 2023 13:59:37 +0900
Subject: [PATCH 23/23] Add missing truncate blocks table

---
 backend/src/api/database-migration.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 6e6e6855f..e732d15a5 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -486,6 +486,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion < 55) {
       await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
       await this.updateToSchemaVersion(55);
     }
   }