Merge branch 'master' into mononaut/seo-ssr

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.14.1", "19.6.1"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"
@@ -55,7 +55,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.14.1", "19.6.1"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"

.github/workflows/cypress.yml

@@ -1,8 +1,11 @@
 name: Cypress Tests
 
 on:
+  push:
+    branches: [master]
   pull_request:
-    types: [opened, review_requested, synchronize]
+    types: [opened, synchronize]
+
 jobs:
   cypress:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
   "editor.tabSize": 2,
+  "typescript.preferences.importModuleSpecifier": "relative",
   "typescript.tsdk": "./backend/node_modules/typescript/lib"
 }

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2022 The Mempool Open Source Project Developers
+Copyright (c) 2019-2023 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:

README.md

@@ -1,5 +1,7 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
+https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
+
 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.

backend/README.md

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
 Then, run the watcher:
 
 ```
-nodemon src/index.ts --ignore cache/ --ignore pools.json
+nodemon src/index.ts --ignore cache/
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -219,6 +219,16 @@ Generate block at regular interval (every 10 seconds in this example):
    watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
    ```
 
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+
 ### Re-index tables
 
 You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
@@ -235,4 +245,4 @@ Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables wi
 Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
 ```
 
-Reference: https://github.com/mempool/mempool/pull/1269
+Reference: https://github.com/mempool/mempool/pull/1269

backend/mempool-config.sample.json

@@ -22,7 +22,7 @@
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,

backend/src/__fixtures__/mempool-config.template.json

@@ -3,12 +3,11 @@
     "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
-    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
     "CLEAR_PROTECTION_MINUTES": 4,
@@ -28,7 +27,8 @@
     "AUDIT": "__MEMPOOL_AUDIT__",
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -36,11 +36,12 @@ describe('Mempool Backend Config', () => {
         USER_AGENT: 'mempool',
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
         AUDIT: false,
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/audit.ts

@@ -119,7 +119,8 @@ class Audit {
     }
 
     const numCensored = Object.keys(isCensored).length;
-    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
 
     return {
       censored: Object.keys(isCensored),

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -172,4 +172,35 @@ export namespace IBitcoinApi {
     }
   }
 
+  export interface BlockStats {
+    "avgfee": number;
+    "avgfeerate": number;
+    "avgtxsize": number;
+    "blockhash": string;
+    "feerate_percentiles": [number, number, number, number, number];
+    "height": number;
+    "ins": number;
+    "maxfee": number;
+    "maxfeerate": number;
+    "maxtxsize": number;
+    "medianfee": number;
+    "mediantime": number;
+    "mediantxsize": number;
+    "minfee": number;
+    "minfeerate": number;
+    "mintxsize": number;
+    "outs": number;
+    "subsidy": number;
+    "swtotal_size": number;
+    "swtotal_weight": number;
+    "swtxs": number;
+    "time": number;
+    "total_out": number;
+    "total_size": number;
+    "total_weight": number;
+    "totalfee": number;
+    "txs": number;
+    "utxo_increase": number;
+    "utxo_size_inc": number;
+  }
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       size: block.size,
       weight: block.weight,
       previousblockhash: block.previousblockhash,
+      mediantime: block.mediantime,
     };
   }
 

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -95,6 +95,8 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -215,7 +217,15 @@ class BitcoinRoutes {
       res.json(cpfpInfo);
       return;
     } else {
-      const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      let cpfpInfo;
+      if (config.DATABASE.ENABLED) {
+        cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      } else {
+        res.json({
+          ancestors: []
+        });
+        return;
+      }
       if (cpfpInfo) {
         res.json(cpfpInfo);
         return;
@@ -402,6 +412,41 @@ class BitcoinRoutes {
     }
   }
 
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -88,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    mediantime: number;
   }
 
   export interface Address {

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
 import loadingIndicators from './loading-indicators';
 import BitcoinApi from './bitcoin/bitcoin-api';
-import { prepareBlock } from '../utils/blocks-utils';
 import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
@@ -25,6 +24,7 @@ import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
+import chainTips from './chain-tips';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -142,7 +142,7 @@ class Blocks {
    * @param block
    * @returns BlockSummary
    */
-  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+  public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
     const stripped = block.tx.map((tx) => {
       return {
         txid: tx.txid,
@@ -165,33 +165,81 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = priceUpdater.latestPrices.USD;
+    const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    
+    const blk: Partial<BlockExtended> = Object.assign({}, block);
+    const extras: Partial<BlockExtension> = {};
+
+    extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
+    extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
 
     if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      extras.medianFee = 0; // 50th percentiles
+      extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      extras.totalFees = 0;
+      extras.avgFee = 0;
+      extras.avgFeeRate = 0;
+      extras.utxoSetChange = 0;
+      extras.avgTxSize = 0;
+      extras.totalInputs = 0;
+      extras.totalOutputs = 1;
+      extras.totalOutputAmt = 0;
+      extras.segwitTotalTxs = 0;
+      extras.segwitTotalSize = 0;
+      extras.segwitTotalWeight = 0;
     } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
+      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      extras.totalFees = stats.totalfee;
+      extras.avgFee = stats.avgfee;
+      extras.avgFeeRate = stats.avgfeerate;
+      extras.utxoSetChange = stats.utxo_increase;
+      extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      extras.totalInputs = stats.ins;
+      extras.totalOutputs = stats.outs;
+      extras.totalOutputAmt = stats.total_out;
+      extras.segwitTotalTxs = stats.swtxs;
+      extras.segwitTotalSize = stats.swtotal_size;
+      extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled()) {
+      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+      if (extras.feePercentiles !== null) {
+        extras.medianFeeAmt = extras.feePercentiles[3];
+      }
+    }
+  
+    extras.virtualSize = block.weight / 4.0;
+    if (coinbaseTx?.vout.length > 0) {
+      extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+      extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
+    } else {
+      extras.coinbaseAddress = null;
+      extras.coinbaseSignature = null;
+      extras.coinbaseSignatureAscii = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      extras.utxoSetSize = txoutset.txouts,
+      extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      extras.utxoSetSize = null;
+      extras.totalInputAmt = null;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      if (coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -201,25 +249,27 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
       } else {
-        blockExtended.extras.pool = {
-          id: pool.id,
+        extras.pool = {
+          id: pool.uniqueId,
           name: pool.name,
           slug: pool.slug,
         };
       }
 
+      extras.matchRate = null;
       if (config.MEMPOOL.AUDIT) {
         const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
         if (auditScore != null) {
-          blockExtended.extras.matchRate = auditScore.matchRate;
+          extras.matchRate = auditScore.matchRate;
         }
       }
     }
 
-    return blockExtended;
+    blk.extras = <BlockExtension>extras;
+    return <BlockExtended>blk;
   }
 
   /**
@@ -245,15 +295,18 @@ class Blocks {
     } else {
       pools = poolsParser.miningPools;
     }
+
     for (let i = 0; i < pools.length; ++i) {
       if (address !== undefined) {
-        const addresses: string[] = JSON.parse(pools[i].addresses);
+        const addresses: string[] = typeof pools[i].addresses === 'string' ?
+          JSON.parse(pools[i].addresses) : pools[i].addresses;
         if (addresses.indexOf(address) !== -1) {
           return pools[i];
         }
       }
 
-      const regexes: string[] = JSON.parse(pools[i].regexes);
+      const regexes: string[] = typeof pools[i].regexes === 'string' ?
+        JSON.parse(pools[i].regexes) : pools[i].regexes;
       for (let y = 0; y < regexes.length; ++y) {
         const regex = new RegExp(regexes[y], 'i');
         const match = asciiScriptSig.match(regex);
@@ -431,7 +484,7 @@ class Blocks {
             loadingIndicators.setProgress('block-indexing', progress, false);
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+          const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
           const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
           const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -479,13 +532,13 @@ class Blocks {
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+        const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
+          const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -500,6 +553,7 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        await chainTips.updateOrphanedBlocks();
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -516,18 +570,18 @@ class Blocks {
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
-          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
-            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
+          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
             // We assume there won't be a reorg with more than 10 block depth
-            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
-            await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
+            await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
             for (let i = 10; i >= 0; --i) {
-              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              const newBlock = await this.$indexBlock(lastBlock.height - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
               if (config.MEMPOOL.CPFP_INDEXING) {
-                await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
+                await this.$indexCPFP(newBlock.id, lastBlock.height - i);
               }
             }
             await mining.$indexDifficultyAdjustments();
@@ -603,12 +657,12 @@ class Blocks {
     if (Common.indexingEnabled()) {
       const dbBlock = await blocksRepository.$getBlockByHeight(height);
       if (dbBlock !== null) {
-        return prepareBlock(dbBlock);
+        return dbBlock;
       }
     }
 
     const blockHash = await bitcoinApi.$getBlockHash(height);
-    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
     const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -616,11 +670,11 @@ class Blocks {
       await blocksRepository.$saveBlockInDatabase(blockExtended);
     }
 
-    return prepareBlock(blockExtended);
+    return blockExtended;
   }
 
   /**
-   * Index a block by hash if it's missing from the database. Returns the block after indexing
+   * Get one block by its hash
    */
   public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
     // Check the memory cache
@@ -629,31 +683,14 @@ class Blocks {
       return blockByHash;
     }
 
-    // Block has already been indexed
-    if (Common.indexingEnabled()) {
-      const dbBlock = await blocksRepository.$getBlockByHash(hash);
-      if (dbBlock != null) {
-        return prepareBlock(dbBlock);
-      }
-    }
-
-    // Not Bitcoin network, return the block as it
+    // Not Bitcoin network, return the block as it from the bitcoin backend
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return await bitcoinApi.$getBlock(hash);
     }
 
-    let block = await bitcoinClient.getBlock(hash);
-    block = prepareBlock(block);
-
     // Bitcoin network, add our custom data on top
-    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
-    const blockExtended = await this.$getBlockExtended(block, transactions);
-    if (Common.indexingEnabled()) {
-      delete(blockExtended['coinbaseTx']);
-      await blocksRepository.$saveBlockInDatabase(blockExtended);
-    }
-
-    return blockExtended;
+    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
+    return await this.$indexBlock(block.height);
   }
 
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@@ -687,8 +724,19 @@ class Blocks {
     return summary.transactions;
   }
 
+  /**
+   * Get 15 blocks
+   * 
+   * Internally this function uses two methods to get the blocks, and
+   * the method is automatically selected:
+   *  - Using previous block hash links
+   *  - Using block height
+   * 
+   * @param fromHeight 
+   * @param limit 
+   * @returns 
+   */
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-
     let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
     if (currentHeight > this.currentBlockHeight) {
       limit -= currentHeight - this.currentBlockHeight;
@@ -700,27 +748,15 @@ class Blocks {
       return returnBlocks;
     }
 
-    // Check if block height exist in local cache to skip the hash lookup
-    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
-    let startFromHash: string | null = null;
-    if (blockByHeight) {
-      startFromHash = blockByHeight.id;
-    } else if (!Common.indexingEnabled()) {
-      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
-    }
-
-    let nextHash = startFromHash;
     for (let i = 0; i < limit && currentHeight >= 0; i++) {
       let block = this.getBlocks().find((b) => b.height === currentHeight);
       if (block) {
+        // Using the memory cache (find by height)
         returnBlocks.push(block);
-      } else if (Common.indexingEnabled()) {
+      } else {
+        // Using indexing (find by height, index on the fly, save in database)
         block = await this.$indexBlock(currentHeight);
         returnBlocks.push(block);
-      } else if (nextHash != null) {
-        block = await this.$indexBlock(currentHeight);
-        nextHash = block.previousblockhash;
-        returnBlocks.push(block);
       }
       currentHeight--;
     }
@@ -728,6 +764,114 @@ class Blocks {
     return returnBlocks;
   }
 
+  /**
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
+
+    const blocks: any[] = [];
+
+    while (fromHeight <= toHeight) {
+      let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
+      if (!block) {
+        await this.$indexBlock(fromHeight);
+        block = await blocksRepository.$getBlockByHeight(fromHeight);
+        if (!block) {
+          continue;
+        }
+      }
+
+      // Cleanup fields before sending the response
+      const cleanBlock: any = {
+        height: block.height ?? null,
+        hash: block.id ?? null,
+        timestamp: block.timestamp ?? null,
+        median_timestamp: block.mediantime ?? null,
+        previous_block_hash: block.previousblockhash ?? null,
+        difficulty: block.difficulty ?? null,
+        header: block.extras.header ?? null,
+        version: block.version ?? null,
+        bits: block.bits ?? null,
+        nonce: block.nonce ?? null,
+        size: block.size ?? null,
+        weight: block.weight ?? null,
+        tx_count: block.tx_count ?? null,
+        merkle_root: block.merkle_root ?? null,
+        reward: block.extras.reward ?? null,
+        total_fee_amt: block.extras.totalFees ?? null,
+        avg_fee_amt: block.extras.avgFee ?? null,
+        median_fee_amt: block.extras.medianFeeAmt ?? null,
+        fee_amt_percentiles: block.extras.feePercentiles ?? null,
+        avg_fee_rate: block.extras.avgFeeRate ?? null,
+        median_fee_rate: block.extras.medianFee ?? null,
+        fee_rate_percentiles: block.extras.feeRange ?? null,
+        total_inputs: block.extras.totalInputs ?? null,
+        total_input_amt: block.extras.totalInputAmt ?? null,
+        total_outputs: block.extras.totalOutputs ?? null,
+        total_output_amt: block.extras.totalOutputAmt ?? null,
+        segwit_total_txs: block.extras.segwitTotalTxs ?? null,
+        segwit_total_size: block.extras.segwitTotalSize ?? null,
+        segwit_total_weight: block.extras.segwitTotalWeight ?? null,
+        avg_tx_size: block.extras.avgTxSize ?? null,
+        utxoset_change: block.extras.utxoSetChange ?? null,
+        utxoset_size: block.extras.utxoSetSize ?? null,
+        coinbase_raw: block.extras.coinbaseRaw ?? null,
+        coinbase_address: block.extras.coinbaseAddress ?? null,
+        coinbase_signature: block.extras.coinbaseSignature ?? null,
+        coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
+        pool_slug: block.extras.pool.slug ?? null,
+        pool_id: block.extras.pool.id ?? null,
+      };
+
+      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
+        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        if (cleanBlock.fee_amt_percentiles === null) {
+          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+          const summary = this.summarizeBlock(block);
+          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        }
+        if (cleanBlock.fee_amt_percentiles !== null) {
+          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+        }
+      }
+
+      cleanBlock.fee_amt_percentiles = {
+        'min': cleanBlock.fee_amt_percentiles[0],
+        'perc_10': cleanBlock.fee_amt_percentiles[1],
+        'perc_25': cleanBlock.fee_amt_percentiles[2],
+        'perc_50': cleanBlock.fee_amt_percentiles[3],
+        'perc_75': cleanBlock.fee_amt_percentiles[4],
+        'perc_90': cleanBlock.fee_amt_percentiles[5],
+        'max': cleanBlock.fee_amt_percentiles[6],
+      };
+      cleanBlock.fee_rate_percentiles = {
+        'min': cleanBlock.fee_rate_percentiles[0],
+        'perc_10': cleanBlock.fee_rate_percentiles[1],
+        'perc_25': cleanBlock.fee_rate_percentiles[2],
+        'perc_50': cleanBlock.fee_rate_percentiles[3],
+        'perc_75': cleanBlock.fee_rate_percentiles[4],
+        'perc_90': cleanBlock.fee_rate_percentiles[5],
+        'max': cleanBlock.fee_rate_percentiles[6],
+      };
+
+      // Re-org can happen after indexing so we need to always get the
+      // latest state from core
+      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
+
+      blocks.push(cleanBlock);
+      fromHeight++;
+    }
+
+    return blocks;
+  }
+
   public async $getBlockAuditSummary(hash: string): Promise<any> {
     let summary;
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {

backend/src/api/chain-tips.ts

@@ -0,0 +1,61 @@
+import logger from '../logger';
+import bitcoinClient from './bitcoin/bitcoin-client';
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
+        }
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
+    if (height === undefined) {
+      return [];
+    }
+
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();

backend/src/api/common.ts

@@ -237,14 +237,21 @@ export class Common {
     ].join('x');
   }
 
-  static utcDateToMysql(date?: number): string {
+  static utcDateToMysql(date?: number | null): string | null {
+    if (date === null) {
+      return null;
+    }
     const d = new Date((date || 0) * 1000);
     return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
   }
 
   static findSocketNetwork(addr: string): {network: string | null, url: string} {
     let network: string | null = null;
-    let url = addr.split('://')[1];
+    let url: string = addr;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      url = addr.split('://')[1];
+    }
 
     if (!url) {
       return {
@@ -261,7 +268,7 @@ export class Common {
       }
     } else if (addr.indexOf('i2p') !== -1) {
       network = 'i2p';
-    } else if (addr.indexOf('ipv4') !== -1) {
+    } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
       const ipv = isIP(url.split(':')[0]);
       if (ipv === 4) {
         network = 'ipv4';
@@ -271,7 +278,7 @@ export class Common {
           url: addr,
         };
       }
-    } else if (addr.indexOf('ipv6') !== -1) {
+    } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
       url = url.split('[')[1].split(']')[0];
       const ipv = isIP(url);
       if (ipv === 6) {

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 54;
+  private static currentVersion = 57;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -62,8 +62,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion <= 2) {
       // Disable some spam logs when they're not relevant
-      this.uniqueLogs.push(this.blocksTruncatedMessage);
-      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
     }
 
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -483,6 +483,28 @@ class DatabaseMigration {
       }
       await this.updateToSchemaVersion(54);
     }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.updateToSchemaVersion(55);
+    }
+
+    if (databaseSchemaVersion < 56) {
+      await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
+      await this.$executeQuery('TRUNCATE TABLE `blocks`');
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('DELETE FROM `pools`');
+      await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      this.uniqueLog(logger.notice, '`pools` table has been truncated`');
+      await this.updateToSchemaVersion(56);
+    }
+
+    if (databaseSchemaVersion < 57 && isBitcoin === true) {
+      await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
+      await this.updateToSchemaVersion(57);
+    }
   }
 
   /**
@@ -756,6 +778,28 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD coinbase_signature_ascii varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,
@@ -973,26 +1017,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);
 
-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+}
 
   private async $convertCompactCpfpTables(): Promise<void> {
     try {

backend/src/api/disk-cache.ts

@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 1;
+  private cacheSchemaVersion = 3;
 
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
@@ -62,9 +62,24 @@ class DiskCache {
   }
 
   wipeCache() {
-    fs.unlinkSync(DiskCache.FILE_NAME);
+    logger.notice(`Wipping nodejs backend cache/cache*.json files`);
+    try {
+      fs.unlinkSync(DiskCache.FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+
     for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+      const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
+      try {
+        fs.unlinkSync(filename);
+      } catch (e: any) {
+        if (e?.code !== 'ENOENT') {
+          logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
+        }
+      }
     }
   }
 

backend/src/api/explorer/channels.api.ts

@@ -559,6 +559,17 @@ class ChannelsApi {
     const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
     const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
 
+    // https://github.com/mempool/mempool/issues/3006
+    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
+      channel.last_update = null;
+    }
+    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy1.last_update = null;
+    }
+    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy2.last_update = null;
+    }
+
     const query = `INSERT INTO channels
       (
         id,

backend/src/api/explorer/nodes.api.ts

@@ -228,7 +228,7 @@ class NodesApi {
             nodes.capacity
           FROM nodes
           ORDER BY capacity DESC
-          LIMIT 100
+          LIMIT 6
         `;
 
         [rows] = await DB.query(query);
@@ -269,14 +269,26 @@ class NodesApi {
       let query: string;
       if (full === false) {
         query = `
-          SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
-            nodes.channels
+          SELECT
+            nodes.public_key as publicKey,
+            IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
           FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
           ORDER BY channels DESC
-          LIMIT 100;
+          LIMIT 6;
         `;
 
         [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
       } else {
         query = `
           SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
@@ -630,6 +642,11 @@ class NodesApi {
    */
   public async $saveNode(node: ILightningApi.Node): Promise<void> {
     try {
+      // https://github.com/mempool/mempool/issues/3006
+      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
+        node.last_update = null;
+      }
+  
       const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
       const query = `INSERT INTO nodes(
           public_key,

backend/src/api/lightning/lightning-api.interface.ts

@@ -21,7 +21,7 @@ export namespace ILightningApi {
   export interface Channel {
     channel_id: string;
     chan_point: string;
-    last_update: number;
+    last_update: number | null;
     node1_pub: string;
     node2_pub: string;
     capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
     fee_rate_milli_msat: string;
     disabled: boolean;
     max_htlc_msat: string;
-    last_update: number;
+    last_update: number | null;
   }
 
   export interface Node {
-    last_update: number;
+    last_update: number | null;
     pub_key: string;
     alias: string;
     addresses: {

backend/src/api/mining/mining.ts

@@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
 import config from '../../config';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import PricesRepository from '../../repositories/PricesRepository';
+import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory';
+import { IEsploraApi } from '../bitcoin/esplora-api.interface';
 
 class Mining {
   blocksPriceIndexingRunning = false;
@@ -172,7 +174,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
@@ -189,8 +191,8 @@ class Mining {
     try {
       const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
-      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.time * 1000;
+      const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
 
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
@@ -279,7 +281,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
@@ -292,8 +294,8 @@ class Mining {
     const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
     try {
-      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.time * 1000;
+      const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
       const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
       let toTimestamp = Math.round(lastMidnight.getTime());
@@ -394,13 +396,13 @@ class Mining {
     }
 
     const blocks: any = await BlocksRepository.$getBlocksDifficulty();
-    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+    const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
     let currentDifficulty = genesisBlock.difficulty;
     let totalIndexed = 0;
 
     if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
       await DifficultyAdjustmentsRepository.$saveAdjustments({
-        time: genesisBlock.time,
+        time: genesisBlock.timestamp,
         height: 0,
         difficulty: currentDifficulty,
         adjustment: 0.0,
@@ -459,7 +461,7 @@ class Mining {
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -520,6 +522,41 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/pools-parser.ts

@@ -1,289 +1,165 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
-import BlocksRepository from '../repositories/BlocksRepository';
-
-interface Pool {
-  name: string;
-  link: string;
-  regexes: string[];
-  addresses: string[];
-  slug: string;
-}
+import PoolsRepository from '../repositories/PoolsRepository';
+import { PoolTag } from '../mempool.interfaces';
+import diskCache from './disk-cache';
 
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
+    'id': 0,
     'name': 'Unknown',
     'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
     'regexes': '[]',
     'addresses': '[]',
     'slug': 'unknown'
   };
-  slugWarnFlag = false;
+  private uniqueLogs: string[] = [];
+
+  private uniqueLog(loggerFunction: any, msg: string): void {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  public setMiningPools(pools): void {
+    for (const pool of pools) {
+      pool.regexes = pool.tags;
+      pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+      delete(pool.tags);
+    }
+    this.miningPools = pools;
+  }
 
   /**
-   * Parse the pools.json file, consolidate the data and dump it into the database
+   * Populate our db with updated mining pool definition
+   * @param pools 
    */
-  public async migratePoolsJson(poolsJson: object): Promise<void> {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return;
-    }
+  public async migratePoolsJson(): Promise<void> {
+    await this.$insertUnknownPool();
 
-    // First we save every entries without paying attention to pool duplication
-    const poolsDuplicated: Pool[] = [];
-
-    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
-    for (let i = 0; i < coinbaseTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>coinbaseTags[i][1]).name,
-        'link': (<Pool>coinbaseTags[i][1]).link,
-        'regexes': [coinbaseTags[i][0]],
-        'addresses': [],
-        'slug': ''
-      });
-    }
-    const addressesTags = Object.entries(poolsJson['payout_addresses']);
-    for (let i = 0; i < addressesTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>addressesTags[i][1]).name,
-        'link': (<Pool>addressesTags[i][1]).link,
-        'regexes': [],
-        'addresses': [addressesTags[i][0]],
-        'slug': ''
-      });
-    }
-
-    // Then, we find unique mining pool names
-    const poolNames: string[] = [];
-    for (let i = 0; i < poolsDuplicated.length; ++i) {
-      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
-        poolNames.push(poolsDuplicated[i].name);
+    for (const pool of this.miningPools) {
+      if (!pool.id) {
+        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
+        continue;
       }
-    }
-    logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
 
-    // Get existing pools from the db
-    let existingPools;
-    try {
-      if (config.DATABASE.ENABLED === true) {
-        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
+      if (!poolDB) {
+        // New mining pool
+        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.debug(`Inserting new mining pool ${pool.name}`);
+        await PoolsRepository.$insertNewMiningPool(pool, slug);
+        await this.$deleteUnknownBlocks();
       } else {
-        existingPools = [];
-      }
-    } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
-      return;
-    }
-
-    this.miningPools = [];
-
-    // Finally, we generate the final consolidated pools data
-    const finalPoolDataAdd: Pool[] = [];
-    const finalPoolDataUpdate: Pool[] = [];
-    const finalPoolDataRename: Pool[] = [];
-    for (let i = 0; i < poolNames.length; ++i) {
-      let allAddresses: string[] = [];
-      let allRegexes: string[] = [];
-      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
-
-      for (let y = 0; y < match.length; ++y) {
-        allAddresses = allAddresses.concat(match[y].addresses);
-        allRegexes = allRegexes.concat(match[y].regexes);
-      }
-
-      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
-
-      let slug: string | undefined;
-      try {
-        slug = poolsJson['slugs'][poolNames[i]];
-      } catch (e) {
-        if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
-          this.slugWarnFlag = true;
+        if (poolDB.name !== pool.name) {
+          // Pool has been renamed
+          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
+          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
         }
-      }
-
-      if (slug === undefined) {
-        // Only keep alphanumerical
-        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
-      }
-
-      const poolObj = {
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': allRegexes,
-        'addresses': allAddresses,
-        'slug': slug
-      };
-
-      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
-      if (existingPool !== undefined) {
-        // Check if any data was actually updated
-        const equals = (a, b) =>
-          a.length === b.length &&
-          a.every((v, i) => v === b[i]);
-        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
-          finalPoolDataUpdate.push(poolObj);
+        if (poolDB.link !== pool.link) {
+          // Pool link has changed
+          logger.debug(`Updating link for ${pool.name} mining pool`);
+          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
         }
-      } else if (config.DATABASE.ENABLED) {
-        // Double check that if we're not just renaming a pool (same address same regex)
-        const [poolToRename]: any[] = await DB.query(`
-          SELECT * FROM pools
-          WHERE addresses = ? OR regexes = ?`,
-          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
-        );
-        if (poolToRename && poolToRename.length > 0) {
-          // We're actually renaming an existing pool
-          finalPoolDataRename.push({
-            'name': poolObj.name,
-            'link': poolObj.link,
-            'regexes': allRegexes,
-            'addresses': allAddresses,
-            'slug': slug
-          });
-          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
-        } else {
-          logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
-          finalPoolDataAdd.push(poolObj);
+        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
+          JSON.stringify(pool.regexes) !== poolDB.regexes) {
+          // Pool addresses changed or coinbase tags changed
+          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
+          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
+          await this.$deleteBlocksForPool(poolDB);
         }
       }
-
-      this.miningPools.push({
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': JSON.stringify(allRegexes),
-        'addresses': JSON.stringify(allAddresses),
-        'slug': slug
-      });
-    }
-
-    if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
-      return;
-    }
-
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
-      finalPoolDataRename.length > 0
-    ) {    
-      logger.debug(`Update pools table now`, logger.tags.mining);
-
-      // Add new mining pools into the database
-      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-      }
-      queryAdd = queryAdd.slice(0, -1) + ';';
-
-      // Updated existing mining pools in the database
-      const updateQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-        updateQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-          slug='${finalPoolDataUpdate[i].slug}'
-          WHERE name='${finalPoolDataUpdate[i].name}'
-        ;`);
-      }
-
-      // Rename mining pools
-      const renameQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataRename.length; ++i) {
-        renameQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
-            slug='${finalPoolDataRename[i].slug}'
-          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
-            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
-        ;`);
-      }
-
-      try {
-        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
-          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
-        }
-
-        if (finalPoolDataAdd.length > 0) {
-          await DB.query({ sql: queryAdd, timeout: 120000 });
-        }
-        for (const query of updateQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        for (const query of renameQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        await this.insertUnknownPool();
-        logger.info('Mining pools.json import completed', logger.tags.mining);
-      } catch (e) {
-        logger.err(`Cannot import pools in the database`, logger.tags.mining);
-        throw e;
-      }
     }
 
-    try {
-      await this.insertUnknownPool();
-    } catch (e) {
-      logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
-      throw e;
-    }
+    logger.info('Mining pools-v2.json import completed');
   }
 
   /**
    * Manually add the 'unknown pool'
    */
-  private async insertUnknownPool() {
+  public async $insertUnknownPool(): Promise<void> {
+    if (!config.DATABASE.ENABLED) {
+      return;
+    }
+
     try {
       const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
       if (rows.length === 0) {
         await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
-          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
+          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
         `});
       } else {
         await DB.query(`UPDATE pools
-          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
           regexes='[]', addresses='[]',
-          slug='unknown'
-          WHERE name='Unknown'
+          slug='${this.unknownPool.slug}',
+          unique_id=0
+          WHERE slug='${this.unknownPool.slug}'
         `);
       }
     } catch (e) {
-      logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
+      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
     }
   }
 
   /**
-   * Delete blocks which needs to be reindexed
+   * Delete indexed blocks for an updated mining pool
+   * 
+   * @param pool 
    */
-   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
     if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
       return;
     }
 
-    const blockCount = await BlocksRepository.$blockCount(null, null);
-    if (blockCount === 0) {
-      return;
-    }
-
-    for (const updatedPool of finalPoolDataUpdate) {
-      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
-      if (pool.length > 0) {
-        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
-        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
-      }
-    }
-
-    // Ignore early days of Bitcoin as there were not mining pool yet
-    logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
+    // Ignore early days of Bitcoin as there were no mining pool yet
+    const [oldestPoolBlock]: any[] = await DB.query(`
+      SELECT height
+      FROM blocks
+      WHERE pool_id = ?
+      ORDER BY height
+      LIMIT 1`,
+      [pool.id]
+    );
+    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
     const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
+      [unknownPool[0].id]
+    );
+    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ?`,
+      [pool.id]
+    );
 
-    logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
-    await DB.query(`DELETE FROM hashrates`);
+    // We also need to wipe the backend cache to make sure we don't serve blocks with
+    // the wrong mining pool (usually happen with unknown blocks)
+    diskCache.wipeCache();
+  }
+
+  private async $deleteUnknownBlocks(): Promise<void> {
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= 130635`,
+      [unknownPool[0].id]
+    );
+
+    // We also need to wipe the backend cache to make sure we don't serve blocks with
+    // the wrong mining pool (usually happen with unknown blocks)
+    diskCache.wipeCache();
   }
 }
 

backend/src/api/transaction-utils.ts

@@ -14,6 +14,7 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)

backend/src/config.ts

@@ -32,6 +32,7 @@ interface IConfig {
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -147,12 +148,13 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
-    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'AUDIT': false,
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/database.ts

@@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
 
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
     }
   }
 

backend/src/index.ts

@@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
+import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';
 
 class Server {
@@ -82,11 +83,8 @@ class Server {
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
       try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
         }
         await databaseMigration.$initializeOrMigrateDatabase();
         if (Common.indexingEnabled()) {
@@ -133,6 +131,7 @@ class Server {
     }
 
     priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
 
@@ -180,7 +179,14 @@ class Server {
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
     } catch (e: any) {
-      const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
+      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
+      if (e?.stack) {
+        loggerMsg += ` Stack trace: ${e.stack}`;
+      }
+      // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
+      // From the second Exception, `logger.warn` the Exception and increase the retry delay
+      // Maximum retry delay is 60 seconds
       if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
@@ -200,8 +206,8 @@ class Server {
     try {
       await fundingTxFetcher.$init();
       await networkSyncService.$startService();
-      await forensicsService.$startService();
       await lightningStatsUpdater.$startService();
+      await forensicsService.$startService();
     } catch(e) {
       logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
       await Common.sleep$(1000 * 60);

backend/src/indexer.ts

@@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
   tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
 
-  public reindex() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async runSingleTask(task: 'blocksPrices') {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
     if (!Common.indexingEnabled()) {
       return;
     }
@@ -28,20 +77,27 @@ class Indexer {
       this.tasksRunning.push(task);
       const lastestPriceId = await PricesRepository.$getLatestPriceId();
       if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
         setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
           this.runSingleTask('blocksPrices');
         }, 10000);
       } else {
-        logger.debug(`Blocks prices indexer will run now`)
+        logger.debug(`Blocks prices indexer will run now`);
         await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
       }
     }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
   }
 
-  public async $run() {
+  public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -57,7 +113,9 @@ class Indexer {
     this.runIndexer = false;
     this.indexerRunning = true;
 
-    logger.debug(`Running mining indexer`);
+    logger.info(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();
 
     try {
       await priceUpdater.$run();
@@ -93,7 +151,7 @@ class Indexer {
     setTimeout(() => this.reindex(), runEvery);
   }
 
-  async $resetHashratesIndexingState() {
+  async $resetHashratesIndexingState(): Promise<void> {
     try {
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);

backend/src/mempool.interfaces.ts

@@ -1,8 +1,10 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
-import { HeapNode } from "./utils/pairing-heap";
+import { OrphanedBlock } from './api/chain-tips';
+import { HeapNode } from './utils/pairing-heap';
 
 export interface PoolTag {
-  id: number; // mysql row id
+  id: number;
+  uniqueId: number;
   name: string;
   link: string;
   regexes: string; // JSON array
@@ -64,6 +66,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -145,23 +148,44 @@ export interface TransactionStripped {
 }
 
 export interface BlockExtension {
-  totalFees?: number;
-  medianFee?: number;
-  feeRange?: number[];
-  reward?: number;
-  coinbaseTx?: TransactionMinerInfo;
-  matchRate?: number;
-  pool?: {
-    id: number;
+  totalFees: number;
+  medianFee: number; // median fee rate
+  feeRange: number[]; // fee rate percentiles
+  reward: number;
+  matchRate: number | null;
+  pool: {
+    id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
     name: string;
     slug: string;
   };
-  avgFee?: number;
-  avgFeeRate?: number;
-  coinbaseRaw?: string;
-  usd?: number | null;
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  orphans: OrphanedBlock[] | null;
+  coinbaseAddress: string | null;
+  coinbaseSignature: string | null;
+  coinbaseSignatureAscii: string | null;
+  virtualSize: number;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number | null; // median fee in sats
+  feePercentiles: number[] | null, // fee percentiles in sats
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize: number | null;
+  totalInputAmt: number | null;
 }
 
+/**
+ * Note: Everything that is added in here will be automatically returned through
+ * /api/v1/block and /api/v1/blocks APIs
+ */
 export interface BlockExtended extends IEsploraApi.Block {
   extras: BlockExtension;
 }

backend/src/repositories/BlocksRepository.ts

@@ -1,8 +1,7 @@
-import { BlockExtended, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
-import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
@@ -10,27 +9,90 @@ import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
 import bitcoinClient from '../api/bitcoin/bitcoin-client';
 import config from '../config';
+import chainTips from '../api/chain-tips';
+import blocks from '../api/blocks';
+import BlocksAuditsRepository from './BlocksAuditsRepository';
+
+const BLOCK_DB_FIELDS = `
+  blocks.hash AS id,
+  blocks.height,
+  blocks.version,
+  UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
+  blocks.bits,
+  blocks.nonce,
+  blocks.difficulty,
+  blocks.merkle_root,
+  blocks.tx_count,
+  blocks.size,
+  blocks.weight,
+  blocks.previous_block_hash AS previousblockhash,
+  UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
+  blocks.fees AS totalFees,
+  blocks.median_fee AS medianFee,
+  blocks.fee_span AS feeRange,
+  blocks.reward,
+  pools.unique_id AS poolId,
+  pools.name AS poolName,
+  pools.slug AS poolSlug,
+  blocks.avg_fee AS avgFee,
+  blocks.avg_fee_rate AS avgFeeRate,
+  blocks.coinbase_raw AS coinbaseRaw,
+  blocks.coinbase_address AS coinbaseAddress,
+  blocks.coinbase_signature AS coinbaseSignature,
+  blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
+  blocks.avg_tx_size AS avgTxSize,
+  blocks.total_inputs AS totalInputs,
+  blocks.total_outputs AS totalOutputs,
+  blocks.total_output_amt AS totalOutputAmt,
+  blocks.median_fee_amt AS medianFeeAmt,
+  blocks.fee_percentiles AS feePercentiles,
+  blocks.segwit_total_txs AS segwitTotalTxs,
+  blocks.segwit_total_size AS segwitTotalSize,
+  blocks.segwit_total_weight AS segwitTotalWeight,
+  blocks.header,
+  blocks.utxoset_change AS utxoSetChange,
+  blocks.utxoset_size AS utxoSetSize,
+  blocks.total_input_amt AS totalInputAmts
+`;
 
 class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
   public async $saveBlockInDatabase(block: BlockExtended) {
+    const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
+    const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
+
     try {
       const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,             hash,                blockTimestamp,    size,
+        weight,             tx_count,            coinbase_raw,      difficulty,
+        pool_id,            fees,                fee_span,          median_fee,
+        reward,             version,             bits,              nonce,
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   header,              coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt,     coinbase_signature_ascii
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?
       )`;
 
+      const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
+      if (!poolDbId) {
+        throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
+      }
+
       const params: any[] = [
         block.height,
         block.id,
@@ -40,7 +102,7 @@ class BlocksRepository {
         block.tx_count,
         block.extras.coinbaseRaw,
         block.difficulty,
-        block.extras.pool?.id, // Should always be set to something
+        poolDbId.id,
         block.extras.totalFees,
         JSON.stringify(block.extras.feeRange),
         block.extras.medianFee,
@@ -52,19 +114,63 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
+        block.mediantime,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        truncatedCoinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
+        truncatedCoinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
+        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
       } else {
-        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
         throw e;
       }
     }
   }
 
+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -250,34 +356,17 @@ class BlocksRepository {
   /**
    * Get blocks mined by a specific mining pool
    */
-  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
+  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
     const pool = await PoolsRepository.$getPool(slug);
     if (!pool) {
       throw new Error('This mining pool does not exist ' + escape(slug));
     }
 
     const params: any[] = [];
-    let query = ` SELECT
-      blocks.height,
-      hash as id,
-      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-      size,
-      weight,
-      tx_count,
-      coinbase_raw,
-      difficulty,
-      fees,
-      fee_span,
-      median_fee,
-      reward,
-      version,
-      bits,
-      nonce,
-      merkle_root,
-      previous_block_hash as previousblockhash,
-      avg_fee,
-      avg_fee_rate
+    let query = `
+      SELECT ${BLOCK_DB_FIELDS}
       FROM blocks
+      JOIN pools ON blocks.pool_id = pools.id
       WHERE pool_id = ?`;
     params.push(pool.id);
 
@@ -290,11 +379,11 @@ class BlocksRepository {
       LIMIT 10`;
 
     try {
-      const [rows] = await DB.query(query, params);
+      const [rows]: any[] = await DB.query(query, params);
 
       const blocks: BlockExtended[] = [];
-      for (const block of <object[]>rows) {
-        blocks.push(prepareBlock(block));
+      for (const block of rows) {
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
       }
 
       return blocks;
@@ -307,46 +396,21 @@ class BlocksRepository {
   /**
    * Get one block by height
    */
-  public async $getBlockByHeight(height: number): Promise<object | null> {
+  public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
     try {
-      const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
-        hash,
-        hash as id,
-        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
-        pools.id as pool_id,
-        pools.name as pool_name,
-        pools.link as pool_link,
-        pools.slug as pool_slug,
-        pools.addresses as pool_addresses,
-        pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
+      const [rows]: any[] = await DB.query(`
+        SELECT ${BLOCK_DB_FIELDS}
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE blocks.height = ${height}
-      `);
+        WHERE blocks.height = ?`,
+        [height]
+      );
 
       if (rows.length <= 0) {
         return null;
       }
 
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -359,10 +423,7 @@ class BlocksRepository {
   public async $getBlockByHash(hash: string): Promise<object | null> {
     try {
       const query = `
-        SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
-        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
-        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
-        previous_block_hash as previousblockhash
+        SELECT ${BLOCK_DB_FIELDS}
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE hash = ?;
@@ -372,9 +433,8 @@ class BlocksRepository {
       if (rows.length <= 0) {
         return null;
       }
-
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
+ 
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
     } catch (e) {
       logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -465,8 +525,15 @@ class BlocksRepository {
   public async $validateChain(): Promise<boolean> {
     try {
       const start = new Date().getTime();
-      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
+      const [blocks]: any[] = await DB.query(`
+        SELECT
+          height,
+          hash,
+          previous_block_hash,
+          UNIX_TIMESTAMP(blockTimestamp) AS timestamp
+        FROM blocks
+        ORDER BY height
+      `);
 
       let partialMsg = false;
       let idx = 1;
@@ -694,7 +761,6 @@ class BlocksRepository {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
-    return [];
   }
 
   /**
@@ -741,7 +807,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -754,6 +820,132 @@ class BlocksRepository {
       }
     }
   }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Convert a mysql row block into a BlockExtended. Note that you
+   * must provide the correct field into dbBlk object param
+   * 
+   * @param dbBlk 
+   */
+  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
+    const blk: Partial<BlockExtended> = {};
+    const extras: Partial<BlockExtension> = {};
+
+    // IEsploraApi.Block
+    blk.id = dbBlk.id;
+    blk.height = dbBlk.height;
+    blk.version = dbBlk.version;
+    blk.timestamp = dbBlk.timestamp;
+    blk.bits = dbBlk.bits;
+    blk.nonce = dbBlk.nonce;
+    blk.difficulty = dbBlk.difficulty;
+    blk.merkle_root = dbBlk.merkle_root;
+    blk.tx_count = dbBlk.tx_count;
+    blk.size = dbBlk.size;
+    blk.weight = dbBlk.weight;
+    blk.previousblockhash = dbBlk.previousblockhash;
+    blk.mediantime = dbBlk.mediantime;
+    
+    // BlockExtension
+    extras.totalFees = dbBlk.totalFees;
+    extras.medianFee = dbBlk.medianFee;
+    extras.feeRange = JSON.parse(dbBlk.feeRange);
+    extras.reward = dbBlk.reward;
+    extras.pool = {
+      id: dbBlk.poolId,
+      name: dbBlk.poolName,
+      slug: dbBlk.poolSlug,
+    };
+    extras.avgFee = dbBlk.avgFee;
+    extras.avgFeeRate = dbBlk.avgFeeRate;
+    extras.coinbaseRaw = dbBlk.coinbaseRaw;
+    extras.coinbaseAddress = dbBlk.coinbaseAddress;
+    extras.coinbaseSignature = dbBlk.coinbaseSignature;
+    extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
+    extras.avgTxSize = dbBlk.avgTxSize;
+    extras.totalInputs = dbBlk.totalInputs;
+    extras.totalOutputs = dbBlk.totalOutputs;
+    extras.totalOutputAmt = dbBlk.totalOutputAmt;
+    extras.medianFeeAmt = dbBlk.medianFeeAmt;
+    extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
+    extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
+    extras.segwitTotalSize = dbBlk.segwitTotalSize;
+    extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
+    extras.header = dbBlk.header,
+    extras.utxoSetChange = dbBlk.utxoSetChange;
+    extras.utxoSetSize = dbBlk.utxoSetSize;
+    extras.totalInputAmt = dbBlk.totalInputAmt;
+    extras.virtualSize = dbBlk.weight / 4.0;
+
+    // Re-org can happen after indexing so we need to always get the
+    // latest state from core
+    extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
+
+    // Match rate is not part of the blocks table, but it is part of APIs so we must include it
+    extras.matchRate = null;
+    if (config.MEMPOOL.AUDIT) {
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
+      if (auditScore != null) {
+        extras.matchRate = auditScore.matchRate;
+      }
+    }
+
+    // If we're missing block summary related field, check if we can populate them on the fly now
+    if (Common.blocksSummariesIndexingEnabled() &&
+      (extras.medianFeeAmt === null || extras.feePercentiles === null))
+    {
+      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
+      if (extras.feePercentiles === null) {
+        const block = await bitcoinClient.getBlock(dbBlk.id, 2);
+        const summary = blocks.summarizeBlock(block);
+        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
+      }
+      if (extras.feePercentiles !== null) {
+        extras.medianFeeAmt = extras.feePercentiles[3];
+      }
+    }
+
+    blk.extras = <BlockExtension>extras;
+    return <BlockExtended>blk;
+  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -80,6 +80,48 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/repositories/PoolsRepository.ts

@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
@@ -9,7 +10,7 @@ class PoolsRepository {
    * Get all pools tagging info
    */
   public async $getPools(): Promise<PoolTag[]> {
-    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
+    const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
     return <PoolTag[]>rows;
   }
 
@@ -17,7 +18,11 @@ class PoolsRepository {
    * Get unknown pool tagging info
    */
   public async $getUnknownPool(): Promise<PoolTag> {
-    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
+    if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
+      await poolsParser.$insertUnknownPool();
+      [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
+    }
     return <PoolTag>rows[0];
   }
 
@@ -59,7 +64,7 @@ class PoolsRepository {
   /**
    * Get basic pool info and block count between two timestamp
    */
-   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+  public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
     const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
       FROM pools
       LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -75,9 +80,9 @@ class PoolsRepository {
   }
 
   /**
-   * Get mining pool statistics for one pool
+   * Get a mining pool info
    */
-   public async $getPool(slug: string): Promise<PoolTag | null> {
+  public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
     const query = `
       SELECT *
       FROM pools
@@ -90,10 +95,12 @@ class PoolsRepository {
         return null;
       }
 
-      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
       if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else {
+        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
+      } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);
       }
 
@@ -103,6 +110,116 @@ class PoolsRepository {
       throw e;
     }
   }
+
+  /**
+   * Get a mining pool info by its unique id
+   */
+  public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.unique_id = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [id]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else if (parse) {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Insert a new mining pool in the database
+   * 
+   * @param pool 
+   */
+  public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO pools
+        SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
+        [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Rename an existing mining pool
+   * 
+   * @param dbId
+   * @param newSlug
+   * @param newName 
+   */
+  public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET slug = ?, name = ?
+        WHERE id = ?`,
+        [newSlug, newName, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update an exisiting mining pool link
+   * 
+   * @param dbId 
+   * @param newLink 
+   */
+  public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET link = ?
+        WHERE id = ?`,
+        [newLink, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+  }
+
+  /**
+   * Update an existing mining pool addresses or coinbase tags
+   * 
+   * @param dbId 
+   * @param addresses 
+   * @param regexes 
+   */
+  public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET addresses = ?, regexes = ?
+        WHERE id = ?`,
+        [JSON.stringify(addresses), JSON.stringify(regexes), dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
 }
 
 export default new PoolsRepository();

backend/src/repositories/PricesRepository.ts

@@ -40,7 +40,7 @@ export const MAX_PRICES = {
 
 class PricesRepository {
   public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
-    if (prices.USD === 0) {
+    if (prices.USD === -1) {
       // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
       // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
       return;

backend/src/rpc-api/commands.ts

@@ -88,5 +88,7 @@ module.exports = {
   verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
-}
+  walletPassphraseChange: 'walletpassphrasechange',
+  getTxoutSetinfo: 'gettxoutsetinfo',
+  getIndexInfo: 'getindexinfo',
+};

backend/src/tasks/lightning/network-sync.service.ts

@@ -72,7 +72,7 @@ class NetworkSyncService {
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
-      node.last_update = Math.max(node.last_update, latestUpdated);
+      node.last_update = Math.max(node.last_update ?? 0, latestUpdated);
 
       await nodesApi.$saveNode(node);
       graphNodesPubkeys.push(node.pub_key);

backend/src/tasks/pools-updater.ts

@@ -8,7 +8,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
 
 /**
- * Maintain the most recent version of pools.json
+ * Maintain the most recent version of pools-v2.json
  */
 class PoolsUpdater {
   lastRun: number = 0;
@@ -31,14 +31,8 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
-    } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
-    }
-
     try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
       if (githubSha === undefined) {
         return;
       }
@@ -47,32 +41,57 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
       if (this.currentSha !== undefined && this.currentSha === githubSha) {
         return;
       }
 
+      // See backend README for more details about the mining pools update process
+      if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
+        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        !process.env.npm_config_update_pools // We're not manually updating mining pool
+      ) {
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        return;
+      }
+
+      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
       if (this.currentSha === undefined) {
-        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
-      await poolsParser.migratePoolsJson(poolsJson);
-      await this.updateDBSha(githubSha);
-      logger.notice(`PoolsUpdater completed`, logger.tags.mining);
+      poolsParser.setMiningPools(poolsJson);
+
+      if (config.DATABASE.ENABLED === false) { // Don't run db operations
+        logger.info('Mining pools-v2.json import completed (no database)');
+        return;
+      }
+
+      try {
+        await DB.query('START TRANSACTION;');
+        await poolsParser.migratePoolsJson();
+        await this.updateDBSha(githubSha);
+        await DB.query('COMMIT;');
+      } catch (e) {
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        await DB.query('ROLLBACK;');
+      }
+      logger.notice('PoolsUpdater completed');
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
   private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
@@ -81,46 +100,46 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       }
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
   private async getShaFromDb(): Promise<string | undefined> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       return undefined;
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from github
+   * Fetch our latest pools-v2.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
     const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
-        if (file['path'] === 'pools.json') {
+        if (file['path'] === 'pools-v2.json') {
           return file['sha'];
         }
       }
     }
 
-    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
     return undefined;
   }
 
   /**
    * Http request wrapper
    */
-  private async query(path): Promise<object | undefined> {
+  private async query(path): Promise<any[] | undefined> {
     type axiosOptions = {
       headers: {
         'User-Agent': string

backend/src/utils/blocks-utils.ts

@@ -1,33 +0,0 @@
-import { BlockExtended } from '../mempool.interfaces';
-
-export function prepareBlock(block: any): BlockExtended {
-  return <BlockExtended>{
-    id: block.id ?? block.hash, // hash for indexed block
-    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
-    height: block.height,
-    version: block.version,
-    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
-    nonce: block.nonce,
-    difficulty: block.difficulty,
-    merkle_root: block.merkle_root ?? block.merkleroot,
-    tx_count: block.tx_count ?? block.nTx,
-    size: block.size,
-    weight: block.weight,
-    previousblockhash: block.previousblockhash,
-    extras: {
-      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
-      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
-      feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span,
-      reward: block.reward ?? block?.extras?.reward,
-      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
-      avgFee: block?.extras?.avgFee ?? block.avg_fee,
-      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
-      pool: block?.extras?.pool ?? (block?.pool_id ? {
-        id: block.pool_id,
-        name: block.pool_name,
-        slug: block.pool_slug,
-      } : undefined),
-      usd: block?.extras?.usd ?? block.usd ?? null,
-    }
-  };
-}

docker/README.md

@@ -102,15 +102,16 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "BLOCKS_SUMMARIES_INDEXING": false,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
+    "EXTERNAL_ASSETS": [],
     "STDOUT_LOG_MIN_PRIORITY": "info",
     "INDEXING_BLOCKS_AMOUNT": false,
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
+    "MAX_BLOCKS_BULK_QUERY": 0,
   },
 ```
 
@@ -141,6 +142,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_AUDIT: ""
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
+      MAX_BLOCKS_BULK_QUERY: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -25,7 +25,8 @@
     "AUDIT": __MEMPOOL_AUDIT__,
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
-    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__
+    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -24,12 +24,13 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
-__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
+__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -142,6 +143,7 @@ sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

docker/frontend/entrypoint.sh

@@ -35,6 +35,7 @@ __AUDIT__=${AUDIT:=false}
 __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
+__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 
 # Export as environment variables to be used by envsubst
 export __TESTNET_ENABLED__
@@ -60,6 +61,7 @@ export __AUDIT__
 export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
 export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
 export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
+export __HISTORICAL_PRICE__
 
 folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
 echo ${folder}

frontend/.gitignore

@@ -54,6 +54,7 @@ src/resources/assets-testnet.json
 src/resources/assets-testnet.minimal.json
 src/resources/pools.json
 src/resources/mining-pools/*
+src/resources/*.mp4
 
 # environment config
 mempool-frontend-config.json

frontend/mempool-frontend-config.sample.json

@@ -21,5 +21,6 @@
   "MAINNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
-  "LIGHTNING": false
+  "LIGHTNING": false,
+  "HISTORICAL_PRICE": true
 }

frontend/src/app/components/about/about.component.html

@@ -13,19 +13,9 @@
     <p i18n>Our mempool and blockchain explorer for the Bitcoin community, focusing on the transaction fee market and multi-layer ecosystem, completely self-hosted without any trusted third-parties.</p>
   </div>
 
-  <div class="social-icons">
-    <a target="_blank" href="https://github.com/mempool/mempool">
-      <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" class="svg-inline--fa fa-github fa-w-16 fa-4x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg>
-    </a>
-    <a target="_blank" href="https://twitter.com/mempool">
-      <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="twitter" class="svg-inline--fa fa-twitter fa-w-16 fa-4x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"></path></svg>
-    </a>
-    <a target="_blank" href="https://matrix.to/#/#mempool:bitcoin.kyoto">
-      <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="matrix" class="svg-inline--fa fa-matrix fa-w-16 fa-4x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1536 1792"><path fill="currentColor" d="M40.467 163.152v1465.696H145.92V1664H0V128h145.92v35.152zm450.757 464.64v74.14h2.069c19.79-28.356 43.717-50.215 71.483-65.575 27.765-15.656 59.963-23.336 96-23.336 34.56 0 66.165 6.795 94.818 20.086 28.652 13.293 50.216 37.22 65.28 70.893 16.246-23.926 38.4-45.194 66.166-63.507 27.766-18.314 60.848-27.472 98.954-27.472 28.948 0 55.828 3.545 80.64 10.635 24.812 7.088 45.785 18.314 63.508 33.968 17.722 15.656 31.31 35.742 41.354 60.85 9.747 25.107 14.768 55.236 14.768 90.683v366.573h-150.35V865.28c0-18.314-.59-35.741-2.068-51.987-1.476-16.247-5.316-30.426-11.52-42.24-6.499-12.112-15.656-21.563-28.062-28.653-12.405-7.088-29.242-10.634-50.214-10.634-21.268 0-38.4 4.135-51.397 12.112-12.997 8.27-23.336 18.608-30.72 31.901-7.386 12.997-12.407 27.765-14.77 44.602-2.363 16.542-3.84 33.379-3.84 50.216v305.133H692.971v-307.2c0-16.247-.294-32.197-1.18-48.149-.591-15.95-3.84-30.424-9.157-44.011-5.317-13.293-14.178-24.223-26.585-32.197-12.406-7.976-30.425-12.112-54.646-12.112-7.088 0-16.542 1.478-28.062 4.726-11.52 3.25-23.04 9.157-33.968 18.02-10.93 8.86-20.383 21.563-28.063 38.103-7.68 16.543-11.52 38.4-11.52 65.28v317.834H349.44V627.792zm1004.309 1001.056V163.152H1390.08V128H1536v1536h-145.92v-35.152z"/></svg>
-    </a>
-  </div>
+  <video src="/resources/mempool-promo.mp4" poster="/resources/mempool-promo.jpg" controls loop playsinline [autoplay]="true" [muted]="true"></video>
 
-  <div class="enterprise-sponsor">
+  <div class="enterprise-sponsor" id="enterprise-sponsors">
     <h3 i18n="about.sponsors.enterprise.withRocket">Enterprise Sponsors 🚀</h3>
     <div class="wrapper">
       <a href="https://spiral.xyz/" target="_blank" title="Spiral">
@@ -173,7 +163,7 @@
     </div>
   </div>
 
-  <div class="community-sponsor">
+  <div class="community-sponsor" id="community-sponsors">
     <h3 i18n="about.sponsors.withHeart">Community Sponsors ❤️</h3>
 
     <div class="wrapper">
@@ -187,7 +177,7 @@
     </div>
   </div>
 
-  <div class="community-integrations-sponsor">
+  <div class="community-integrations-sponsor" id="community-integrations">
     <h3 i18n="about.community-integrations">Community Integrations</h3>
     <div class="wrapper">
       <a href="https://github.com/getumbrel/umbrel" target="_blank" title="Umbrel">
@@ -281,7 +271,7 @@
     </div>
   </div>
 
-  <div class="alliances">
+  <div class="alliances" id="community-alliances">
     <h3 i18n="about.alliances">Community Alliances</h3>
     <div class="wrapper">
       <a href="https://liquid.net/" title="Liquid Network">
@@ -297,7 +287,7 @@
   </div>
 
   <ng-container *ngIf="translators$ | async | keyvalue as translators else loadingSponsors">
-    <div class="project-translators">
+    <div class="project-translators" id="project-translators">
       <h3 i18n="about.translators">Project Translators</h3>
       <div class="wrapper">
         <ng-template ngFor let-translator [ngForOf]="translators">
@@ -311,7 +301,7 @@
   </ng-container>
 
   <ng-container *ngIf="allContributors$ | async as contributors else loadingSponsors">
-    <div class="contributors">
+    <div class="contributors" id="project-contributors">
       <h3 i18n="about.contributors">Project Contributors</h3>
       <div class="wrapper">
         <ng-template ngFor let-contributor [ngForOf]="contributors.regular">
@@ -323,7 +313,7 @@
       </div>
     </div>
 
-    <div class="maintainers" *ngIf="contributors.core.length">
+    <div class="maintainers" *ngIf="contributors.core.length" id="project-members">
       <h3 i18n="about.project_members">Project Members</h3>
       <div class="wrapper">
         <ng-template ngFor let-contributor [ngForOf]="contributors.core">
@@ -336,7 +326,7 @@
     </div>
   </ng-container>
 
-  <div class="maintainers">
+  <div class="maintainers" id="project-maintainers">
     <h3 i18n="about.maintainers">Project Maintainers</h3>
     <div class="wrapper">
         <a href="https://twitter.com/softsimon_" target="_blank" title="softsimon">
@@ -352,7 +342,7 @@
 
   <div class="copyright">
     <div class="title">
-      Copyright &copy; 2019-2022<br>
+      Copyright &copy; 2019-2023<br>
       The Mempool Open Source Project
     </div>
     <p>
@@ -383,6 +373,17 @@
   <div class="footer-links">
     <a href="/3rdpartylicenses.txt">Third-party Licenses</a>
     <a [routerLink]="['/terms-of-service']" i18n="shared.terms-of-service|Terms of Service">Terms of Service</a>
+    <div class="social-icons">
+      <a target="_blank" href="https://github.com/mempool/mempool">
+        <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" class="svg-inline--fa fa-github fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg>
+      </a>
+      <a target="_blank" href="https://twitter.com/mempool">
+        <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="twitter" class="svg-inline--fa fa-twitter fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"></path></svg>
+      </a>
+      <a target="_blank" href="https://matrix.to/#/#mempool:bitcoin.kyoto">
+        <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="matrix" class="svg-inline--fa fa-matrix fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1536 1792"><path fill="currentColor" d="M40.467 163.152v1465.696H145.92V1664H0V128h145.92v35.152zm450.757 464.64v74.14h2.069c19.79-28.356 43.717-50.215 71.483-65.575 27.765-15.656 59.963-23.336 96-23.336 34.56 0 66.165 6.795 94.818 20.086 28.652 13.293 50.216 37.22 65.28 70.893 16.246-23.926 38.4-45.194 66.166-63.507 27.766-18.314 60.848-27.472 98.954-27.472 28.948 0 55.828 3.545 80.64 10.635 24.812 7.088 45.785 18.314 63.508 33.968 17.722 15.656 31.31 35.742 41.354 60.85 9.747 25.107 14.768 55.236 14.768 90.683v366.573h-150.35V865.28c0-18.314-.59-35.741-2.068-51.987-1.476-16.247-5.316-30.426-11.52-42.24-6.499-12.112-15.656-21.563-28.062-28.653-12.405-7.088-29.242-10.634-50.214-10.634-21.268 0-38.4 4.135-51.397 12.112-12.997 8.27-23.336 18.608-30.72 31.901-7.386 12.997-12.407 27.765-14.77 44.602-2.363 16.542-3.84 33.379-3.84 50.216v305.133H692.971v-307.2c0-16.247-.294-32.197-1.18-48.149-.591-15.95-3.84-30.424-9.157-44.011-5.317-13.293-14.178-24.223-26.585-32.197-12.406-7.976-30.425-12.112-54.646-12.112-7.088 0-16.542 1.478-28.062 4.726-11.52 3.25-23.04 9.157-33.968 18.02-10.93 8.86-20.383 21.563-28.063 38.103-7.68 16.543-11.52 38.4-11.52 65.28v317.834H349.44V627.792zm1004.309 1001.056V163.152H1390.08V128H1536v1536h-145.92v-35.152z"/></svg>
+      </a>
+    </div>
   </div>
 
   <div class="footer-version" *ngIf="officialMempoolSpace">

frontend/src/app/components/about/about.component.scss

@@ -34,6 +34,13 @@
     padding: 10px 15px 15px;
   }
 
+  video {
+    width: 640px;
+    height: 360px;
+    max-width: 90%;
+    margin-top: 0;
+  }
+
   .social-icons {
     a {
       margin: auto 10px;
@@ -46,6 +53,7 @@
   .maintainers {
     margin-top: 68px;
     margin-bottom: 68px;
+    scroll-margin: 30px;
   }
 
   .maintainers {
@@ -117,6 +125,7 @@
   .project-translators,
   .community-integrations-sponsor,
   .maintainers {
+    scroll-margin: 30px;
     .wrapper {
       display: inline-block;
       a {
@@ -186,6 +195,11 @@
         margin: 20px auto 30px;
       }
     }
+    .social-icons {
+      a {
+        margin: 45px 10px;
+      }
+    }
   }
 
   .footer-version {

frontend/src/app/components/about/about.component.ts

@@ -5,9 +5,10 @@ import { StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
 import { ApiService } from '../../services/api.service';
 import { IBackendInfo } from '../../interfaces/websocket.interface';
-import { Router } from '@angular/router';
-import { map } from 'rxjs/operators';
+import { Router, ActivatedRoute } from '@angular/router';
+import { map, tap } from 'rxjs/operators';
 import { ITranslators } from '../../interfaces/node-api.interface';
+import { DOCUMENT } from '@angular/common';
 
 @Component({
   selector: 'app-about',
@@ -31,7 +32,9 @@ export class AboutComponent implements OnInit {
     public stateService: StateService,
     private apiService: ApiService,
     private router: Router,
+    private route: ActivatedRoute,
     @Inject(LOCALE_ID) public locale: string,
+    @Inject(DOCUMENT) private document: Document,
   ) { }
 
   ngOnInit() {
@@ -39,17 +42,21 @@ export class AboutComponent implements OnInit {
     this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`);
     this.websocketService.want(['blocks']);
 
-    this.sponsors$ = this.apiService.getDonation$();
+    this.sponsors$ = this.apiService.getDonation$()
+      .pipe(
+        tap(() => this.goToAnchor())
+      );
     this.translators$ = this.apiService.getTranslators$()
       .pipe(
         map((translators) => {
           for (const t in translators) {
             if (translators[t] === '') {
-              delete translators[t]
+              delete translators[t];
             }
           }
           return translators;
-        })
+        }),
+        tap(() => this.goToAnchor())
       );
     this.allContributors$ = this.apiService.getContributor$().pipe(
       map((contributors) => {
@@ -57,9 +64,24 @@ export class AboutComponent implements OnInit {
           regular: contributors.filter((user) => !user.core_constributor),
           core: contributors.filter((user) => user.core_constributor),
         };
-      })
+      }),
+      tap(() => this.goToAnchor())
     );
   }
+  
+  ngAfterViewInit() {
+    this.goToAnchor();
+  }
+
+  goToAnchor() {
+    setTimeout(() => {
+      if (this.route.snapshot.fragment) {
+        if (this.document.getElementById(this.route.snapshot.fragment)) {
+          this.document.getElementById(this.route.snapshot.fragment).scrollIntoView({behavior: 'smooth'});
+        }
+      }
+    }, 1);
+  }
 
   sponsor(): void {
     if (this.officialMempoolSpace && this.stateService.env.BASE_MODULE === 'mempool') {

frontend/src/app/components/amount/amount.component.ts

@@ -1,7 +1,7 @@
 import { Component, OnInit, OnDestroy, Input, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
 import { StateService } from '../../services/state.service';
 import { Observable, Subscription } from 'rxjs';
-import { Price } from 'src/app/services/price.service';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-amount',

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -5,7 +5,7 @@ import BlockScene from './block-scene';
 import TxSprite from './tx-sprite';
 import TxView from './tx-view';
 import { Position } from './sprite-types';
-import { Price } from 'src/app/services/price.service';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block-overview-graph',

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -1,7 +1,7 @@
 import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
 import { TransactionStripped } from '../../interfaces/websocket.interface';
 import { Position } from '../../components/block-overview-graph/sprite-types.js';
-import { Price } from 'src/app/services/price.service';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block-overview-tooltip',

frontend/src/app/components/block/block-preview.component.html

@@ -8,7 +8,7 @@
         <div class="block-titles">
           <h1 class="title">
             <ng-template [ngIf]="blockHeight === 0"><ng-container i18n="@@2303359202781425764">Genesis</ng-container></ng-template>
-            <ng-template [ngIf]="blockHeight" i18n="shared.block-title">{{ blockHeight }}</ng-template>
+            <ng-template [ngIf]="blockHeight">{{ blockHeight }}</ng-template>
           </h1>
           <div class="blockhash" *ngIf="blockHash">
             <h2 class="truncate right">{{ blockHash.slice(0,32) }}</h2>

frontend/src/app/components/block/block.component.ts

@@ -13,7 +13,7 @@ import { BlockAudit, BlockExtended, TransactionStripped } from '../../interfaces
 import { ApiService } from '../../services/api.service';
 import { BlockOverviewGraphComponent } from '../../components/block-overview-graph/block-overview-graph.component';
 import { detectWebGL } from '../../shared/graphs.utils';
-import { PriceService, Price } from 'src/app/services/price.service';
+import { PriceService, Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block',

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html

@@ -43,10 +43,8 @@
           <div [attr.data-cy]="'bitcoin-block-' + i + '-transactions'" class="transaction-count">
             <ng-container
               *ngTemplateOutlet="block.tx_count === 1 ? transactionsSingular : transactionsPlural; context: {$implicit: block.tx_count | number}"></ng-container>
-            <ng-template #transactionsSingular let-i i18n="shared.transaction-count.singular">{{ i }}
-              transaction</ng-template>
-            <ng-template #transactionsPlural let-i i18n="shared.transaction-count.plural">{{ i }}
-              transactions</ng-template>
+            <ng-template #transactionsSingular let-i i18n="shared.transaction-count.singular">{{ i }} transaction</ng-template>
+            <ng-template #transactionsPlural let-i i18n="shared.transaction-count.plural">{{ i }} transactions</ng-template>
           </div>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-time'" class="time-difference">
             <app-time-since [time]="block.timestamp" [fastRender]="true"></app-time-since></div>

frontend/src/app/components/search-form/search-results/search-results.component.html

@@ -1,30 +1,30 @@
 <div class="dropdown-menu show" *ngIf="results" [hidden]="!results.hashQuickMatch && !results.addresses.length && !results.nodes.length && !results.channels.length">
   <ng-template [ngIf]="results.blockHeight">
-    <div class="card-title">Bitcoin Block Height</div>
+    <div class="card-title" i18n="search.bitcoin-block-height">Bitcoin Block Height</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.txId">
-    <div class="card-title">Bitcoin Transaction</div>
+    <div class="card-title" i18n="search.bitcoin-transaction">Bitcoin Transaction</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText | shortenString : 13 }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText | shortenString : 13 }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.address">
-    <div class="card-title">Bitcoin Address</div>
+    <div class="card-title" i18n="search.bitcoin-address">Bitcoin Address</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText | shortenString : isMobile ? 20 : 30 }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText | shortenString : isMobile ? 20 : 30 }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.blockHash">
-    <div class="card-title">Bitcoin Block</div>
+    <div class="card-title" i18n="search.bitcoin-block">Bitcoin Block</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText | shortenString : 13 }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText | shortenString : 13 }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.addresses.length">
-    <div class="card-title">Bitcoin Addresses</div>
+    <div class="card-title" i18n="search.bitcoin-addresses">Bitcoin Addresses</div>
     <ng-template ngFor [ngForOf]="results.addresses" let-address let-i="index">
       <button (click)="clickItem(results.hashQuickMatch + i)" [class.active]="(results.hashQuickMatch + i) === activeIdx" type="button" role="option" class="dropdown-item">
         <ngb-highlight [result]="address | shortenString : isMobile ? 25 : 36" [term]="results.searchText"></ngb-highlight>
@@ -32,7 +32,7 @@
     </ng-template>
   </ng-template>
   <ng-template [ngIf]="results.nodes.length">
-    <div class="card-title">Lightning Nodes</div>
+    <div class="card-title" i18n="search.lightning-nodes">Lightning Nodes</div>
     <ng-template ngFor [ngForOf]="results.nodes" let-node let-i="index">
       <button (click)="clickItem(results.hashQuickMatch + results.addresses.length + i)" [class.inactive]="node.status === 0" [class.active]="results.hashQuickMatch + results.addresses.length + i === activeIdx" [routerLink]="['/lightning/node' | relativeUrl, node.public_key]" type="button" role="option" class="dropdown-item">
         <ngb-highlight [result]="node.alias" [term]="results.searchText"></ngb-highlight> &nbsp;<span class="symbol">{{ node.public_key | shortenString : 10 }}</span>
@@ -40,7 +40,7 @@
     </ng-template>
   </ng-template>
   <ng-template [ngIf]="results.channels.length">
-    <div class="card-title">Lightning Channels</div>
+    <div class="card-title" i18n="search.lightning-channels">Lightning Channels</div>
     <ng-template ngFor [ngForOf]="results.channels" let-channel let-i="index">
       <button (click)="clickItem(results.hashQuickMatch + results.addresses.length + results.nodes.length + i)" [class.inactive]="channel.status === 2"  [class.active]="results.hashQuickMatch + results.addresses.length + results.nodes.length + i === activeIdx" type="button" role="option" class="dropdown-item">
         <ngb-highlight [result]="channel.short_id" [term]="results.searchText"></ngb-highlight> &nbsp;<span class="symbol">{{ channel.id }}</span>
@@ -48,3 +48,5 @@
     </ng-template>
   </ng-template>
 </div>
+
+<ng-template #goTo let-x i18n="search.go-to">Go to "{{ x }}"</ng-template>

frontend/src/app/components/start/start.component.ts

@@ -267,6 +267,7 @@ export class StartComponent implements OnInit, OnDestroy {
 
   resetScroll(): void {
     this.scrollToBlock(this.chainTip);
+    this.blockchainContainer.nativeElement.scrollLeft = 0;
   }
 
   getPageIndexOf(height: number): number {

frontend/src/app/components/transaction/transaction.component.ts

@@ -22,7 +22,7 @@ import { SeoService } from '../../services/seo.service';
 import { BlockExtended, CpfpInfo } from '../../interfaces/node-api.interface';
 import { LiquidUnblinding } from './liquid-ublinding';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
-import { Price, PriceService } from 'src/app/services/price.service';
+import { Price, PriceService } from '../../services/price.service';
 
 @Component({
   selector: 'app-transaction',

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -9,7 +9,7 @@ import { AssetsService } from '../../services/assets.service';
 import { filter, map, tap, switchMap, shareReplay } from 'rxjs/operators';
 import { BlockExtended } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
-import { PriceService } from 'src/app/services/price.service';
+import { PriceService } from '../../services/price.service';
 
 @Component({
   selector: 'app-transactions-list',

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts

@@ -1,6 +1,6 @@
 import { Component, ElementRef, ViewChild, Input, OnChanges, OnInit } from '@angular/core';
 import { tap } from 'rxjs';
-import { Price, PriceService } from 'src/app/services/price.service';
+import { Price, PriceService } from '../../services/price.service';
 
 interface Xput {
   type: 'input' | 'output' | 'fee';

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -199,8 +199,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     this.outputs = this.initLines('out', voutWithFee, totalValue, this.maxStrands);
 
     this.middle = {
-      path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.25} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.25}`,
-      style: `stroke-width: ${this.combinedWeight + 0.5}; stroke: ${this.gradient[1]}`
+      path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.5} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.5}`,
+      style: `stroke-width: ${this.combinedWeight + 1}; stroke: ${this.gradient[1]}`
     };
 
     this.hasLine = this.inputs.reduce((line, put) => line || !put.zeroValue, false)
@@ -257,7 +257,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     const lineParams = weights.map((w, i) => {
       return {
         weight: w,
-        thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.min(this.combinedWeight + 0.5, Math.max(this.minWeight - 1, w) + 1),
+        thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.max(this.minWeight - 1, w) + 1,
         offset: 0,
         innerY: 0,
         outerY: 0,
@@ -269,7 +269,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
 
     // bounds of the middle segment
     const innerTop = (this.height / 2) - (this.combinedWeight / 2);
-    const innerBottom = innerTop + this.combinedWeight + 0.5;
+    const innerBottom = innerTop + this.combinedWeight;
     // tracks the visual bottom of the endpoints of the previous line
     let lastOuter = 0;
     let lastInner = innerTop;
@@ -294,7 +294,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
 
       // set the vertical position of the (center of the) outer side of the line
       line.outerY = lastOuter + (line.thickness / 2);
-      line.innerY = Math.min(innerBottom - (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2)));
+      line.innerY = Math.min(innerBottom + (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2)));
 
       // special case to center single input/outputs
       if (xputs.length === 1) {

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -2907,6 +2907,219 @@ export const restApiDocsData = [
     }
   },
   ...
+]`
+        },
+        codeSampleLiquid: emptyCodeSample,
+        codeSampleLiquidTestnet: emptyCodeSample,
+        codeSampleBisq: emptyCodeSample,
+      }
+    }
+  },
+  {
+    type: "endpoint",
+    category: "blocks",
+    httpRequestMethod: "GET",
+    fragment: "get-blocks-bulk",
+    title: "GET Blocks (Bulk)",
+    description: {
+      default: "<p>Returns details on the range of blocks between <code>:minHeight</code> and <code>:maxHeight</code>, inclusive, up to 10 blocks. If <code>:maxHeight</code> is not specified, it defaults to the current tip.</p><p>To return data for more than 10 blocks, consider becoming an <a href='/enterprise'>enterprise sponsor</a>.</p>"
+    },
+    urlString: "/v1/blocks-bulk/:minHeight[/:maxHeight]",
+    showConditions: bitcoinNetworks,
+    showJsExamples: showJsExamplesDefaultFalse,
+    codeExample: {
+      default: {
+        codeTemplate: {
+          curl: `/api/v1/blocks-bulk/%{1}/%{2}`,
+          commonJS: ``,
+        },
+        codeSampleMainnet: {
+          esModule: [],
+          commonJS: [],
+          curl: [100000,100000],
+          response: `[
+  {
+    "height": 100000,
+    "hash": "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506",
+    "timestamp": 1293623863,
+    "median_timestamp": 1293622620,
+    "previous_block_hash": "000000000002d01c1fccc21636b607dfd930d31d01c3a62104612a1719011250",
+    "difficulty": 14484.1623612254,
+    "header": "0100000050120119172a610421a6c3011dd330d9df07b63616c2cc1f1cd00200000000006657a9252aacd5c0b2940996ecff952228c3067cc38d4885efb5a4ac4247e9f337221b4d4c86041b0f2b5710",
+    "version": 1,
+    "bits": 453281356,
+    "nonce": 274148111,
+    "size": 957,
+    "weight": 3828,
+    "tx_count": 4,
+    "merkle_root": "f3e94742aca4b5ef85488dc37c06c3282295ffec960994b2c0d5ac2a25a95766",
+    "reward": 5000000000,
+    "total_fee_amt": 0,
+    "avg_fee_amt": 0,
+    "median_fee_amt": 0,
+    "fee_amt_percentiles": {
+      "min": 0,
+      "perc_10": 0,
+      "perc_25": 0,
+      "perc_50": 0,
+      "perc_75": 0,
+      "perc_90": 0,
+      "max": 0
+    },
+    "avg_fee_rate": 0,
+    "median_fee_rate": 0,
+    "fee_rate_percentiles": {
+      "min": 0,
+      "perc_10": 0,
+      "perc_25": 0,
+      "perc_50": 0,
+      "perc_75": 0,
+      "perc_90": 0,
+      "max": 0
+    },
+    "total_inputs": 3,
+    "total_input_amt": 5301000000,
+    "total_outputs": 6,
+    "total_output_amt": 5301000000,
+    "segwit_total_txs": 0,
+    "segwit_total_size": 0,
+    "segwit_total_weight": 0,
+    "avg_tx_size": 185.25,
+    "utxoset_change": 3,
+    "utxoset_size": 71888,
+    "coinbase_raw": "044c86041b020602",
+    "coinbase_address": null,
+    "coinbase_signature": "OP_PUSHBYTES_65 041b0e8c2567c12536aa13357b79a073dc4444acb83c4ec7a0e2f99dd7457516c5817242da796924ca4e99947d087fedf9ce467cb9f7c6287078f801df276fdf84 OP_CHECKSIG",
+    "coinbase_signature_ascii": "\u0004L<34>\u0004\u001b\u0002\u0006\u0002",
+    "pool_slug": "unknown",
+    "orphans": []
+  }
+]`,
+        },
+        codeSampleTestnet: {
+          esModule: [],
+          commonJS: [],
+          curl: [100000,100000],
+          response: `[
+  {
+    "height": 100000,
+    "hash": "00000000009e2958c15ff9290d571bf9459e93b19765c6801ddeccadbb160a1e",
+    "timestamp": 1376123972,
+    "median_timestamp": 1677396660,
+    "previous_block_hash": "000000004956cc2edd1a8caa05eacfa3c69f4c490bfc9ace820257834115ab35",
+    "difficulty": 271.7576739288896,
+    "header": "0200000035ab154183570282ce9afc0b494c9fc6a3cfea05aa8c1add2ecc56490000000038ba3d78e4500a5a7570dbe61960398add4410d278b21cd9708e6d9743f374d544fc055227f1001c29c1ea3b",
+    "version": 2,
+    "bits": 469823783,
+    "nonce": 1005240617,
+    "size": 221,
+    "weight": 884,
+    "tx_count": 1,
+    "merkle_root": "d574f343976d8e70d91cb278d21044dd8a396019e6db70755a0a50e4783dba38",
+    "reward": 5000000000,
+    "total_fee_amt": 0,
+    "avg_fee_amt": 0,
+    "median_fee_amt": 0,
+    "fee_amt_percentiles": {
+      "min": 0,
+      "perc_10": 0,
+      "perc_25": 0,
+      "perc_50": 0,
+      "perc_75": 0,
+      "perc_90": 0,
+      "max": 0
+    },
+    "avg_fee_rate": 0,
+    "median_fee_rate": 0,
+    "fee_rate_percentiles": {
+      "min": 0,
+      "perc_10": 0,
+      "perc_25": 0,
+      "perc_50": 0,
+      "perc_75": 0,
+      "perc_90": 0,
+      "max": 0
+    },
+    "total_inputs": 0,
+    "total_input_amt": null,
+    "total_outputs": 1,
+    "total_output_amt": 0,
+    "segwit_total_txs": 0,
+    "segwit_total_size": 0,
+    "segwit_total_weight": 0,
+    "avg_tx_size": 0,
+    "utxoset_change": 1,
+    "utxoset_size": null,
+    "coinbase_raw": "03a08601000427f1001c046a510100522cfabe6d6d0000000000000000000068692066726f6d20706f6f6c7365727665726aac1eeeed88",
+    "coinbase_address": "mtkbaiLiUH3fvGJeSzuN3kUgmJzqinLejJ",
+    "coinbase_signature": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 912e2b234f941f30b18afbb4fa46171214bf66c8 OP_EQUALVERIFY OP_CHECKSIG",
+    "coinbase_signature_ascii": "\u0003 <20>\u0001\u0000\u0004'ñ\u0000\u001c\u0004jQ\u0001\u0000R,ú¾mm\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000hi from poolserverj¬\u001eîí<C3AE>",
+    "pool_slug": "unknown",
+    "orphans": []
+  }
+]`
+        },
+        codeSampleSignet: {
+          esModule: [],
+          commonJS: [],
+          curl: [100000,100000],
+          response: `[
+  {
+    "height": 100000,
+    "hash": "0000008753108390007b3f5c26e5d924191567e147876b84489b0c0cf133a0bf",
+    "timestamp": 1658421183,
+    "median_timestamp": 1658418056,
+    "previous_block_hash": "000000b962a13c3dd3f81917bc8646a0c98224adcd5124026d4fdfcb76a76d30",
+    "difficulty": 0.002781447610743506,
+    "header": "00000020306da776cbdf4f6d022451cdad2482c9a04686bc1719f8d33d3ca162b90000001367fb15320ebb1932fd589f8f38866b692ca8a4ad6100a4bc732d212916d0efbf7fd9628567011e47662d00",
+    "version": 536870912,
+    "bits": 503408517,
+    "nonce": 2975303,
+    "size": 343,
+    "weight": 1264,
+    "tx_count": 1,
+    "merkle_root": "efd01629212d73bca40061ada4a82c696b86388f9f58fd3219bb0e3215fb6713",
+    "reward": 5000000000,
+    "total_fee_amt": 0,
+    "avg_fee_amt": 0,
+    "median_fee_amt": 0,
+    "fee_amt_percentiles": {
+      "min": 0,
+      "perc_10": 0,
+      "perc_25": 0,
+      "perc_50": 0,
+      "perc_75": 0,
+      "perc_90": 0,
+      "max": 0
+    },
+    "avg_fee_rate": 0,
+    "median_fee_rate": 0,
+    "fee_rate_percentiles": {
+      "min": 0,
+      "perc_10": 0,
+      "perc_25": 0,
+      "perc_50": 0,
+      "perc_75": 0,
+      "perc_90": 0,
+      "max": 0
+    },
+    "total_inputs": 0,
+    "total_input_amt": null,
+    "total_outputs": 2,
+    "total_output_amt": 0,
+    "segwit_total_txs": 0,
+    "segwit_total_size": 0,
+    "segwit_total_weight": 0,
+    "avg_tx_size": 0,
+    "utxoset_change": 2,
+    "utxoset_size": null,
+    "coinbase_raw": "03a08601",
+    "coinbase_address": "tb1psfjl80vk0yp3agcq6ylueas29rau00mfq90mhejerpgccg33xhasd9gjyd",
+    "coinbase_signature": "OP_PUSHNUM_1 OP_PUSHBYTES_32 8265f3bd9679031ea300d13fccf60a28fbc7bf69015fbbe65918518c223135fb",
+    "coinbase_signature_ascii": "\u0003 <20>\u0001",
+    "pool_slug": "unknown",
+    "orphans": []
+  }
 ]`
         },
         codeSampleLiquid: emptyCodeSample,

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -10,7 +10,7 @@
       <div class="doc-content">
 
         <div id="disclaimer">
-          <table><tr><td><svg viewBox="0 0 304 304" xmlns="http://www.w3.org/2000/svg"><g fill-rule="evenodd" style="fill:#ffc107;fill-opacity:1"><path d="M135.3 34.474c-15.62 27.306-54.206 95.63-85.21 150.534L9.075 257.583C5.382 264.08 6.76 269.217 7.908 271.7c2.326 5.028 7.29 7.537 11.155 8.215l.78.133 264.698.006-.554-.02c4.152.255 9.664-1.24 12.677-6.194 1.926-3.18 3.31-8.589-1.073-16.278L213.637 114.37l-45.351-79.205c-5.681-9.932-12.272-12.022-16.8-12.022-4.42 0-10.818 1.964-16.181 11.331h-.006zm-69.072 159.94c30.997-54.885 69.563-123.184 85.16-150.446l.186-.297c.2.303.393.582.618.981l45.363 79.22s72.377 126.47 78.569 137.283l-247.618-.007 37.719-66.734" style="fill:#ffc107;fill-opacity:1"/><path d="M152.597 247.445c8.02 0 14.518-6.728 14.518-15.025 0-8.29-6.499-15.018-14.518-15.018-8.031 0-14.529 6.728-14.529 15.018 0 8.297 6.498 15.025 14.53 15.025m-.001-147.18c11.586 0 22.23 10.958 20.977 21.7l-9.922 75.564c-.966 6.601-4.95 11.433-11.055 11.433s-10.102-4.832-11.056-11.433l-9.927-75.564c-1.26-10.742 9.39-21.7 20.983-21.7" style="fill:#ffc107;fill-opacity:1"/></g></svg></td><td><p><b>mempool.space merely provides data about the Bitcoin network.</b> It cannot help you with retrieving funds, confirming your transaction quicker, etc.</p><p>For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).</p></td></tr></table>
+          <table><tr><td><svg viewBox="0 0 304 304" xmlns="http://www.w3.org/2000/svg"><g fill-rule="evenodd" style="fill:#ffc107;fill-opacity:1"><path d="M135.3 34.474c-15.62 27.306-54.206 95.63-85.21 150.534L9.075 257.583C5.382 264.08 6.76 269.217 7.908 271.7c2.326 5.028 7.29 7.537 11.155 8.215l.78.133 264.698.006-.554-.02c4.152.255 9.664-1.24 12.677-6.194 1.926-3.18 3.31-8.589-1.073-16.278L213.637 114.37l-45.351-79.205c-5.681-9.932-12.272-12.022-16.8-12.022-4.42 0-10.818 1.964-16.181 11.331h-.006zm-69.072 159.94c30.997-54.885 69.563-123.184 85.16-150.446l.186-.297c.2.303.393.582.618.981l45.363 79.22s72.377 126.47 78.569 137.283l-247.618-.007 37.719-66.734" style="fill:#ffc107;fill-opacity:1"/><path d="M152.597 247.445c8.02 0 14.518-6.728 14.518-15.025 0-8.29-6.499-15.018-14.518-15.018-8.031 0-14.529 6.728-14.529 15.018 0 8.297 6.498 15.025 14.53 15.025m-.001-147.18c11.586 0 22.23 10.958 20.977 21.7l-9.922 75.564c-.966 6.601-4.95 11.433-11.055 11.433s-10.102-4.832-11.056-11.433l-9.927-75.564c-1.26-10.742 9.39-21.7 20.983-21.7" style="fill:#ffc107;fill-opacity:1"/></g></svg></td><td><p i18n="faq.big-disclaimer"><b>mempool.space merely provides data about the Bitcoin network.</b> It cannot help you with retrieving funds, confirming your transaction quicker, etc.</p><p>For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).</p></td></tr></table>
 
         </div>
 
@@ -39,7 +39,7 @@
       <div class="doc-content">
 
         <p class="doc-welcome-note">Below is a reference for the {{ network.val === '' ? 'Bitcoin' : network.val.charAt(0).toUpperCase() + network.val.slice(1) }} <ng-container i18n="api-docs.title">REST API service</ng-container>.</p>
-        <p class="doc-welcome-note api-note" *ngIf="officialMempoolInstance">Note that we enforce rate limits. If you exceed these limits, you will get a polite error encouraging you to run your own Mempool instance. If you repeatedly exceed the limits, you may be banned from accessing the service altogether.</p>
+        <p class="doc-welcome-note api-note" *ngIf="officialMempoolInstance">Note that we enforce rate limits. If you exceed these limits, you will get an HTTP 429 error. If you repeatedly exceed the limits, you may be banned from accessing the service altogether. Consider an <a [routerLink]="['/enterprise']">enterprise sponsorship</a> if you need higher API limits.</p>
 
         <div class="doc-item-container" *ngFor="let item of restDocs">
           <h3 *ngIf="( item.type === 'category' ) && ( item.showConditions.indexOf(network.val) > -1 )">{{ item.title }}</h3>

frontend/src/app/docs/docs/docs.component.html

@@ -32,7 +32,7 @@
         </ng-template>
       </li>
 
-      <li [ngbNavItem]="3" *ngIf="showElectrsTab" role="presentation">
+      <li [ngbNavItem]="3" *ngIf="showElectrsTab" role="presentation" class="hide-on-mobile">
         <a ngbNavLink [routerLink]="['/docs/api/electrs' | relativeUrl]" role="tab">API - Electrum RPC</a>
         <ng-template ngbNavContent>
 

frontend/src/app/docs/docs/docs.component.scss

@@ -7,3 +7,9 @@
 #footer {
   clear: both;
 }
+
+@media (max-width: 992px) {
+  .hide-on-mobile {
+    display: none;
+  }
+}

frontend/src/app/interfaces/node-api.interface.ts

@@ -114,7 +114,6 @@ export interface BlockExtension {
   medianFee?: number;
   feeRange?: number[];
   reward?: number;
-  coinbaseTx?: Transaction;
   coinbaseRaw?: string;
   matchRate?: number;
   pool?: {

frontend/src/app/lightning/channel/channel.component.html

@@ -1,25 +1,32 @@
 <div class="container-xl" *ngIf="(channel$ | async) as channel; else skeletonLoader">
-  <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.channel">Lightning channel</h5>
-  <div class="title-container">
-    <h1 class="mb-0">{{ channel.short_id }}</h1>
-    <span class="tx-link">
-      <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">{{ channel.id }}</a>
-      <app-clipboard [text]="channel.id"></app-clipboard>
-    </span>
-  </div>
-  <div class="badges mb-2">
-    <span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0" i18n="status.inactive">Inactive</span>
-    <span class="badge rounded-pill badge-success" *ngIf="channel.status === 1" i18n="status.active">Active</span>
-    <span class="badge rounded-pill badge-danger" *ngIf="channel.status === 2" i18n="status.closed">Closed</span>
-    <app-closing-type *ngIf="channel.closing_reason" [type]="channel.closing_reason"></app-closing-type>
-  </div>
+
+  <ng-container *ngIf="!error">
+    <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.channel">Lightning channel</h5>
+    <div class="title-container">
+      <h1 class="mb-0">{{ channel.short_id }}</h1>
+      <span class="tx-link">
+        <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">{{ channel.id }}</a>
+        <app-clipboard [text]="channel.id"></app-clipboard>
+      </span>
+    </div>
+    <div class="badges mb-2">
+      <span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0" i18n="status.inactive">Inactive</span>
+      <span class="badge rounded-pill badge-success" *ngIf="channel.status === 1" i18n="status.active">Active</span>
+      <span class="badge rounded-pill badge-danger" *ngIf="channel.status === 2" i18n="status.closed">Closed</span>
+      <app-closing-type *ngIf="channel.closing_reason" [type]="channel.closing_reason"></app-closing-type>
+    </div>
+  </ng-container>
 
   <div class="clearfix"></div>
 
+  <div *ngIf="error" class="d-flex flex-column justify-content-around align-items-center mt-5 w-100" style="min-height: 100px">
+    <span class="text-center" i18n="lightning.channel-not-found">No channel found for short id "{{ channel.short_id }}"</span>
+  </div>
+
   <app-nodes-channels-map *ngIf="!error && (channelGeo$ | async) as channelGeo" [style]="'channelpage'"
     [channel]="channelGeo"></app-nodes-channels-map>
 
-  <div class="box">
+  <div class="box" *ngIf="!error">
 
       <div class="row">
         <div class="col-md">
@@ -65,7 +72,7 @@
 
   <br>
 
-  <div class="row row-cols-1 row-cols-md-2">
+  <div class="row row-cols-1 row-cols-md-2" *ngIf="!error">
     <div class="col">
       <app-channel-box [channel]="channel.node_left"></app-channel-box>
       <app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [local]="channel.node_left" [remote]="channel.node_right"></app-channel-close-box>
@@ -104,14 +111,6 @@
 
 <br>
 
-<ng-template [ngIf]="error">
-  <div class="text-center">
-    <span i18n="error.general-loading-data">Error loading data.</span>
-    <br><br>
-    <i>{{ error.status }}: {{ error.error }}</i>
-  </div>
-</ng-template>
-
 <ng-template #skeletonLoader>
   <div class="container-xl">
     <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.channel">Lightning channel</h5>

frontend/src/app/lightning/channel/channel.component.ts

@@ -38,7 +38,9 @@ export class ChannelComponent implements OnInit {
               }),
               catchError((err) => {
                 this.error = err;
-                return of(null);
+                return [{
+                  short_id: params.get('short_id')
+                }];
               })
             );
         }),

frontend/src/app/lightning/channel/closing-type/closing-type.component.ts

@@ -17,19 +17,19 @@ export class ClosingTypeComponent implements OnChanges {
   getLabelFromType(type: number): { label: string; class: string } {
     switch (type) {
       case 1: return { 
-        label: 'Mutually closed',
+        label: $localize`Mutually closed`,
         class: 'success',
       };
       case 2: return {
-        label: 'Force closed',
+        label: $localize`Force closed`,
         class: 'warning',
       };
       case 3: return {
-        label: 'Force closed with penalty',
+        label: $localize`Force closed with penalty`,
         class: 'danger',
       };
       default: return {
-        label: 'Unknown',
+        label: $localize`:@@e5d8bb389c702588877f039d72178f219453a72d:Unknown`,
         class: 'secondary',
       };
     }

frontend/src/app/lightning/channels-statistics/channels-statistics.component.html

@@ -1,9 +1,9 @@
 <div class="widget-toggler">
   <a href="" (click)="switchMode('avg')" class="toggler-option"
-    [ngClass]="{'inactive': mode === 'avg'}"><small>avg</small></a>
+    [ngClass]="{'inactive': mode === 'avg'}"><small i18n="statistics.average-small">avg</small></a>
   <span style="color: #ffffff66; font-size: 8px"> | </span>
   <a href="" (click)="switchMode('med')" class="toggler-option"
-    [ngClass]="{'inactive': mode === 'med'}"><small>med</small></a>
+    [ngClass]="{'inactive': mode === 'med'}"><small i18n="statistics.median-small">med</small></a>
 </div>
 
 <div class="fee-estimation-wrapper" *ngIf="statistics$ | async as statistics; else loadingReward">

frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html

@@ -55,7 +55,7 @@
     </div>
 
     <!-- Top nodes per capacity -->
-    <div class="col">
+    <div class="col" style="max-height: 410px">
       <div class="card">
         <div class="card-body">
           <a class="title-link" href="" [routerLink]="['/lightning/nodes/rankings/liquidity' | relativeUrl]">
@@ -69,7 +69,7 @@
     </div>
 
     <!-- Top nodes per channels -->
-    <div class="col">
+    <div class="col" style="max-height: 410px">
       <div class="card">
         <div class="card-body">
           <a class="title-link" href="" [routerLink]="['/lightning/nodes/rankings/connectivity' | relativeUrl]">

frontend/src/app/lightning/node-fee-chart/node-fee-chart.component.ts

@@ -167,7 +167,7 @@ export class NodeFeeChartComponent implements OnInit {
         padding: 10,
         data: [
           {
-            name: 'Outgoing Fees',
+            name: $localize`Outgoing Fees`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -175,7 +175,7 @@ export class NodeFeeChartComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: 'Incoming Fees',
+            name: $localize`Incoming Fees`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -205,7 +205,7 @@ export class NodeFeeChartComponent implements OnInit {
       series: outgoingData.length === 0 ? undefined : [
         {
           zlevel: 0,
-          name: 'Outgoing Fees',
+          name: $localize`Outgoing Fees`,
           data: outgoingData.map(bucket => ({
             value: bucket.capacity,
             label: bucket.label,
@@ -219,7 +219,7 @@ export class NodeFeeChartComponent implements OnInit {
         },
         {
           zlevel: 0,
-          name: 'Incoming Fees',
+          name: $localize`Incoming Fees`,
           data: incomingData.map(bucket => ({
             value: -bucket.capacity,
             label: bucket.label,

frontend/src/app/lightning/node/node.component.html

@@ -1,20 +1,23 @@
 <div class="container-xl" *ngIf="(node$ | async) as node; else skeletonLoader">
-  <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.node">Lightning node</h5>
-  <div class="title-container mb-2" *ngIf="!error">
-    <h1 class="mb-0 text-truncate">{{ node.alias }}</h1>
-    <span class="tx-link">
-      <span class="node-id">
-        <app-truncate [text]="node.public_key" [lastChars]="8" [link]="['/lightning/node' | relativeUrl, node.public_key]">
-          <app-clipboard [text]="node.public_key"></app-clipboard>
-        </app-truncate>
+
+  <ng-container *ngIf="!error">
+    <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.node">Lightning node</h5>
+    <div class="title-container mb-2">
+      <h1 class="mb-0 text-truncate">{{ node.alias }}</h1>
+      <span class="tx-link">
+        <span class="node-id">
+          <app-truncate [text]="node.public_key" [lastChars]="8" [link]="['/lightning/node' | relativeUrl, node.public_key]">
+            <app-clipboard [text]="node.public_key"></app-clipboard>
+          </app-truncate>
+        </span>
       </span>
-    </span>
-  </div>
+    </div>
+  </ng-container>
 
   <div class="clearfix"></div>
 
   <div *ngIf="error" class="d-flex flex-column justify-content-around align-items-center mt-5 w-100" style="min-height: 100px">
-    <span i18n="lightning.node-not-found">No node found for public key "{{ node.public_key | shortenString : 12}}"</span>
+    <span class="text-center" i18n="lightning.node-not-found">No node found for public key "{{ node.public_key | shortenString : 12}}"</span>
   </div>
 
   <div class="box" *ngIf="!error">

frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts

@@ -161,28 +161,7 @@ export class NodesNetworksChartComponent implements OnInit {
       {
         zlevel: 1,
         yAxisIndex: 0,
-        name: $localize`Reachable on Clearnet Only`,
-        showSymbol: false,
-        symbol: 'none',
-        data: data.clearnet_nodes,
-        type: 'line',
-        lineStyle: {
-          width: 2,
-        },
-        areaStyle: {
-          opacity: 0.5,
-        },
-        stack: 'Total',
-        color: new graphic.LinearGradient(0, 0.75, 0, 1, [
-          { offset: 0, color: '#FFB300' },
-          { offset: 1, color: '#FFB300AA' },
-        ]),
-        smooth: false,
-      },
-      {
-        zlevel: 1,
-        yAxisIndex: 0,
-        name: $localize`Reachable on Clearnet and Darknet`,
+        name: $localize`Clearnet and Darknet`,
         showSymbol: false,
         symbol: 'none',
         data: data.clearnet_tor_nodes,
@@ -203,7 +182,28 @@ export class NodesNetworksChartComponent implements OnInit {
       {
         zlevel: 1,
         yAxisIndex: 0,
-        name: $localize`Reachable on Darknet Only`,
+        name: $localize`Clearnet (IPv4, IPv6)`,
+        showSymbol: false,
+        symbol: 'none',
+        data: data.clearnet_nodes,
+        type: 'line',
+        lineStyle: {
+          width: 2,
+        },
+        areaStyle: {
+          opacity: 0.5,
+        },
+        stack: 'Total',
+        color: new graphic.LinearGradient(0, 0.75, 0, 1, [
+          { offset: 0, color: '#FFB300' },
+          { offset: 1, color: '#FFB300AA' },
+        ]),
+        smooth: false,
+      },
+      {
+        zlevel: 1,
+        yAxisIndex: 0,
+        name: $localize`Darknet Only (Tor, I2P, cjdns)`,
         showSymbol: false,
         symbol: 'none',
         data: data.tor_nodes,
@@ -284,7 +284,7 @@ export class NodesNetworksChartComponent implements OnInit {
         padding: 10,
         data: [
           {
-            name: $localize`Reachable on Darknet Only`,
+            name: $localize`Darknet Only (Tor, I2P, cjdns)`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -292,7 +292,7 @@ export class NodesNetworksChartComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: $localize`Reachable on Clearnet and Darknet`,
+            name: $localize`Clearnet (IPv4, IPv6)`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -300,7 +300,7 @@ export class NodesNetworksChartComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: $localize`Reachable on Clearnet Only`,
+            name: $localize`Clearnet and Darknet`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -317,9 +317,9 @@ export class NodesNetworksChartComponent implements OnInit {
           },
         ],
         selected: this.widget ? undefined : JSON.parse(this.storageService.getValue('nodes_networks_legend'))  ?? {
-          '$localize`Reachable on Darknet Only`': true,
-          '$localize`Reachable on Clearnet Only`': true,
-          '$localize`Reachable on Clearnet and Darknet`': true,
+          '$localize`Darknet Only (Tor, I2P, cjdns)`': true,
+          '$localize`Clearnet (IPv4, IPv6)`': true,
+          '$localize`Clearnet and Darknet`': true,
           '$localize`:@@e5d8bb389c702588877f039d72178f219453a72d:Unknown`': true,
         }
       },

frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html

@@ -1,71 +1,56 @@
-<div [class]="!widget ? 'container-xl full-height' : ''">
-  <h1 *ngIf="!widget" class="float-left">
-    <span i18n="lightning.top-100-liquidity">Top 100 nodes liquidity ranking</span>
-  </h1>
 
-  <div [class]="widget ? 'widget' : 'full'">
-    <table class="table table-borderless table-fixed">
+
+<div class="container-xl" style="min-height: 335px" [ngClass]="{'widget': widget, 'full-height': !widget}">
+  <h1 *ngIf="!widget" class="float-left" i18n="lightning.liquidity-ranking">Liquidity Ranking</h1>
+
+  <div class="clearfix"></div>
+
+  <div style="min-height: 295px">
+    <table class="table table-borderless">
       <thead>
-        <th class="rank"></th>
-        <th class="alias text-left" i18n="nodes.alias">Alias</th>
-        <th class="capacity text-right" i18n="node.liquidity">Liquidity</th>
-        <th *ngIf="!widget" class="channels text-right" i18n="lightning.channels">Channels</th>
-        <th *ngIf="!widget" class="timestamp-first text-left" i18n="transaction.first-seen|Transaction first seen">First seen</th>
-        <th *ngIf="!widget" class="timestamp-update text-left" i18n="lightning.last_update">Last update</th>
-        <th *ngIf="!widget" class="location text-right" i18n="lightning.location">Location</th>
+        <th class="text-left" i18n="nodes.alias">Alias</th>
+        <th class="liquidity text-right" i18n="node.liquidity">Liquidity</th>
+        <th class="d-table-cell fiat text-right" [class]="{'widget': widget}">{{ currency$ | async }}</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell channels text-right" i18n="lightning.channels">Channels</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell timestamp text-right" i18n="transaction.first-seen|Transaction first seen">First seen</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell timestamp text-right" i18n="lightning.last_update">Last update</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell text-right" i18n="lightning.location">Location</th>
       </thead>
-      <tbody *ngIf="topNodesPerCapacity$ | async as nodes; else skeleton">
-        <tr *ngFor="let node of nodes; let i = index;">
-          <td class="rank text-left">
-            {{ i + 1 }}
+      <tbody *ngIf="topNodesPerCapacity$ | async as nodes">
+        <tr *ngFor="let node of nodes;">
+          <td class="pool text-left">
+            <div class="tooltip-custom d-block w-100">
+              <a class="link d-block w-100" [routerLink]="['/lightning/node' | relativeUrl, node.publicKey]">
+                <span class="pool-name w-100">{{ node.alias }}</span>
+              </a>
+            </div>
           </td>
-          <td class="alias text-left">
-            <a [routerLink]="['/lightning/node' | relativeUrl, node.publicKey]">{{ node.alias }}</a>
-          </td>
-          <td class="capacity text-right">
+          <td class="text-right">
             <app-amount [satoshis]="node.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
           </td>
-          <td *ngIf="!widget" class="channels text-right">
+          <td class="d-table-cell fiat text-right" [ngClass]="{'widget': widget}">
+            <app-fiat [value]="node.capacity"></app-fiat>
+          </td>
+          <td *ngIf="!widget" class="d-none d-md-table-cell text-right">
             {{ node.channels | number }}
           </td>
-          <td *ngIf="!widget" class="timestamp-first text-left">
+          <td *ngIf="!widget" class="d-none d-md-table-cell text-right">
             <app-timestamp [customFormat]="'yyyy-MM-dd'" [unixTime]="node.firstSeen" [hideTimeSince]="true"></app-timestamp>
           </td>
-          <td *ngIf="!widget" class="timestamp-update text-left">
+          <td *ngIf="!widget" class="d-none d-md-table-cell text-right">
             <app-timestamp [customFormat]="'yyyy-MM-dd'" [unixTime]="node.updatedAt" [hideTimeSince]="true"></app-timestamp>
           </td>
-          <td *ngIf="!widget" class="location text-right text-truncate">
+          <td *ngIf="!widget" class="d-none d-md-table-cell text-right text-truncate">
             <app-geolocation [data]="node.geolocation" [type]="'list-isp'"></app-geolocation>
           </td>
         </tr>
       </tbody>
-      <ng-template #skeleton>
-        <tbody>
-          <tr *ngFor="let item of skeletonRows">
-            <td class="rank text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td class="alias text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td class="capacity text-right">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="channels text-right">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="timestamp-first text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="timestamp-update text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="location text-right text-truncate">
-              <span class="skeleton-loader"></span>
-            </td>
-          </tr>
-        </tbody>
-      </ng-template>
     </table>
+
+    <ng-template [ngIf]="!widget">
+      <div class="clearfix"></div>
+      <br>
+    </ng-template>
   </div>
-</div>
+  
+</div>

frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss

@@ -1,91 +1,52 @@
 .container-xl {
   max-width: 1400px;
-  padding-bottom: 100px;
-  @media (min-width: 960px) {
-    padding-left: 50px;
-    padding-right: 50px;
-  }
+}
+.container-xl.widget {
+  padding-right: 0px;
+  padding-left: 0px;
+  padding-bottom: 0px;
 }
 
-.table td, .table th {
-  padding: 0.5rem;
+tr, td, th {
+  border: 0px;
+  padding-top: 0.65rem !important;
+  padding-bottom: 0.7rem !important;
 }
 
-.full .rank {
-  width: 5%;
-}
-.widget .rank {
-  @media (min-width: 960px) {
-    width: 13%;
-  }
-  @media (max-width: 960px) {
-    padding-left: 0px;
-    padding-right: 0px;
-  }
+.clear-link {
+  color: white;
 }
 
-.full .alias {
-  width: 20%;
+.pool {
+  width: 15%;
+  @media (max-width: 575px) {
+    width: 75%;
+  }
   overflow: hidden;
   text-overflow: ellipsis;
-  max-width: 350px;
-  @media (max-width: 960px) {
-    width: 40%;
-    max-width: 500px;
-  }
+  white-space: nowrap;
+  max-width: 160px;
 }
-.widget .alias {
-  width: 60%;
-  overflow: hidden;
+.pool-name {
+  display: inline-block;
+  vertical-align: text-top;
   text-overflow: ellipsis;
-  max-width: 350px;
-  @media (max-width: 960px) {
-    max-width: 175px;
-  }
+  overflow: hidden;
 }
 
-.full .capacity {
+.liquidity {
   width: 10%;
-  @media (max-width: 960px) {
-    width: 30%;
-  }
-}
-.widget .capacity {
-  width: 32%;
-  @media (max-width: 960px) {
-    padding-left: 0px;
-    padding-right: 0px;
+  @media (max-width: 575px) {
+    width: 25%;
   }
 }
 
-.full .channels {
+.fiat {
   width: 15%;
-  padding-right: 50px;
-  @media (max-width: 960px) {
-    display: none;
+  @media (min-width: 768px) and (max-width: 991px) {
+    display: none !important;
+  }
+  @media (max-width: 575px) {
+    display: none !important;
   }
 }
-
-.full .timestamp-first {
-  width: 10%;
-  @media (max-width: 960px) {
-    display: none;
-  }
-}
-
-.full .timestamp-update {
-  width: 10%;
-  @media (max-width: 960px) {
-    display: none;
-  }
-}
-
-.full .location {
-  width: 15%;
-  @media (max-width: 960px) {
-    width: 30%;
-  }
-  @media (max-width: 600px) {
-    display: none;
-  }
-}

frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts

@@ -2,7 +2,7 @@ import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core
 import { map, Observable } from 'rxjs';
 import { INodesRanking, ITopNodesPerCapacity } from '../../../interfaces/node-api.interface';
 import { SeoService } from '../../../services/seo.service';
-import { isMobile } from '../../../shared/common.utils';
+import { StateService } from '../../../services/state.service';
 import { GeolocationData } from '../../../shared/components/geolocation/geolocation.component';
 import { LightningApiService } from '../../lightning-api.service';
 
@@ -18,18 +18,22 @@ export class TopNodesPerCapacity implements OnInit {
   
   topNodesPerCapacity$: Observable<ITopNodesPerCapacity[]>;
   skeletonRows: number[] = [];
+  currency$: Observable<string>;
 
   constructor(
     private apiService: LightningApiService,
-    private seoService: SeoService
+    private seoService: SeoService,
+    private stateService: StateService,
   ) {}
 
   ngOnInit(): void {
+    this.currency$ = this.stateService.fiatCurrency$;
+
     if (!this.widget) {
       this.seoService.setTitle($localize`:@@2d9883d230a47fbbb2ec969e32a186597ea27405:Liquidity Ranking`);
     }
 
-    for (let i = 1; i <= (this.widget ? (isMobile() ? 8 : 7) : 100); ++i) {
+    for (let i = 1; i <= (this.widget ? 6 : 100); ++i) {
       this.skeletonRows.push(i);
     }
 
@@ -50,7 +54,7 @@ export class TopNodesPerCapacity implements OnInit {
     } else {
       this.topNodesPerCapacity$ = this.nodes$.pipe(
         map((ranking) => {
-          return ranking.topByCapacity.slice(0, isMobile() ? 8 : 7);
+          return ranking.topByCapacity.slice(0, 6);
         })
       );
     }

frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html

@@ -1,71 +1,56 @@
-<div [class]="!widget ? 'container-xl full-height' : ''">
-  <h1 *ngIf="!widget" class="float-left">
-    <span i18n="lightning.top-100-connectivity">Top 100 nodes connectivity ranking</span>
-  </h1>
 
-  <div [class]="widget ? 'widget' : 'full'">
+
+<div class="container-xl" style="min-height: 335px" [ngClass]="{'widget': widget, 'full-height': !widget}">
+  <h1 *ngIf="!widget" class="float-left" i18n="lightning.liquidity-ranking">Liquidity Ranking</h1>
+
+  <div class="clearfix"></div>
+
+  <div style="min-height: 295px">
     <table class="table table-borderless">
       <thead>
-        <th class="rank"></th>
-        <th class="alias text-left" i18n="nodes.alias">Alias</th>
-        <th class="channels text-right" i18n="node.channels">Channels</th>
-        <th *ngIf="!widget" class="capacity text-right" i18n="lightning.liquidity">Liquidity</th>
-        <th *ngIf="!widget" class="timestamp-first text-left" i18n="transaction.first-seen|Transaction first seen">First seen</th>
-        <th *ngIf="!widget" class="timestamp-update text-left" i18n="lightning.last_update">Last update</th>
-        <th *ngIf="!widget" class="location text-right" i18n="lightning.location">Location</th>
+        <th class="pool text-left" i18n="nodes.alias" [ngClass]="{'widget': widget}">Alias</th>
+        <th class="liquidity text-right" i18n="node.channels">Channels</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell channels text-right" i18n="lightning.channels">Capacity</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell text-right" i18n="node.liquidity">{{ currency$ | async }}</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell timestamp text-right" i18n="transaction.first-seen|Transaction first seen">First seen</th>
+        <th *ngIf="!widget" class="d-none d-md-table-cell timestamp text-right" i18n="lightning.last_update">Last update</th>
+        <th class="geolocation d-table-cell text-right" i18n="lightning.location">Location</th>
       </thead>
-      <tbody *ngIf="topNodesPerChannels$ | async as nodes; else skeleton">
-        <tr *ngFor="let node of nodes; let i = index;">
-          <td class="rank text-left">
-            {{ i + 1 }}
+      <tbody *ngIf="topNodesPerChannels$ | async as nodes">
+        <tr *ngFor="let node of nodes;">
+          <td class="pool text-left">
+            <div class="tooltip-custom d-block w-100">
+              <a class="link d-block w-100" [routerLink]="['/lightning/node' | relativeUrl, node.publicKey]">
+                <span class="pool-name w-100">{{ node.alias }}</span>
+              </a>
+            </div>
           </td>
-          <td class="alias text-left">
-            <a [routerLink]="['/lightning/node' | relativeUrl, node.publicKey]">{{ node.alias }}</a>
+          <td class="text-right">
+            {{ node.channels ? (node.channels | number) : '~' }}
           </td>
-          <td class="channels text-right">
-            {{ node.channels | number }}
-          </td>
-          <td *ngIf="!widget" class="capacity text-right">
+          <td *ngIf="!widget" class="d-none d-md-table-cell capacity text-right">
             <app-amount [satoshis]="node.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
           </td>
-          <td *ngIf="!widget" class="timestamp-first text-left">
+          <td *ngIf="!widget" class="fiat d-none d-md-table-cell text-right">
+            <app-fiat [value]="node.capacity"></app-fiat>
+          </td>
+          <td *ngIf="!widget" class="d-none d-md-table-cell text-right">
             <app-timestamp [customFormat]="'yyyy-MM-dd'" [unixTime]="node.firstSeen" [hideTimeSince]="true"></app-timestamp>
           </td>
-          <td *ngIf="!widget" class="timestamp-update text-left">
+          <td *ngIf="!widget" class="d-none d-md-table-cell text-right">
             <app-timestamp [customFormat]="'yyyy-MM-dd'" [unixTime]="node.updatedAt" [hideTimeSince]="true"></app-timestamp>
           </td>
-          <td *ngIf="!widget" class="location text-right text-truncate">
+          <td class="geolocation d-table-cell text-right text-truncate">
             <app-geolocation [data]="node.geolocation" [type]="'list-isp'"></app-geolocation>
           </td>
-      </tr>
+        </tr>
       </tbody>
-      <ng-template #skeleton>
-        <tbody>
-          <tr *ngFor="let item of skeletonRows">
-            <td class="rank text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td class="alias text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td class="channels text-right">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="capacity text-right">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="timestamp-first text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="timestamp-update text-left">
-              <span class="skeleton-loader"></span>
-            </td>
-            <td *ngIf="!widget" class="location text-right text-truncate">
-              <span class="skeleton-loader"></span>
-            </td>
-          </tr>
-        </tbody>
-      </ng-template>
     </table>
+
+    <ng-template [ngIf]="!widget">
+      <div class="clearfix"></div>
+      <br>
+    </ng-template>
   </div>
-</div>
+  
+</div>

frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss

@@ -1,91 +1,54 @@
 .container-xl {
   max-width: 1400px;
-  padding-bottom: 100px;
-  @media (min-width: 960px) {
-    padding-left: 50px;
-    padding-right: 50px;
-  }
+}
+.container-xl.widget {
+  padding-right: 0px;
+  padding-left: 0px;
+  padding-bottom: 0px;
 }
 
-.table td, .table th {
-  padding: 0.5rem;
+tr, td, th {
+  border: 0px;
+  padding-top: 0.65rem !important;
+  padding-bottom: 0.7rem !important;
 }
 
-.full .rank {
-  width: 5%;
-}
-.widget .rank {
-  @media (min-width: 960px) {
-    width: 13%;
-  }
-  @media (max-width: 960px) {
-    padding-left: 0px;
-    padding-right: 0px;
-  }
+.clear-link {
+  color: white;
 }
 
-.full .alias {
-  width: 20%;
+.pool {
+  width: 15%;
+  @media (max-width: 576px) {
+    width: 75%;
+  }
   overflow: hidden;
   text-overflow: ellipsis;
-  max-width: 350px;
-  @media (max-width: 960px) {
-    width: 40%;
-    max-width: 500px;
-  }
+  white-space: nowrap;
+  max-width: 160px;
 }
-.widget .alias {
-  width: 60%;
-  overflow: hidden;
+.pool-name {
+  display: inline-block;
+  vertical-align: text-top;
   text-overflow: ellipsis;
-  max-width: 350px;
-  @media (max-width: 960px) {
-    max-width: 175px;
-  }
+  overflow: hidden;
+}
+.pool.widget {
+  width: 45%;
 }
 
-.full .capacity {
+.liquidity {
   width: 10%;
-  @media (max-width: 960px) {
-    width: 30%;
-  }
-}
-.widget .capacity {
-  width: 32%;
-  @media (max-width: 960px) {
-    padding-left: 0px;
-    padding-right: 0px;
+  @media (max-width: 576px) {
+    width: 25%;
   }
 }
 
-.full .channels {
-  width: 15%;
-  padding-right: 50px;
-  @media (max-width: 960px) {
-    display: none;
+.geolocation {
+  @media (min-width: 768px) and (max-width: 991px) {
+    display: none !important;
   }
-}
-
-.full .timestamp-first {
-  width: 10%;
-  @media (max-width: 960px) {
-    display: none;
-  }
-}
-
-.full .timestamp-update {
-  width: 10%;
-  @media (max-width: 960px) {
-    display: none;
-  }
-}
-
-.full .location {
-  width: 15%;
-  @media (max-width: 960px) {
-    width: 30%;
-  }
-  @media (max-width: 600px) {
-    display: none;
+  @media (max-width: 575px) {
+    display: none !important;
   }
 }

frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts

@@ -1,7 +1,7 @@
 import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core';
 import { map, Observable } from 'rxjs';
 import { INodesRanking, ITopNodesPerChannels } from '../../../interfaces/node-api.interface';
-import { isMobile } from '../../../shared/common.utils';
+import { StateService } from '../../../services/state.service';
 import { GeolocationData } from '../../../shared/components/geolocation/geolocation.component';
 import { LightningApiService } from '../../lightning-api.service';
 
@@ -17,13 +17,17 @@ export class TopNodesPerChannels implements OnInit {
   
   topNodesPerChannels$: Observable<ITopNodesPerChannels[]>;
   skeletonRows: number[] = [];
+  currency$: Observable<string>;
 
   constructor(
     private apiService: LightningApiService,
+    private stateService: StateService,
   ) {}
 
   ngOnInit(): void {
-    for (let i = 1; i <= (this.widget ? (isMobile() ? 8 : 7) : 100); ++i) {
+    this.currency$ = this.stateService.fiatCurrency$;
+    
+    for (let i = 1; i <= (this.widget ? 6 : 100); ++i) {
       this.skeletonRows.push(i);
     }
 
@@ -44,7 +48,15 @@ export class TopNodesPerChannels implements OnInit {
     } else {
       this.topNodesPerChannels$ = this.nodes$.pipe(
         map((ranking) => {
-          return ranking.topByChannels.slice(0, isMobile() ? 8 : 7);
+          for (const i in ranking.topByChannels) {
+            ranking.topByChannels[i].geolocation = <GeolocationData>{
+              country: ranking.topByChannels[i].country?.en,
+              city: ranking.topByChannels[i].city?.en,
+              subdivision: ranking.topByChannels[i].subdivision?.en,
+              iso: ranking.topByChannels[i].iso_code,
+            };
+          }
+          return ranking.topByChannels.slice(0, 6);
         })
       );
     }

frontend/src/app/services/cache.service.ts

@@ -17,6 +17,7 @@ export class CacheService {
 
   txCache: { [txid: string]: Transaction } = {};
 
+  network: string;
   blockCache: { [height: number]: BlockExtended } = {};
   blockLoading: { [height: number]: boolean } = {};
   copiesInBlockQueue: { [height: number]: number } = {};
@@ -33,6 +34,10 @@ export class CacheService {
     this.stateService.chainTip$.subscribe((height) => {
       this.tip = height;
     });
+    this.stateService.networkChanged$.subscribe((network) => {
+      this.network = network;
+      this.resetBlockCache();
+    });
   }
 
   setTxCache(transactions) {
@@ -68,15 +73,17 @@ export class CacheService {
       } catch (e) {
         console.log("failed to load blocks: ", e.message);
       }
-      for (let i = 0; i < chunkSize; i++) {
-        delete this.blockLoading[maxHeight - i];
-      }
       if (result && result.length) {
         result.forEach(block => {
-          this.addBlockToCache(block);
-          this.loadedBlocks$.next(block);
+          if (this.blockLoading[block.height]) {
+            this.addBlockToCache(block);
+            this.loadedBlocks$.next(block);
+          }
         });
       }
+      for (let i = 0; i < chunkSize; i++) {
+        delete this.blockLoading[maxHeight - i];
+      }
       this.clearBlocks();
     } else {
       this.bumpBlockPriority(height);
@@ -104,6 +111,14 @@ export class CacheService {
     }
   }
 
+  // remove all blocks from the cache
+  resetBlockCache() {
+    this.blockCache = {};
+    this.blockLoading = {};
+    this.copiesInBlockQueue = {};
+    this.blockPriorities = [];
+  }
+
   getCachedBlock(height) {
     return this.blockCache[height];
   }

frontend/src/app/services/price.service.ts

@@ -70,7 +70,7 @@ export class PriceService {
   }
 
   getBlockPrice$(blockTimestamp: number, singlePrice = false): Observable<Price | undefined> {
-    if (this.stateService.env.BASE_MODULE !== 'mempool') {
+    if (this.stateService.env.BASE_MODULE !== 'mempool' || !this.stateService.env.HISTORICAL_PRICE) {
       return of(undefined);
     }
 

frontend/src/app/services/state.service.ts

@@ -43,6 +43,7 @@ export interface Env {
   MAINNET_BLOCK_AUDIT_START_HEIGHT: number;
   TESTNET_BLOCK_AUDIT_START_HEIGHT: number;
   SIGNET_BLOCK_AUDIT_START_HEIGHT: number;
+  HISTORICAL_PRICE: boolean;
 }
 
 const defaultEnv: Env = {
@@ -72,6 +73,7 @@ const defaultEnv: Env = {
   'MAINNET_BLOCK_AUDIT_START_HEIGHT': 0,
   'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0,
   'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0,
+  'HISTORICAL_PRICE': true,
 };
 
 @Injectable({

frontend/sync-assets.js

@@ -51,7 +51,7 @@ function downloadMiningPoolLogos() {
     response.on('data', (fragments) => {
       chunks_of_data.push(fragments);
     });
-  
+
     response.on('end', () => {
       let response_body = Buffer.concat(chunks_of_data);
       try {
@@ -63,7 +63,7 @@ function downloadMiningPoolLogos() {
         console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
       }
     });
-  
+
     response.on('error', (error) => {
       throw new Error(error);
     });
@@ -81,6 +81,9 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
 const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
 const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
 
+const promoVideo = PATH + 'mempool-promo.mp4';
+const promoVideoUrl = 'https://raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4';
+
 console.log('Downloading assets');
 download(PATH + 'assets.json', assetsJsonUrl);
 console.log('Downloading assets minimal');
@@ -89,5 +92,9 @@ console.log('Downloading testnet assets');
 download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl);
 console.log('Downloading testnet assets minimal');
 download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl);
+if (!fs.existsSync(promoVideo)) {
+  console.log('Downloading promo video');
+  download(promoVideo, promoVideoUrl);
+}
 console.log('Downloading mining pool logos');
 downloadMiningPoolLogos();

production/bitcoin.conf

@@ -1,6 +1,7 @@
 datadir=/bitcoin
 server=1
 txindex=1
+coinstatsindex=1
 listen=1
 discover=1
 par=16