From 9f4107319fe002600cd49164b31d604341bddf9d Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 7 Jan 2023 11:17:30 +0100
Subject: [PATCH 01/25] Fetch coinbase tx with core when esplora fails to do so

---
 backend/src/api/blocks.ts            | 23 +++++++++++++++++------
 backend/src/api/transaction-utils.ts | 16 ++++++++++++++--
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 472ef48ef..8292fe241 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -101,12 +101,23 @@ class Blocks {
           transactions.push(tx);
           transactionsFetched++;
         } catch (e) {
-          if (i === 0) {
-            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
-            logger.err(msg);
-            throw new Error(msg);
-          } else {
-            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+          try {
+            if (config.MEMPOOL.BACKEND === 'esplora') {
+              // Try again with core
+              const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true);
+              transactions.push(tx);
+              transactionsFetched++;
+            } else {
+              throw e;
+            }
+          } catch (e) {
+            if (i === 0) {
+              const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+              logger.err(msg);
+              throw new Error(msg);
+            } else {
+              logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+            }
           }
         }
       }
diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts
index 5b92cea5f..5da168584 100644
--- a/backend/src/api/transaction-utils.ts
+++ b/backend/src/api/transaction-utils.ts
@@ -3,6 +3,7 @@ import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import config from '../config';
 import { Common } from './common';
+import bitcoinClient from './bitcoin/bitcoin-client';
 
 class TransactionUtils {
   constructor() { }
@@ -21,8 +22,19 @@ class TransactionUtils {
     };
   }
 
-  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
-    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+  /**
+   * @param txId 
+   * @param addPrevouts 
+   * @param lazyPrevouts 
+   * @param forceCore - See https://github.com/mempool/mempool/issues/2904
+   */
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<TransactionExtended> {
+    let transaction: IEsploraApi.Transaction;
+    if (forceCore === true) {
+      transaction  = await bitcoinClient.$getRawTransaction(txId, true);
+    } else {
+      transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+    }
     return this.extendTransaction(transaction);
   }
 

From d987669b1e3b419b4237ea69b3001e44551192d4 Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Sun, 8 Jan 2023 12:56:27 +0900
Subject: [PATCH 02/25] ops: fix mysql_credentials path in install/build
 scripts

---
 production/install           | 4 +++-
 production/mempool-build-all | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/production/install b/production/install
index ae28c2cdc..656143714 100755
--- a/production/install
+++ b/production/install
@@ -251,6 +251,7 @@ MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
 MEMPOOL_HOME=/mempool
 MEMPOOL_USER=mempool
 MEMPOOL_GROUP=mempool
+MEMPOOL_MYSQL_CREDENTIALS="${MEMPOOL_HOME}/.mysql_credentials"
 # name of Tor hidden service in torrc
 MEMPOOL_TOR_HS=mempool
 
@@ -1869,7 +1870,7 @@ grant all on mempool_bisq.* to '${MEMPOOL_BISQ_USER}'@'localhost' identified by
 _EOF_
 
 echo "[*] save MySQL credentials"
-cat > ${MEMPOOL_HOME}/mysql_credentials << _EOF_
+cat > "${MEMPOOL_MYSQL_CREDENTIALS}" << _EOF_
 declare -x MEMPOOL_MAINNET_USER="${MEMPOOL_MAINNET_USER}"
 declare -x MEMPOOL_MAINNET_PASS="${MEMPOOL_MAINNET_PASS}"
 declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}"
@@ -1889,6 +1890,7 @@ declare -x MEMPOOL_LIQUIDTESTNET_PASS="${MEMPOOL_LIQUIDTESTNET_PASS}"
 declare -x MEMPOOL_BISQ_USER="${MEMPOOL_BISQ_USER}"
 declare -x MEMPOOL_BISQ_PASS="${MEMPOOL_BISQ_PASS}"
 _EOF_
+chown "${MEMPOOL_USER}:${MEMPOOL_GROUP}" "${MEMPOOL_MYSQL_CREDENTIALS}"
 
 ##### nginx
 
diff --git a/production/mempool-build-all b/production/mempool-build-all
index 048aeefdc..a4cb650d7 100755
--- a/production/mempool-build-all
+++ b/production/mempool-build-all
@@ -12,7 +12,7 @@ ELEMENTS_RPC_USER=$(grep '^rpcuser' /elements/elements.conf | cut -d '=' -f2)
 ELEMENTS_RPC_PASS=$(grep '^rpcpassword' /elements/elements.conf | cut -d '=' -f2)
 
 # get mysql credentials
-MYSQL_CRED_FILE=${HOME}/mempool/mysql_credentials
+MYSQL_CRED_FILE=${HOME}/.mysql_credentials
 if [ -f "${MYSQL_CRED_FILE}" ];then
     . ${MYSQL_CRED_FILE}
 fi

From 472d00a0670c1d3fea08c9ef97802846e3061e8a Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Sun, 8 Jan 2023 13:01:05 +0900
Subject: [PATCH 03/25] ops: add tk7 nodes to prod bitcoin.conf

---
 production/bitcoin.conf | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/production/bitcoin.conf b/production/bitcoin.conf
index 0fa1e943f..99fbaeed1 100644
--- a/production/bitcoin.conf
+++ b/production/bitcoin.conf
@@ -36,6 +36,12 @@ zmqpubrawtx=tcp://127.0.0.1:8335
 #addnode=[2401:b140:2::92:204]:8333
 #addnode=[2401:b140:2::92:205]:8333
 #addnode=[2401:b140:2::92:206]:8333
+#addnode=[2401:b140:3::92:201]:8333
+#addnode=[2401:b140:3::92:202]:8333
+#addnode=[2401:b140:3::92:203]:8333
+#addnode=[2401:b140:3::92:204]:8333
+#addnode=[2401:b140:3::92:205]:8333
+#addnode=[2401:b140:3::92:206]:8333
 
 [test]
 daemon=1
@@ -57,6 +63,12 @@ zmqpubrawtx=tcp://127.0.0.1:18335
 #addnode=[2401:b140:2::92:204]:18333
 #addnode=[2401:b140:2::92:205]:18333
 #addnode=[2401:b140:2::92:206]:18333
+#addnode=[2401:b140:3::92:201]:18333
+#addnode=[2401:b140:3::92:202]:18333
+#addnode=[2401:b140:3::92:203]:18333
+#addnode=[2401:b140:3::92:204]:18333
+#addnode=[2401:b140:3::92:205]:18333
+#addnode=[2401:b140:3::92:206]:18333
 
 [signet]
 daemon=1
@@ -78,3 +90,9 @@ zmqpubrawtx=tcp://127.0.0.1:38335
 #addnode=[2401:b140:2::92:204]:38333
 #addnode=[2401:b140:2::92:205]:38333
 #addnode=[2401:b140:2::92:206]:38333
+#addnode=[2401:b140:3::92:201]:38333
+#addnode=[2401:b140:3::92:202]:38333
+#addnode=[2401:b140:3::92:203]:38333
+#addnode=[2401:b140:3::92:204]:38333
+#addnode=[2401:b140:3::92:205]:38333
+#addnode=[2401:b140:3::92:206]:38333

From 33d6892aa4d927897b169ef3a247481984e200d9 Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Sun, 8 Jan 2023 13:05:50 +0900
Subject: [PATCH 04/25] ops: add mempool-reset-all script with ./reset symlink

---
 production/install           | 1 +
 production/mempool-reset-all | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100755 production/mempool-reset-all

diff --git a/production/install b/production/install
index 656143714..240af8dfc 100755
--- a/production/install
+++ b/production/install
@@ -1010,6 +1010,7 @@ osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_
 osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-build-all upgrade
 osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-kill-all stop
 osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-start-all start
+osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-reset-all reset
 
 
 case $OS in
diff --git a/production/mempool-reset-all b/production/mempool-reset-all
new file mode 100755
index 000000000..9081fb395
--- /dev/null
+++ b/production/mempool-reset-all
@@ -0,0 +1,3 @@
+#!/usr/bin/env zsh
+rm $HOME/backend/mempool-config.json{,.orig}
+rm $HOME/frontend/mempool-frontend-config.json{,.orig}

From 5a8d1795a662467b6b040beb3b5a1fed03d4ee97 Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Sun, 8 Jan 2023 13:11:01 +0900
Subject: [PATCH 05/25] ops: fix wildcards in mempool-reset-all script

---
 production/mempool-reset-all | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/production/mempool-reset-all b/production/mempool-reset-all
index 9081fb395..22f004610 100755
--- a/production/mempool-reset-all
+++ b/production/mempool-reset-all
@@ -1,3 +1,3 @@
 #!/usr/bin/env zsh
-rm $HOME/backend/mempool-config.json{,.orig}
-rm $HOME/frontend/mempool-frontend-config.json{,.orig}
+rm $HOME/*/backend/mempool-config.json
+rm $HOME/*/frontend/mempool-frontend-config.json

From 71d1c3de04d64f07d5929bc042a084241002f73d Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sun, 8 Jan 2023 11:24:23 +0100
Subject: [PATCH 06/25] Update `bitcoin-api-factory` so we can use core only if
 needed

---
 backend/src/api/audit.ts                       | 7 +------
 backend/src/api/bitcoin/bitcoin-api-factory.ts | 2 ++
 backend/src/api/transaction-utils.ts           | 6 ++----
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts
index 6aafc9ded..b6b36dbdc 100644
--- a/backend/src/api/audit.ts
+++ b/backend/src/api/audit.ts
@@ -1,10 +1,5 @@
 import config from '../config';
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { Common } from './common';
-import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces';
-import blocksRepository from '../repositories/BlocksRepository';
-import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
-import blocks from '../api/blocks';
+import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts
index f89d07b50..24916b97b 100644
--- a/backend/src/api/bitcoin/bitcoin-api-factory.ts
+++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts
@@ -17,4 +17,6 @@ function bitcoinApiFactory(): AbstractBitcoinApi {
   }
 }
 
+export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
+
 export default bitcoinApiFactory();
diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts
index 5da168584..fb5aeea42 100644
--- a/backend/src/api/transaction-utils.ts
+++ b/backend/src/api/transaction-utils.ts
@@ -1,9 +1,7 @@
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
-import config from '../config';
 import { Common } from './common';
-import bitcoinClient from './bitcoin/bitcoin-client';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 
 class TransactionUtils {
   constructor() { }
@@ -31,7 +29,7 @@ class TransactionUtils {
   public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<TransactionExtended> {
     let transaction: IEsploraApi.Transaction;
     if (forceCore === true) {
-      transaction  = await bitcoinClient.$getRawTransaction(txId, true);
+      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, true);
     } else {
       transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
     }

From 73c55c450cfbc31777c653980c64656b6980c59d Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sun, 8 Jan 2023 11:43:18 +0100
Subject: [PATCH 07/25] Ignore pool logo download failure as it's not a
 critical error

---
 frontend/sync-assets.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js
index 9c447bf7d..8b43f61e5 100644
--- a/frontend/sync-assets.js
+++ b/frontend/sync-assets.js
@@ -54,9 +54,13 @@ function downloadMiningPoolLogos() {
   
     response.on('end', () => {
       let response_body = Buffer.concat(chunks_of_data);
-      const poolLogos = JSON.parse(response_body.toString());
-      for (const poolLogo of poolLogos) {
-          download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url);
+      try {
+        const poolLogos = JSON.parse(response_body.toString());
+        for (const poolLogo of poolLogos) {
+            download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url);
+        }
+      } catch (e) {
+        console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
       }
     });
   

From 7f903b033112d05e5b04d63d9bfcac661c733bf5 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Wed, 21 Dec 2022 09:39:22 -0600
Subject: [PATCH 08/25] Add missing vars to docker config & readme

---
 .../__fixtures__/mempool-config.template.json  |  6 +++---
 docker/README.md                               | 18 +++++++++++++++++-
 docker/backend/mempool-config.json             |  5 ++++-
 docker/backend/start.sh                        |  5 +++++
 4 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json
index 2e9221c7a..7971b65ca 100644
--- a/backend/src/__fixtures__/mempool-config.template.json
+++ b/backend/src/__fixtures__/mempool-config.template.json
@@ -26,9 +26,9 @@
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__POOLS_JSON_URL__",
-    "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
-    "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
-    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
+    "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
+    "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
+    "TRANSACTION_INDEXING": "__MEMPOOL_TRANSACTION_INDEXING__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
diff --git a/docker/README.md b/docker/README.md
index 4061f420c..b891748d0 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -100,12 +100,18 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "BLOCK_WEIGHT_UNITS": 4000000,
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "BLOCKS_SUMMARIES_INDEXING": false,
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
     "STDOUT_LOG_MIN_PRIORITY": "info",
+    "INDEXING_BLOCKS_AMOUNT": false,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
-    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "ADVANCED_GBT_AUDIT": false,
+    "ADVANCED_GBT_MEMPOOL": false,
+    "TRANSACTION_INDEXING": false,
   },
 ```
 
@@ -125,15 +131,25 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_BLOCK_WEIGHT_UNITS: ""
       MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
       MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
+      MEMPOOL_BLOCKS_SUMMARIES_INDEXING: ""
       MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
+      MEMPOOL_INDEXING_BLOCKS_AMOUNT: ""
+      MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
+      MEMPOOL_ADVANCED_GBT_AUDIT: ""
+      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
+      MEMPOOL_TRANSACTION_INDEXING: ""
       ...
 ```
 
+`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
+
+`TRANSACTION_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
+
 <br/>
 
 `mempool-config.json`:
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json
index 378bba8db..8f5eae65e 100644
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@@ -22,7 +22,10 @@
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
     "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
-    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__
+    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
+    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
+    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
+    "TRANSACTION_INDEXING": __MEMPOOL_TRANSACTION_INDEXING__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
diff --git a/docker/backend/start.sh b/docker/backend/start.sh
index c8e2a1502..370b329ba 100755
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@@ -27,6 +27,9 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
+__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
+__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
+__MEMPOOL_TRANSACTION_INDEXING__=${MEMPOOL_TRANSACTION_INDEXING:=false}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -136,6 +139,8 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_TRANSACTION_INDEXING__!${__MEMPOOL_TRANSACTION_INDEXING__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

From 2e45dab4b7c9b248716b48849388d466e6b227ad Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Wed, 28 Dec 2022 05:28:37 -0600
Subject: [PATCH 09/25] Rename TRANSACTION_INDEXING to CPFP_INDEXING and add to
 mainnet prod config

---
 backend/mempool-config.sample.json                    | 2 +-
 backend/src/__fixtures__/mempool-config.template.json | 2 +-
 backend/src/__tests__/config.test.ts                  | 2 +-
 backend/src/api/blocks.ts                             | 4 ++--
 backend/src/api/common.ts                             | 2 +-
 backend/src/config.ts                                 | 4 ++--
 production/mempool-config.mainnet.json                | 1 +
 7 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json
index dbbc8412d..1f64214ce 100644
--- a/backend/mempool-config.sample.json
+++ b/backend/mempool-config.sample.json
@@ -27,7 +27,7 @@
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "TRANSACTION_INDEXING": false
+    "CPFP_INDEXING": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json
index 7971b65ca..e699c9458 100644
--- a/backend/src/__fixtures__/mempool-config.template.json
+++ b/backend/src/__fixtures__/mempool-config.template.json
@@ -28,7 +28,7 @@
     "POOLS_JSON_URL": "__POOLS_JSON_URL__",
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "TRANSACTION_INDEXING": "__MEMPOOL_TRANSACTION_INDEXING__"
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts
index 58cf3a214..4158d3df1 100644
--- a/backend/src/__tests__/config.test.ts
+++ b/backend/src/__tests__/config.test.ts
@@ -40,7 +40,7 @@ describe('Mempool Backend Config', () => {
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
-        TRANSACTION_INDEXING: false,
+        CPFP_INDEXING: false,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 8292fe241..1b551f0b5 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -530,7 +530,7 @@ class Blocks {
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock['height'] - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
-              if (config.MEMPOOL.TRANSACTION_INDEXING) {
+              if (config.MEMPOOL.CPFP_INDEXING) {
                 await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
               }
             }
@@ -558,7 +558,7 @@ class Blocks {
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
           }
-          if (config.MEMPOOL.TRANSACTION_INDEXING) {
+          if (config.MEMPOOL.CPFP_INDEXING) {
             this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
           }
         }
diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts
index 621f021ba..f0c5c6b88 100644
--- a/backend/src/api/common.ts
+++ b/backend/src/api/common.ts
@@ -190,7 +190,7 @@ export class Common {
   static cpfpIndexingEnabled(): boolean {
     return (
       Common.indexingEnabled() &&
-      config.MEMPOOL.TRANSACTION_INDEXING === true
+      config.MEMPOOL.CPFP_INDEXING === true
     );
   }
 
diff --git a/backend/src/config.ts b/backend/src/config.ts
index e97deb5e5..fb06c84fb 100644
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -31,7 +31,7 @@ interface IConfig {
     POOLS_JSON_TREE_URL: string,
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
-    TRANSACTION_INDEXING: boolean;
+    CPFP_INDEXING: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -152,7 +152,7 @@ const defaults: IConfig = {
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
-    'TRANSACTION_INDEXING': false,
+    'CPFP_INDEXING': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json
index ab2fa69c1..fc22e6b03 100644
--- a/production/mempool-config.mainnet.json
+++ b/production/mempool-config.mainnet.json
@@ -10,6 +10,7 @@
     "POLL_RATE_MS": 1000,
     "INDEXING_BLOCKS_AMOUNT": -1,
     "BLOCKS_SUMMARIES_INDEXING": true,
+    "CPFP_INDEXING": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": false,
     "USE_SECOND_NODE_FOR_MINFEE": true

From f5922b7b71e3b81552e1461bdcfd03e514e2082a Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 9 Jan 2023 10:34:55 -0600
Subject: [PATCH 10/25] Swap TRANSACTION_INDEXING for CPFP_INDEXING in docker
 config

---
 docker/README.md                   | 6 +++---
 docker/backend/mempool-config.json | 2 +-
 docker/backend/start.sh            | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index b891748d0..0c53cee97 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -111,7 +111,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "TRANSACTION_INDEXING": false,
+    "CPFP_INDEXING": false,
   },
 ```
 
@@ -142,13 +142,13 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_POOLS_JSON_TREE_URL: ""
       MEMPOOL_ADVANCED_GBT_AUDIT: ""
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
-      MEMPOOL_TRANSACTION_INDEXING: ""
+      MEMPOOL_CPFP_INDEXING: ""
       ...
 ```
 
 `ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
 
-`TRANSACTION_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
+`CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
 
 <br/>
 
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json
index 8f5eae65e..2e3826f1d 100644
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@@ -25,7 +25,7 @@
     "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
-    "TRANSACTION_INDEXING": __MEMPOOL_TRANSACTION_INDEXING__
+    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
diff --git a/docker/backend/start.sh b/docker/backend/start.sh
index 370b329ba..b5b6e863d 100755
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@@ -29,7 +29,7 @@ __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubuserconte
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
-__MEMPOOL_TRANSACTION_INDEXING__=${MEMPOOL_TRANSACTION_INDEXING:=false}
+__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -140,7 +140,7 @@ sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REI
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_TRANSACTION_INDEXING__!${__MEMPOOL_TRANSACTION_INDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

From 649d14011e1b9c81edf5c76d66ffb23591523d7e Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 9 Jan 2023 10:35:25 -0600
Subject: [PATCH 11/25] Remove cpfp_indexing=true from mainnet prod config

---
 production/mempool-config.mainnet.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json
index fc22e6b03..ab2fa69c1 100644
--- a/production/mempool-config.mainnet.json
+++ b/production/mempool-config.mainnet.json
@@ -10,7 +10,6 @@
     "POLL_RATE_MS": 1000,
     "INDEXING_BLOCKS_AMOUNT": -1,
     "BLOCKS_SUMMARIES_INDEXING": true,
-    "CPFP_INDEXING": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": false,
     "USE_SECOND_NODE_FOR_MINFEE": true

From fcd34eb87616c704fd95a91f3d9b7d03720a1b9d Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Tue, 10 Jan 2023 16:13:16 +0100
Subject: [PATCH 12/25] Add new mempool lightning nodes and format the array so
 it's human readable

---
 backend/src/api/explorer/nodes.routes.ts | 63 ++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 3 deletions(-)

diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts
index c19bde236..e2dbcb0b6 100644
--- a/backend/src/api/explorer/nodes.routes.ts
+++ b/backend/src/api/explorer/nodes.routes.ts
@@ -41,13 +41,70 @@ class NodesRoutes {
       let nodes: any[] = [];
       switch (config.MEMPOOL.NETWORK) {
         case 'testnet':
-          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
+          nodesList = [
+            '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
+            '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
+            '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
+            '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0',
+            '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3',
+            '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af',
+            '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab',
+            '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8',
+            '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605',
+            '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205',
+            '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18',
+            '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584',
+            '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c',
+            '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5',
+            '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075',
+            '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382',
+            '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1',
+            '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f',
+          ];
           break;
         case 'signet':
-          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
+          nodesList = [
+            '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
+            '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
+            '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
+            '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029',
+            '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe',
+            '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43',
+            '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991',
+            '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34',
+            '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86',
+            '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7',
+            '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761',
+            '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7',
+            '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070',
+            '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45',
+            '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097',
+            '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853',
+            '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240',
+            '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2',
+          ];
           break;
         default:
-          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
+          nodesList = [
+            '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
+            '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
+            '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',
+            '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108',
+            '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c',
+            '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5',
+            '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a',
+            '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9',
+            '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34',
+            '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf',
+            '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c',
+            '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43',
+            '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06',
+            '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e',
+            '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7',
+            '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192',
+            '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09',
+            '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57',
+          ];
       }
 
       for (let pubKey of nodesList) {

From 8ebe04baa7168deaa3348142dcbfd51e4dc26ac4 Mon Sep 17 00:00:00 2001
From: softsimon <softsimon@protonmail.com>
Date: Tue, 10 Jan 2023 21:27:26 +0400
Subject: [PATCH 13/25] Fallback alias name to pubkey

fixes #2935
---
 .../app/components/address-labels/address-labels.component.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frontend/src/app/components/address-labels/address-labels.component.ts b/frontend/src/app/components/address-labels/address-labels.component.ts
index be5bdeead..f4b3d0ca5 100644
--- a/frontend/src/app/components/address-labels/address-labels.component.ts
+++ b/frontend/src/app/components/address-labels/address-labels.component.ts
@@ -36,7 +36,9 @@ export class AddressLabelsComponent implements OnChanges {
 
   handleChannel() {
     const type = this.vout ? 'open' : 'close';
-    this.label = `Channel ${type}: ${this.channel.node_left.alias} <> ${this.channel.node_right.alias}`;
+    const leftNodeName = this.channel.node_left.alias || this.channel.node_left.public_key.substring(0, 10);
+    const rightNodeName = this.channel.node_right.alias || this.channel.node_right.public_key.substring(0, 10);
+    this.label = `Channel ${type}: ${leftNodeName} <> ${rightNodeName}`;
   }
 
   handleVin() {

From 5379ec0f308c31dc2d017b914101523d491b58d6 Mon Sep 17 00:00:00 2001
From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com>
Date: Tue, 10 Jan 2023 21:54:34 -0800
Subject: [PATCH 14/25] Expose whether Lightning is enabled on the backend

---
 backend/src/api/backend-info.ts   | 4 +++-
 backend/src/mempool.interfaces.ts | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/backend/src/api/backend-info.ts b/backend/src/api/backend-info.ts
index 57bb5fe13..fc3181524 100644
--- a/backend/src/api/backend-info.ts
+++ b/backend/src/api/backend-info.ts
@@ -2,6 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
+import config from '../config';
 
 class BackendInfo {
   private backendInfo: IBackendInfo;
@@ -22,7 +23,8 @@ class BackendInfo {
     this.backendInfo = {
       hostname: os.hostname(),
       version: versionInfo.version,
-      gitCommit: versionInfo.gitCommit
+      gitCommit: versionInfo.gitCommit,
+      lightning: config.LIGHTNING.ENABLED
     };
   }
 
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index 046083322..f79786279 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -274,6 +274,7 @@ export interface IBackendInfo {
   hostname: string;
   gitCommit: string;
   version: string;
+  lightning: boolean;
 }
 
 export interface IDifficultyAdjustment {
@@ -337,4 +338,4 @@ export interface IOldestNodes {
   updatedAt?: number,
   city?: any,
   country?: any,
-}
\ No newline at end of file
+}

From 2e5d4a6df997fb8003641e3c5868ac0f17b2f64e Mon Sep 17 00:00:00 2001
From: softsimon <softsimon@protonmail.com>
Date: Wed, 11 Jan 2023 13:37:49 +0400
Subject: [PATCH 15/25] Dark mode inputs

---
 .../search-form/search-form.component.html    |  2 +-
 .../search-form/search-form.component.scss    |  3 --
 frontend/src/styles.scss                      | 32 +++++++++++++++++--
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/frontend/src/app/components/search-form/search-form.component.html b/frontend/src/app/components/search-form/search-form.component.html
index 4e38ea6e0..b881c6ea7 100644
--- a/frontend/src/app/components/search-form/search-form.component.html
+++ b/frontend/src/app/components/search-form/search-form.component.html
@@ -5,7 +5,7 @@
       <app-search-results #searchResults [hidden]="dropdownHidden" [results]="typeAhead$ | async" (selectedResult)="selectedResult($event)"></app-search-results>
     </div>
     <div>
-      <button [disabled]="isSearching" type="submit" class="btn btn-block btn-primary">
+      <button [disabled]="isSearching" type="submit" class="btn btn-block btn-purple">
         <fa-icon *ngIf="!(isTypeaheading$ | async) else searchLoading" [icon]="['fas', 'search']" [fixedWidth]="true" i18n-title="search-form.search-title" title="Search"></fa-icon>
       </button>
     </div>
diff --git a/frontend/src/app/components/search-form/search-form.component.scss b/frontend/src/app/components/search-form/search-form.component.scss
index 7c8161196..d59acadb9 100644
--- a/frontend/src/app/components/search-form/search-form.component.scss
+++ b/frontend/src/app/components/search-form/search-form.component.scss
@@ -43,9 +43,6 @@ form {
   @media (min-width: 1200px) {
     min-width: 300px;
   }
-  input {
-    border: 0px;
-  }
   .btn {
     width: 100px;
   }
diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss
index d3d16d12e..b0cdb5ef6 100644
--- a/frontend/src/styles.scss
+++ b/frontend/src/styles.scss
@@ -115,10 +115,38 @@ body {
 }
 
 .form-control {
-  color: #495057;
+  color: #fff;
+  background-color: #2d3348;
+  border: 1px solid rgba(17, 19, 31, 0.2);
 }
+
 .form-control:focus {
-  color: #000;
+  color: #fff;
+  background-color: #2d3348;
+}
+
+.btn-purple {
+  background-color: #653b9c;
+  border-color: #653b9c;
+}
+
+.btn-purple:not(:disabled):not(.disabled):active, .btn-purple:not(:disabled):not(.disabled).active, .show > .btn-purple.dropdown-toggle {
+  color: #fff;
+  background-color: #4d2d77;
+  border-color: #472a6e;
+}
+
+.btn-purple:focus, .btn-purple.focus {
+  color: #fff;
+  background-color: #533180;
+  border-color: #4d2d77;
+  box-shadow: 0 0 0 0.2rem rgb(124 88 171 / 50%);
+}
+
+.btn-purple:hover {
+  color: #fff;
+  background-color: #533180;
+  border-color: #4d2d77;
 }
 
 .form-control.form-control-secondary {

From 0d921cf7a63ef775521cff2fd95f2745e8ee632a Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Thu, 5 Jan 2023 13:02:53 -0600
Subject: [PATCH 16/25] don't rely on blocks table for cpfp indexing progress

---
 backend/src/api/blocks.ts                    |  1 -
 backend/src/api/database-migration.ts        |  7 +++++-
 backend/src/repositories/BlocksRepository.ts | 24 +++++++++++++++-----
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 8292fe241..1376964eb 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -806,7 +806,6 @@ class Blocks {
         ancestors[vin.txid] = true;
       });
     }
-    await blocksRepository.$setCPFPIndexed(hash);
   }
 }
 
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 6e0e95699..68087fdb5 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 49;
+  private static currentVersion = 50;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -442,6 +442,11 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
       await this.updateToSchemaVersion(49);
     }
+
+    if (databaseSchemaVersion < 50 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`');
+      await this.updateToSchemaVersion(50);
+    }
   }
 
   /**
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index 78a8fcce2..5c52a8ba0 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -8,6 +8,8 @@ import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
 import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
+import bitcoinClient from '../api/bitcoin/bitcoin-client';
+import config from '../config';
 
 class BlocksRepository {
   /**
@@ -667,16 +669,26 @@ class BlocksRepository {
    */
    public async $getCPFPUnindexedBlocks(): Promise<any[]> {
     try {
-      const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`);
-      return rows;
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      const currentBlockHeight = blockchainInfo.blocks;
+      const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`);
+      const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight;
+
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+      const firstHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
+      if (firstHeight < lastHeight) {
+        const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC`, [firstHeight, lastHeight]);
+        return rows;
+      }
     } catch (e) {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
-  }
-
-  public async $setCPFPIndexed(hash: string): Promise<void> {
-    await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]);
+    return [];
   }
 
   /**

From ee95d033ac4386bbfbc99cbca0ad3ff78540a272 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Sat, 7 Jan 2023 15:51:31 -0600
Subject: [PATCH 17/25] remove slow cpfp indexing path

---
 backend/src/api/blocks.ts | 33 +++++++--------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 1376964eb..29fac7850 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -753,33 +753,14 @@ class Blocks {
 
   public async $indexCPFP(hash: string, height: number): Promise<void> {
     let transactions;
-    if (Common.blocksSummariesIndexingEnabled()) {
-      transactions = await this.$getStrippedBlockTransactions(hash);
-      const rawBlock = await bitcoinApi.$getRawBlock(hash);
-      const block = Block.fromBuffer(rawBlock);
-      const txMap = {};
-      for (const tx of block.transactions || []) {
-        txMap[tx.getId()] = tx;
-      }
-      for (const tx of transactions) {
-        // convert from bitcoinjs to esplora vin format
-        if (txMap[tx.txid]?.ins) {
-          tx.vin = txMap[tx.txid].ins.map(vin => {
-            return {
-              txid: vin.hash.slice().reverse().toString('hex')
-            };
-          });
-        }
-      }
-    } else {
-      const block = await bitcoinClient.getBlock(hash, 2);
-      transactions = block.tx.map(tx => {
-        tx.vsize = tx.weight / 4;
-        tx.fee *= 100_000_000;
-        return tx;
-      });
-    }
  
+    const block = await bitcoinClient.getBlock(hash, 2);
+    transactions = block.tx.map(tx => {
+      tx.vsize = tx.weight / 4;
+      tx.fee *= 100_000_000;
+      return tx;
+    });
+
     let cluster: TransactionStripped[] = [];
     let ancestors: { [txid: string]: boolean } = {};
     for (let i = transactions.length - 1; i >= 0; i--) {

From 7793eaecbc193567f9622d9b0c93ab193ed4ccd1 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 9 Jan 2023 08:34:33 -0600
Subject: [PATCH 18/25] fix cpfp indexing rate calculation

---
 backend/src/api/blocks.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 29fac7850..a782dbacc 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -341,6 +341,7 @@ class Blocks {
     try {
       // Get all indexed block hash
       const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
+      logger.info(`Indexing cpfp data for ${unindexedBlocks.length} blocks`);
 
       if (!unindexedBlocks?.length) {
         return;
@@ -357,7 +358,7 @@ class Blocks {
         const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
         if (elapsedSeconds > 5) {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-          const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds);
+          const blockPerSeconds = (countThisRun / elapsedSeconds);
           const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
           logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;

From b50936f001406c3465b7350a3d54cf26c517eddb Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 9 Jan 2023 10:24:12 -0600
Subject: [PATCH 19/25] compact schemas for cpfp tables

---
 backend/src/api/blocks.ts                     | 15 ++--
 backend/src/api/database-migration.ts         | 21 +++++
 backend/src/repositories/BlocksRepository.ts  |  2 +-
 backend/src/repositories/CpfpRepository.ts    | 86 +++++++++++++++++--
 .../src/repositories/TransactionRepository.ts | 53 +++++++-----
 5 files changed, 142 insertions(+), 35 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index a782dbacc..2b1f88518 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -22,12 +22,10 @@ import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import cpfpRepository from '../repositories/CpfpRepository';
-import transactionRepository from '../repositories/TransactionRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
-import { Block } from 'bitcoinjs-lib';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -753,10 +751,8 @@ class Blocks {
   }
 
   public async $indexCPFP(hash: string, height: number): Promise<void> {
-    let transactions;
- 
     const block = await bitcoinClient.getBlock(hash, 2);
-    transactions = block.tx.map(tx => {
+    const transactions = block.tx.map(tx => {
       tx.vsize = tx.weight / 4;
       tx.fee *= 100_000_000;
       return tx;
@@ -775,9 +771,12 @@ class Blocks {
         });
         const effectiveFeePerVsize = totalFee / totalVSize;
         if (cluster.length > 1) {
-          await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize);
-          for (const tx of cluster) {
-            await transactionRepository.$setCluster(tx.txid, cluster[0].txid);
+          const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
+          const equalFee = cluster.reduce((acc, tx) => {
+            return (acc && Math.round(((tx.fee || 0) / tx.vsize) * 100) / 100 === roundedEffectiveFee);
+          }, true);
+          if (!equalFee) {
+            await cpfpRepository.$saveCluster(cluster[0].txid, height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize);
           }
         }
         cluster = [];
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 68087fdb5..6d7901ffa 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -445,6 +445,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion < 50 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`');
+      await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters'));
+      await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions'));
       await this.updateToSchemaVersion(50);
     }
   }
@@ -918,6 +920,25 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateCompactCPFPTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS compact_cpfp_clusters (
+      root binary(32) NOT NULL,
+      height int(10) NOT NULL,
+      txs BLOB DEFAULT NULL,
+      fee_rate float unsigned NOT NULL,
+      PRIMARY KEY (root),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCompactTransactionsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS compact_transactions (
+      txid binary(32) NOT NULL,
+      cluster binary(32) DEFAULT NULL,
+      PRIMARY KEY (txid)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index 5c52a8ba0..6cd5d2785 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -671,7 +671,7 @@ class BlocksRepository {
     try {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
       const currentBlockHeight = blockchainInfo.blocks;
-      const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`);
+      const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from compact_cpfp_clusters`);
       const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight;
 
       let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts
index 563e6ede1..a5f17540a 100644
--- a/backend/src/repositories/CpfpRepository.ts
+++ b/backend/src/repositories/CpfpRepository.ts
@@ -1,34 +1,72 @@
+import cluster, { Cluster } from 'cluster';
+import { RowDataPacket } from 'mysql2';
 import DB from '../database';
 import logger from '../logger';
 import { Ancestor } from '../mempool.interfaces';
+import transactionRepository from '../repositories/TransactionRepository';
 
 class CpfpRepository {
-  public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
+  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
+    if (!txs[0]) {
+      return;
+    }
     try {
-      const txsJson = JSON.stringify(txs);
+      const packedTxs = Buffer.from(this.pack(txs));
       await DB.query(
         `
-          INSERT INTO cpfp_clusters(root, height, txs, fee_rate)
-          VALUE (?, ?, ?, ?)
+          INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+          VALUE (UNHEX(?), ?, ?, ?)
           ON DUPLICATE KEY UPDATE
             height = ?,
             txs = ?,
             fee_rate = ?
         `,
-        [txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height]
+        [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize]
       );
+      for (const tx of txs) {
+        await transactionRepository.$setCluster(tx.txid, clusterRoot);
+      }
     } catch (e: any) {
       logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
+  public async $getCluster(clusterRoot: string): Promise<Cluster> {
+    const [clusterRows]: any = await DB.query(
+      `
+        SELECT *
+        FROM compact_cpfp_clusters
+        WHERE root = UNHEX(?)
+      `,
+      [clusterRoot]
+    );
+    const cluster = clusterRows[0];
+    cluster.txs = this.unpack(cluster.txs);
+    return cluster;
+  }
+
   public async $deleteClustersFrom(height: number): Promise<void> {
     logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
     try {
+      const [rows] = await DB.query(
+        `
+          SELECT txs, height, root from compact_cpfp_clusters
+          WHERE height >= ?
+        `,
+        [height]
+      ) as RowDataPacket[][];
+      if (rows?.length) {
+        for (let clusterToDelete of rows) {
+          const txs = this.unpack(clusterToDelete.txs);
+          for (let tx of txs) {
+            await transactionRepository.$removeTransaction(tx.txid);
+          }
+        }
+      }
       await DB.query(
         `
-          DELETE from cpfp_clusters
+          DELETE from compact_cpfp_clusters
           WHERE height >= ?
         `,
         [height]
@@ -38,6 +76,42 @@ class CpfpRepository {
       throw e;
     }
   }
+
+  public pack(txs: Ancestor[]): ArrayBuffer {
+    const buf = new ArrayBuffer(44 * txs.length);
+    const view = new DataView(buf);
+    txs.forEach((tx, i) => {
+      const offset = i * 44;
+      for (let x = 0; x < 32; x++) {
+        // store txid in little-endian
+        view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16));
+      }
+      view.setUint32(offset + 32, tx.weight);
+      view.setBigUint64(offset + 36, BigInt(Math.round(tx.fee)));
+    });
+    return buf;
+  }
+
+  public unpack(buf: Buffer): Ancestor[] {
+    if (!buf) {
+      return [];
+    }
+
+    const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+    const txs: Ancestor[] = [];
+    const view = new DataView(arrayBuffer);
+    for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
+      const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
+      const weight = view.getUint32(offset + 32);
+      const fee = Number(view.getBigUint64(offset + 36));
+      txs.push({
+        txid,
+        weight,
+        fee
+      });
+    }
+    return txs;
+  }
 }
 
 export default new CpfpRepository();
\ No newline at end of file
diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts
index 74debb833..2d05f0e14 100644
--- a/backend/src/repositories/TransactionRepository.ts
+++ b/backend/src/repositories/TransactionRepository.ts
@@ -1,6 +1,7 @@
 import DB from '../database';
 import logger from '../logger';
 import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+import cpfpRepository from './CpfpRepository';
 
 interface CpfpSummary {
   txid: string;
@@ -12,20 +13,20 @@ interface CpfpSummary {
 }
 
 class TransactionRepository {
-  public async $setCluster(txid: string, cluster: string): Promise<void> {
+  public async $setCluster(txid: string, clusterRoot: string): Promise<void> {
     try {
       await DB.query(
         `
-          INSERT INTO transactions
+          INSERT INTO compact_transactions
           (
             txid,
             cluster
           )
-          VALUE (?, ?)
+          VALUE (UNHEX(?), UNHEX(?))
           ON DUPLICATE KEY UPDATE
-            cluster = ?
+            cluster = UNHEX(?)
         ;`,
-        [txid, cluster, cluster]
+        [txid, clusterRoot, clusterRoot]
       );
     } catch (e: any) {
       logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
@@ -35,18 +36,19 @@ class TransactionRepository {
 
   public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
     try {
-      let query = `
-        SELECT *
-        FROM transactions
-        LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster
-        WHERE transactions.txid = ?
-      `;
-      const [rows]: any = await DB.query(query, [txid]);
-      if (rows.length) {
-        rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[];
-        if (rows[0]?.txs?.length) {
-          return this.convertCpfp(rows[0]);
-        }
+      const [txRows]: any = await DB.query(
+        `
+          SELECT HEX(txid) as id, HEX(cluster) as root
+          FROM compact_transactions
+          WHERE txid = UNHEX(?)
+        `,
+        [txid]
+      );
+      if (txRows.length && txRows[0].root != null) {
+        const txid = txRows[0].id.toLowerCase();
+        const clusterId = txRows[0].root.toLowerCase();
+        const cluster = await cpfpRepository.$getCluster(clusterId);
+        return this.convertCpfp(txid, cluster);
       }
     } catch (e) {
       logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
@@ -54,12 +56,23 @@ class TransactionRepository {
     }
   }
 
-  private convertCpfp(cpfp: CpfpSummary): CpfpInfo {
+  public async $removeTransaction(txid: string): Promise<void> {
+    await DB.query(
+      `
+        DELETE FROM compact_transactions
+        WHERE txid = UNHEX(?)
+      `,
+      [txid]
+    );
+  }
+
+  private convertCpfp(txid, cluster): CpfpInfo {
     const descendants: Ancestor[] = [];
     const ancestors: Ancestor[] = [];
     let matched = false;
-    for (const tx of cpfp.txs) {
-      if (tx.txid === cpfp.txid) {
+
+    for (const tx of cluster.txs) {
+      if (tx.txid === txid) {
         matched = true;
       } else if (!matched) {
         descendants.push(tx);

From 01c96f80f9f99fda558ec9df93ca63f7f88b1c6a Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 9 Jan 2023 10:25:25 -0600
Subject: [PATCH 20/25] add cpfp progress marker to avoid reindexing early
 blocks

---
 backend/src/api/blocks.ts                  |  4 +++-
 backend/src/repositories/CpfpRepository.ts | 28 ++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 2b1f88518..c70bb67f7 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -350,7 +350,7 @@ class Blocks {
       let countThisRun = 0;
       let timer = new Date().getTime() / 1000;
       const startedAt = new Date().getTime() / 1000;
-
+      let lastHeight;
       for (const block of unindexedBlocks) {
         // Logging
         const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
@@ -365,11 +365,13 @@ class Blocks {
 
         await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block
 
+        lastHeight = block.height;
         // Logging
         count++;
         countThisRun++;
       }
       if (count > 0) {
+        await cpfpRepository.$insertProgressMarker(lastHeight);
         logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
       } else {
         logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts
index a5f17540a..6d091437c 100644
--- a/backend/src/repositories/CpfpRepository.ts
+++ b/backend/src/repositories/CpfpRepository.ts
@@ -77,6 +77,34 @@ class CpfpRepository {
     }
   }
 
+  // insert a dummy row to mark that we've indexed as far as this block
+  public async $insertProgressMarker(height: number): Promise<void> {
+    try {
+      const [rows]: any = await DB.query(
+        `
+          SELECT root
+          FROM compact_cpfp_clusters
+          WHERE height = ?
+        `,
+        [height]
+      );
+      if (!rows?.length) {
+        const rootBuffer = Buffer.alloc(32);
+        rootBuffer.writeInt32LE(height);
+        await DB.query(
+          `
+            INSERT INTO compact_cpfp_clusters(root, height, fee_rate)
+            VALUE (?, ?, ?)
+          `,
+          [rootBuffer, height, 0]
+        );
+      }
+    } catch (e: any) {
+      logger.err(`Cannot insert cpfp progress marker. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public pack(txs: Ancestor[]): ArrayBuffer {
     const buf = new ArrayBuffer(44 * txs.length);
     const view = new DataView(buf);

From fcd047f30231ff91eb134cc7188bf0e46fedc4cf Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 9 Jan 2023 10:25:56 -0600
Subject: [PATCH 21/25] remove redundant fields from CPFP interfaces

---
 .../src/repositories/TransactionRepository.ts |  1 -
 .../transaction/transaction.component.ts      | 27 +++++++------------
 .../src/app/interfaces/node-api.interface.ts  |  1 -
 3 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts
index 2d05f0e14..fc6af973d 100644
--- a/backend/src/repositories/TransactionRepository.ts
+++ b/backend/src/repositories/TransactionRepository.ts
@@ -83,7 +83,6 @@ class TransactionRepository {
     return {
       descendants,
       ancestors,
-      effectiveFeePerVsize: cpfp.fee_rate
     };
   }
 }
diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts
index 575c00637..81ae2f055 100644
--- a/frontend/src/app/components/transaction/transaction.component.ts
+++ b/frontend/src/app/components/transaction/transaction.component.ts
@@ -131,26 +131,17 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
           this.cpfpInfo = null;
           return;
         }
-        if (cpfpInfo.effectiveFeePerVsize) {
-          this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize;
-        } else {
-          const lowerFeeParents = cpfpInfo.ancestors.filter(
-            (parent) => parent.fee / (parent.weight / 4) < this.tx.feePerVsize
-          );
-          let totalWeight =
-            this.tx.weight +
-            lowerFeeParents.reduce((prev, val) => prev + val.weight, 0);
-          let totalFees =
-            this.tx.fee +
-            lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
 
-          if (cpfpInfo?.bestDescendant) {
-            totalWeight += cpfpInfo?.bestDescendant.weight;
-            totalFees += cpfpInfo?.bestDescendant.fee;
-          }
+        const relatives = [...cpfpInfo.ancestors, ...cpfpInfo.descendants || [cpfpInfo.bestDescendant]];
+        let totalWeight =
+          this.tx.weight +
+          relatives.reduce((prev, val) => prev + val.weight, 0);
+        let totalFees =
+          this.tx.fee +
+          relatives.reduce((prev, val) => prev + val.fee, 0);
+
+        this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4);
 
-          this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4);
-        }
         if (!this.tx.status.confirmed) {
           this.stateService.markBlock$.next({
             txFeePerVSize: this.tx.effectiveFeePerVsize,
diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts
index 2e6b94988..f72886870 100644
--- a/frontend/src/app/interfaces/node-api.interface.ts
+++ b/frontend/src/app/interfaces/node-api.interface.ts
@@ -24,7 +24,6 @@ export interface CpfpInfo {
   ancestors: Ancestor[];
   descendants?: Ancestor[];
   bestDescendant?: BestDescendant | null;
-  effectiveFeePerVsize?: number;
 }
 
 export interface DifficultyAdjustment {

From 8de3fd098879d677c9c337465117fcfa5743b705 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Tue, 10 Jan 2023 17:13:11 -0600
Subject: [PATCH 22/25] batch db inserts for cpfp data

---
 backend/src/api/blocks.ts                     | 16 ++--
 backend/src/repositories/CpfpRepository.ts    | 82 ++++++++++++++++++-
 .../src/repositories/TransactionRepository.ts | 24 ++++++
 3 files changed, 113 insertions(+), 9 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index c70bb67f7..69fa208b8 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -760,6 +760,8 @@ class Blocks {
       return tx;
     });
 
+    const clusters: any[] = [];
+
     let cluster: TransactionStripped[] = [];
     let ancestors: { [txid: string]: boolean } = {};
     for (let i = transactions.length - 1; i >= 0; i--) {
@@ -773,13 +775,12 @@ class Blocks {
         });
         const effectiveFeePerVsize = totalFee / totalVSize;
         if (cluster.length > 1) {
-          const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
-          const equalFee = cluster.reduce((acc, tx) => {
-            return (acc && Math.round(((tx.fee || 0) / tx.vsize) * 100) / 100 === roundedEffectiveFee);
-          }, true);
-          if (!equalFee) {
-            await cpfpRepository.$saveCluster(cluster[0].txid, height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize);
-          }
+          clusters.push({
+            root: cluster[0].txid,
+            height,
+            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
+            effectiveFeePerVsize,
+          });
         }
         cluster = [];
         ancestors = {};
@@ -789,6 +790,7 @@ class Blocks {
         ancestors[vin.txid] = true;
       });
     }
+    await cpfpRepository.$batchSaveClusters(clusters);
   }
 }
 
diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts
index 6d091437c..6fe041763 100644
--- a/backend/src/repositories/CpfpRepository.ts
+++ b/backend/src/repositories/CpfpRepository.ts
@@ -10,6 +10,15 @@ class CpfpRepository {
     if (!txs[0]) {
       return;
     }
+    // skip clusters of transactions with the same fees
+    const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
+    const equalFee = txs.reduce((acc, tx) => {
+      return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+    }, true);
+    if (equalFee) {
+      return;
+    }
+
     try {
       const packedTxs = Buffer.from(this.pack(txs));
       await DB.query(
@@ -23,8 +32,14 @@ class CpfpRepository {
         `,
         [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize]
       );
-      for (const tx of txs) {
-        await transactionRepository.$setCluster(tx.txid, clusterRoot);
+      const maxChunk = 10;
+      let chunkIndex = 0;
+      while (chunkIndex < txs.length) {
+        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => {
+          return { txid: tx.txid, cluster: clusterRoot };
+        });
+        await transactionRepository.$batchSetCluster(chunk);
+        chunkIndex += maxChunk;
       }
     } catch (e: any) {
       logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
@@ -32,6 +47,69 @@ class CpfpRepository {
     }
   }
 
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<void> {
+    try {
+      const clusterValues: any[] = [];
+      const txs: any[] = [];
+
+      for (const cluster of clusters) {
+        if (cluster.txs?.length > 1) {
+          const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
+          const equalFee = cluster.txs.reduce((acc, tx) => {
+            return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+          }, true);
+          if (!equalFee) {
+            clusterValues.push([
+              cluster.root,
+              cluster.height,
+              Buffer.from(this.pack(cluster.txs)),
+              cluster.effectiveFeePerVsize
+            ]);
+            for (const tx of cluster.txs) {
+              txs.push({ txid: tx.txid, cluster: cluster.root });
+            }
+          }
+        }
+      }
+
+      if (!clusterValues.length) {
+        return;
+      }
+
+      const maxChunk = 100;
+      let chunkIndex = 0;
+      // insert transactions in batches of up to 100 rows
+      while (chunkIndex < txs.length) {
+        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk);
+        await transactionRepository.$batchSetCluster(chunk);
+        chunkIndex += maxChunk;
+      }
+
+      chunkIndex = 0;
+      // insert clusters in batches of up to 100 rows
+      while (chunkIndex < clusterValues.length) {
+        const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk);
+        let query = `
+            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+            VALUES
+        `;
+        query += chunk.map(chunk => {
+          return (' (UNHEX(?), ?, ?, ?)');
+        }) + ';';
+        const values = chunk.flat();
+        await DB.query(
+          query,
+          values
+        );
+        chunkIndex += maxChunk;
+      }
+      return;
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getCluster(clusterRoot: string): Promise<Cluster> {
     const [clusterRows]: any = await DB.query(
       `
diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts
index fc6af973d..061617451 100644
--- a/backend/src/repositories/TransactionRepository.ts
+++ b/backend/src/repositories/TransactionRepository.ts
@@ -34,6 +34,30 @@ class TransactionRepository {
     }
   }
 
+  public async $batchSetCluster(txs): Promise<void> {
+    try {
+      let query = `
+          INSERT IGNORE INTO compact_transactions
+          (
+            txid,
+            cluster
+          )
+          VALUES
+      `;
+      query += txs.map(tx => {
+        return (' (UNHEX(?), UNHEX(?))');
+      }) + ';';
+      const values = txs.map(tx => [tx.txid, tx.cluster]).flat();
+      await DB.query(
+        query,
+        values
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
     try {
       const [txRows]: any = await DB.query(

From f0d3bb87c60b3281e970852459d80386ae537fb8 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Tue, 10 Jan 2023 17:15:55 -0600
Subject: [PATCH 23/25] handle gaps in indexed CPFP data

---
 backend/src/api/blocks.ts                    | 29 +++++++++-----------
 backend/src/repositories/BlocksRepository.ts | 24 ++++++++++------
 backend/src/repositories/CpfpRepository.ts   | 13 +++++----
 3 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 69fa208b8..9ab572401 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -338,10 +338,10 @@ class Blocks {
 
     try {
       // Get all indexed block hash
-      const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
-      logger.info(`Indexing cpfp data for ${unindexedBlocks.length} blocks`);
+      const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
+      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
 
-      if (!unindexedBlocks?.length) {
+      if (!unindexedBlockHeights?.length) {
         return;
       }
 
@@ -350,32 +350,26 @@ class Blocks {
       let countThisRun = 0;
       let timer = new Date().getTime() / 1000;
       const startedAt = new Date().getTime() / 1000;
-      let lastHeight;
-      for (const block of unindexedBlocks) {
+      for (const height of unindexedBlockHeights) {
         // Logging
+        const hash = await bitcoinApi.$getBlockHash(height);
         const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
         if (elapsedSeconds > 5) {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
           const blockPerSeconds = (countThisRun / elapsedSeconds);
-          const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
-          logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
+          logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           countThisRun = 0;
         }
 
-        await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block
+        await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block
 
-        lastHeight = block.height;
         // Logging
         count++;
         countThisRun++;
       }
-      if (count > 0) {
-        await cpfpRepository.$insertProgressMarker(lastHeight);
-        logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
-      } else {
-        logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
-      }
+      logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
     } catch (e) {
       logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
@@ -790,7 +784,10 @@ class Blocks {
         ancestors[vin.txid] = true;
       });
     }
-    await cpfpRepository.$batchSaveClusters(clusters);
+    const result = await cpfpRepository.$batchSaveClusters(clusters);
+    if (!result) {
+      await cpfpRepository.$insertProgressMarker(height);
+    }
   }
 }
 
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index 6cd5d2785..df98719b9 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -671,19 +671,25 @@ class BlocksRepository {
     try {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
       const currentBlockHeight = blockchainInfo.blocks;
-      const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from compact_cpfp_clusters`);
-      const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight;
-
-      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
       if (indexingBlockAmount <= -1) {
         indexingBlockAmount = currentBlockHeight + 1;
       }
-      const firstHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+      const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
 
-      if (firstHeight < lastHeight) {
-        const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC`, [firstHeight, lastHeight]);
-        return rows;
-      }
+      const [rows]: any[] = await DB.query(`
+        SELECT height
+        FROM compact_cpfp_clusters
+        WHERE height <= ? AND height >= ?
+        ORDER BY height DESC;
+      `, [currentBlockHeight, minHeight]);
+
+      const indexedHeights = {};
+      rows.forEach((row) => { indexedHeights[row.height] = true; });
+      const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse();
+      const unindexedHeights = allHeights.filter(x => !indexedHeights[x]);
+
+      return unindexedHeights;
     } catch (e) {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts
index 6fe041763..ce7432d5b 100644
--- a/backend/src/repositories/CpfpRepository.ts
+++ b/backend/src/repositories/CpfpRepository.ts
@@ -6,9 +6,9 @@ import { Ancestor } from '../mempool.interfaces';
 import transactionRepository from '../repositories/TransactionRepository';
 
 class CpfpRepository {
-  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
+  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<boolean> {
     if (!txs[0]) {
-      return;
+      return false;
     }
     // skip clusters of transactions with the same fees
     const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
@@ -16,7 +16,7 @@ class CpfpRepository {
       return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
     }, true);
     if (equalFee) {
-      return;
+      return false;
     }
 
     try {
@@ -41,13 +41,14 @@ class CpfpRepository {
         await transactionRepository.$batchSetCluster(chunk);
         chunkIndex += maxChunk;
       }
+      return true;
     } catch (e: any) {
       logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<void> {
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
     try {
       const clusterValues: any[] = [];
       const txs: any[] = [];
@@ -73,7 +74,7 @@ class CpfpRepository {
       }
 
       if (!clusterValues.length) {
-        return;
+        return false;
       }
 
       const maxChunk = 100;
@@ -103,7 +104,7 @@ class CpfpRepository {
         );
         chunkIndex += maxChunk;
       }
-      return;
+      return true;
     } catch (e: any) {
       logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;

From bd30f2eb12f26ccd5e55d874bb771744e7b29572 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Tue, 10 Jan 2023 17:24:18 -0600
Subject: [PATCH 24/25] migrate cpfp data from old to new schemas

---
 backend/src/api/database-migration.ts | 70 +++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 4 deletions(-)

diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 6d7901ffa..42f223417 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -2,9 +2,12 @@ import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from './common';
+import blocksRepository from '../repositories/BlocksRepository';
+import cpfpRepository from '../repositories/CpfpRepository';
+import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 50;
+  private static currentVersion = 52;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -443,11 +446,27 @@ class DatabaseMigration {
       await this.updateToSchemaVersion(49);
     }
 
-    if (databaseSchemaVersion < 50 && isBitcoin === true) {
+    if (databaseSchemaVersion < 50) {
       await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`');
+      await this.updateToSchemaVersion(50);
+    }
+
+    if (databaseSchemaVersion < 51) {
+      await this.$executeQuery('ALTER TABLE `cpfp_clusters` ADD INDEX `height` (`height`)');
+      await this.updateToSchemaVersion(51);
+    }
+
+    if (databaseSchemaVersion < 52) {
       await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters'));
       await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions'));
-      await this.updateToSchemaVersion(50);
+      try {
+        await this.$convertCompactCpfpTables();
+        await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
+        await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
+        await this.updateToSchemaVersion(52);
+      } catch(e) {
+        logger.warn('' + (e instanceof Error ? e.message : e));
+      }
     }
   }
 
@@ -925,7 +944,7 @@ class DatabaseMigration {
       root binary(32) NOT NULL,
       height int(10) NOT NULL,
       txs BLOB DEFAULT NULL,
-      fee_rate float unsigned NOT NULL,
+      fee_rate float unsigned,
       PRIMARY KEY (root),
       INDEX (height)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
@@ -959,6 +978,49 @@ class DatabaseMigration {
       logger.warn(`Unable to erase indexed data`);
     }
   }
+
+  private async $convertCompactCpfpTables(): Promise<void> {
+    try {
+      const batchSize = 250;
+      const maxHeight = await blocksRepository.$mostRecentBlockHeight() || 0;
+      const [minHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`);
+      const minHeight = (minHeightRows.length && minHeightRows[0].minHeight != null) ? minHeightRows[0].minHeight : maxHeight;
+      let height = maxHeight;
+
+      // Logging
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      while (height > minHeight) {
+        const [rows] = await DB.query(
+          `
+            SELECT * from cpfp_clusters
+            WHERE height <= ? AND height > ?
+            ORDER BY height
+          `,
+          [height, height - batchSize]
+        ) as RowDataPacket[][];
+        if (rows?.length) {
+          await cpfpRepository.$batchSaveClusters(rows.map(row => {
+            return {
+              root: row.root,
+              height: row.height,
+              txs: JSON.parse(row.txs),
+              effectiveFeePerVsize: row.fee_rate,
+            };
+          }));
+        }
+
+        const elapsed = new Date().getTime() / 1000 - timer;
+        const runningFor = new Date().getTime() / 1000 - startedAt;
+        logger.debug(`Migrated cpfp data from block ${height} to ${height - batchSize} in ${elapsed.toFixed(2)} seconds | total elapsed: ${runningFor.toFixed(2)} seconds`);
+        timer = new Date().getTime() / 1000;
+        height -= batchSize;
+      }
+    } catch (e) {
+      logger.warn(`Failed to migrate cpfp transaction data`);
+    }
+  }
 }
 
 export default new DatabaseMigration();

From 2548d2a5e9add8f72d108c4c125b1a88e051f230 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Wed, 11 Jan 2023 08:41:00 -0600
Subject: [PATCH 25/25] fix frontend js error on unconfirmed non-cpfp
 transactions

---
 .../app/components/transaction/transaction.component.ts    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts
index 81ae2f055..e01092189 100644
--- a/frontend/src/app/components/transaction/transaction.component.ts
+++ b/frontend/src/app/components/transaction/transaction.component.ts
@@ -131,8 +131,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
           this.cpfpInfo = null;
           return;
         }
-
-        const relatives = [...cpfpInfo.ancestors, ...cpfpInfo.descendants || [cpfpInfo.bestDescendant]];
+        // merge ancestors/descendants
+        const relatives = [...(cpfpInfo.ancestors || []), ...(cpfpInfo.descendants || [])];
+        if (cpfpInfo.bestDescendant && !cpfpInfo.descendants?.length) {
+          relatives.push(cpfpInfo.bestDescendant);
+        }
         let totalWeight =
           this.tx.weight +
           relatives.reduce((prev, val) => prev + val.weight, 0);