From 4ea2a8244aae18d32fda791246e347879364e3be Mon Sep 17 00:00:00 2001
From: nymkappa <vialjoris@hotmail.com>
Date: Sun, 26 Jun 2022 10:27:07 +0200
Subject: [PATCH 1/9] Remove the condition to show `All` timespan on mining
 charts

---
 .../block-fee-rates-graph/block-fee-rates-graph.component.html  | 2 +-
 .../components/block-fees-graph/block-fees-graph.component.html | 2 +-
 .../block-rewards-graph/block-rewards-graph.component.html      | 2 +-
 .../block-sizes-weights-graph.component.html                    | 2 +-
 .../app/components/hashrate-chart/hashrate-chart.component.html | 2 +-
 .../hashrates-chart-pools/hashrate-chart-pools.component.html   | 2 +-
 .../src/app/components/pool-ranking/pool-ranking.component.html | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html
index 6da8008c6..2dbe4d569 100644
--- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html
+++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html
@@ -35,7 +35,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> ALL
         </label>
       </div>
diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html
index 03197f26e..4700f8c35 100644
--- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html
+++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html
@@ -35,7 +35,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> ALL
         </label>
       </div>
diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html
index 24380d61d..ef9978191 100644
--- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html
+++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html
@@ -36,7 +36,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> ALL
         </label>
       </div>
diff --git a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html
index 5ad2c4003..0429b68c1 100644
--- a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html
+++ b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html
@@ -35,7 +35,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-sizes-weights' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-sizes-weights' | relativeUrl]"> ALL
         </label>
       </div>
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html
index e0ca6b98a..46be9e2d2 100644
--- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html
+++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html
@@ -42,7 +42,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/hashrate-difficulty' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/hashrate-difficulty' | relativeUrl]"> ALL
         </label>
       </div>
diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html
index 66ec09390..51f6abafc 100644
--- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html
+++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html
@@ -21,7 +21,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/pools-dominance' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/pools-dominance' | relativeUrl]"> ALL
         </label>
       </div>
diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html
index 2a84295c6..a4027dfa2 100644
--- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html
+++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html
@@ -66,7 +66,7 @@
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.totalBlockCount >= 157680">
           <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/pools' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.totalBlockCount > 157680">
+        <label ngbButtonLabel class="btn-primary btn-sm">
           <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/pools' | relativeUrl]"><span i18n>All</span>
         </label>
       </div>

From 1d2841b2a67942f3da15fa9e6a2abdf46a282b83 Mon Sep 17 00:00:00 2001
From: nymkappa <vialjoris@hotmail.com>
Date: Sun, 26 Jun 2022 10:50:42 +0200
Subject: [PATCH 2/9] Handle electrs error 504

---
 frontend/src/app/components/address/address.component.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html
index 6111075e4..ed3b1bd7e 100644
--- a/frontend/src/app/components/address/address.component.html
+++ b/frontend/src/app/components/address/address.component.html
@@ -130,7 +130,7 @@
       <span i18n="address.error.loading-address-data">Error loading address data.</span>
       <br>
       <ng-template #displayServerError><i class="small">({{ error.error }})</i></ng-template>
-      <ng-template [ngIf]="error.status === 413 || error.status === 405" [ngIfElse]="displayServerError">
+      <ng-template [ngIf]="error.status === 413 || error.status === 405 || error.status === 504" [ngIfElse]="displayServerError">
         <ng-container i18n="Electrum server limit exceeded error">
           <i>There many transactions on this address, more than your backend can handle. See more on <a href="/docs/faq#address-lookup-issues">setting up a stronger backend</a>.</i>
           <br><br>

From acfdc8163baa2851ee3ac37a6f8ba6c06e017c8c Mon Sep 17 00:00:00 2001
From: nymkappa <vialjoris@hotmail.com>
Date: Sat, 25 Jun 2022 12:14:32 +0200
Subject: [PATCH 3/9] Index difficulty adjustments

---
 backend/src/api/blocks.ts                     | 17 +++-
 backend/src/api/database-migration.ts         | 20 ++++-
 backend/src/api/mining.ts                     | 45 +++++++++-
 backend/src/index.ts                          |  1 +
 backend/src/indexer.ts                        |  1 +
 backend/src/mempool.interfaces.ts             |  7 ++
 backend/src/repositories/BlocksRepository.ts  | 65 +-------------
 .../DifficultyAdjustmentsRepository.ts        | 88 +++++++++++++++++++
 backend/src/routes.ts                         | 15 +++-
 ...ifficulty-adjustments-table.component.html |  4 +-
 ...difficulty-adjustments-table.components.ts | 23 +++--
 .../hashrate-chart.component.ts               |  3 +-
 frontend/src/app/services/api.service.ts      |  9 +-
 production/nginx-cache-warmer                 | 10 +++
 14 files changed, 225 insertions(+), 83 deletions(-)
 create mode 100644 backend/src/repositories/DifficultyAdjustmentsRepository.ts

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index b1a2da18f..cecdee6bd 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -19,6 +19,9 @@ import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import mining from './mining';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import difficultyAdjustment from './difficulty-adjustment';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -449,7 +452,10 @@ class Blocks {
               const newBlock = await this.$indexBlock(lastBlock['height'] - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
             }
-            logger.info(`Re-indexed 10 blocks and summaries`);
+            await mining.$indexDifficultyAdjustments();
+            await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
 
@@ -461,6 +467,15 @@ class Blocks {
       }
 
       if (block.height % 2016 === 0) {
+        if (Common.indexingEnabled()) {
+          await DifficultyAdjustmentsRepository.$saveAdjustments({
+            time: block.timestamp,
+            height: block.height,
+            difficulty: block.difficulty,
+            adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
+          });
+        }
+
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index f0f375309..8d9959cf2 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 21;
+  private static currentVersion = 22;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -226,6 +226,11 @@ class DatabaseMigration {
         await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
         await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
       }
+
+      if (databaseSchemaVersion < 22 && isBitcoin === true) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
+        await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+      }
     } catch (e) {
       throw e;
     }
@@ -513,7 +518,7 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  private getCreateRatesTableQuery(): string {
+  private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table
     return `CREATE TABLE IF NOT EXISTS rates (
       height int(10) unsigned NOT NULL,
       bisq_rates JSON NOT NULL,
@@ -539,6 +544,17 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateDifficultyAdjustmentsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS difficulty_adjustments (
+      time timestamp NOT NULL,
+      height int(10) unsigned NOT NULL,
+      difficulty double unsigned NOT NULL,
+      adjustment float NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 
diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts
index 977cbe4e8..672f0970f 100644
--- a/backend/src/api/mining.ts
+++ b/backend/src/api/mining.ts
@@ -1,4 +1,4 @@
-import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
+import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
 import BlocksRepository from '../repositories/BlocksRepository';
 import PoolsRepository from '../repositories/PoolsRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
@@ -7,6 +7,7 @@ import logger from '../logger';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import { escape } from 'mysql2';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 
 class Mining {
   constructor() {
@@ -377,6 +378,48 @@ class Mining {
     }
   }
 
+  /**
+   * Index difficulty adjustments
+   */
+  public async $indexDifficultyAdjustments(): Promise<void> {
+    const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
+    const indexedHeights = {};
+    for (const height of indexedHeightsArray) {
+      indexedHeights[height] = true;
+    }
+
+    const blocks: any = await BlocksRepository.$getBlocksDifficulty();
+
+    let currentDifficulty = 0;
+    let totalIndexed = 0;
+
+    for (const block of blocks) {
+      if (block.difficulty !== currentDifficulty) {
+        if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed
+          currentDifficulty = block.difficulty;
+          continue;          
+        }
+
+        let adjustment = block.difficulty / Math.max(1, currentDifficulty);
+        adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
+
+        await DifficultyAdjustmentsRepository.$saveAdjustments({
+          time: block.time,
+          height: block.height,
+          difficulty: block.difficulty,
+          adjustment: adjustment,
+        });
+
+        totalIndexed++;
+        currentDifficulty = block.difficulty;
+      }
+    }
+
+    if (totalIndexed > 0) {
+      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);
diff --git a/backend/src/index.ts b/backend/src/index.ts
index 566622055..a3f7d5d99 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -290,6 +290,7 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments)
         ;
     }
 
diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts
index 96cca9f7f..cfdc4a7d0 100644
--- a/backend/src/indexer.ts
+++ b/backend/src/indexer.ts
@@ -36,6 +36,7 @@ class Indexer {
         return;
       }
 
+      await mining.$indexDifficultyAdjustments();
       await this.$resetHashratesIndexingState();
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index a35dc6d76..983434564 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -224,6 +224,13 @@ export interface IDifficultyAdjustment {
   timeOffset: number;
 }
 
+export interface IndexedDifficultyAdjustment {
+  time: number; // UNIX timestamp
+  height: number; // Block height
+  difficulty: number;
+  adjustment: number;
+}
+
 export interface RewardStats {
   totalReward: number;
   totalFee: number;
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index 01b7622f3..e88ac7877 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -7,6 +7,7 @@ import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
 import BlocksSummariesRepository from './BlocksSummariesRepository';
+import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
 
 class BlocksRepository {
   /**
@@ -381,48 +382,9 @@ class BlocksRepository {
   /**
    * Return blocks difficulty
    */
-  public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
-    interval = Common.getSqlInterval(interval);
-
-    // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
-    // Basically, using temporary user defined fields, we are able to extract all
-    // difficulty adjustments from the blocks tables.
-    // This allow use to avoid indexing it in another table.
-    let query = `
-      SELECT
-      *
-      FROM
-      (
-        SELECT
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
-        IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
-          IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
-        ) AS rn
-        FROM blocks YT
-        CROSS JOIN
-        (
-          SELECT @prevStatus := -1, @rn := 1
-        ) AS var
-    `;
-
-    if (interval) {
-      query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += `
-        ORDER BY YT.height
-      ) AS t
-      WHERE t.rn = 1
-      ORDER BY t.height
-    `;
-
+  public async $getBlocksDifficulty(): Promise<object[]> {
     try {
-      const [rows]: any[] = await DB.query(query);
-
-      for (const row of rows) {
-        delete row['rn'];
-      }
-
+      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
       return rows;
     } catch (e) {
       logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
@@ -452,26 +414,6 @@ class BlocksRepository {
     }
   }
 
-  /*
-   * Check if the last 10 blocks chain is valid
-   */
-  public async $validateRecentBlocks(): Promise<boolean> {
-    try {
-      const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
-
-      for (let i = 0; i < lastBlocks.length - 1; ++i) {
-        if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
-          logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
-          return false;
-        }
-      }
-
-      return true;
-    } catch (e) {
-      return true; // Don't do anything if there is a db error
-    }
-  }
-
   /**
    * Check if the chain of block hash is valid and delete data from the stale branch if needed
    */
@@ -498,6 +440,7 @@ class BlocksRepository {
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
           await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
+          await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
         }
         ++idx;
diff --git a/backend/src/repositories/DifficultyAdjustmentsRepository.ts b/backend/src/repositories/DifficultyAdjustmentsRepository.ts
new file mode 100644
index 000000000..76324b5e6
--- /dev/null
+++ b/backend/src/repositories/DifficultyAdjustmentsRepository.ts
@@ -0,0 +1,88 @@
+import { Common } from '../api/common';
+import DB from '../database';
+import logger from '../logger';
+import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
+
+class DifficultyAdjustmentsRepository {
+  public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise<void> {
+    if (adjustment.height === 1) {
+      return;
+    }
+
+    try {
+      const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`;
+      const params: any[] = [
+        adjustment.time,
+        adjustment.height,
+        adjustment.difficulty,
+        adjustment.adjustment,
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    if (descOrder === true) {
+      query += ` ORDER BY time DESC`;
+    } else {
+      query += ` ORDER BY time`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAdjustmentsHeights(): Promise<number[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
+      return rows.map(block => block.height);
+    } catch (e: any) {
+      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
+    try {
+      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
+    } catch (e: any) {
+      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteLastAdjustment(): Promise<void> {
+    try {
+      logger.info(`Delete last difficulty adjustment from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
+    } catch (e: any) {
+      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+}
+
+export default new DifficultyAdjustmentsRepository();
+
diff --git a/backend/src/routes.ts b/backend/src/routes.ts
index b86187e4c..c0ba17a41 100644
--- a/backend/src/routes.ts
+++ b/backend/src/routes.ts
@@ -26,6 +26,7 @@ import mining from './api/mining';
 import BlocksRepository from './repositories/BlocksRepository';
 import HashratesRepository from './repositories/HashratesRepository';
 import difficultyAdjustment from './api/difficulty-adjustment';
+import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository';
 
 class Routes {
   constructor() {}
@@ -653,7 +654,7 @@ class Routes {
 
     try {
       const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
-      const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
+      const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
       const blockCount = await BlocksRepository.$blockCount(null, null);
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
@@ -730,6 +731,18 @@ class Routes {
     }
   }
 
+  public async $getDifficultyAdjustments(req: Request, res: Response) {
+    try {
+      const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlock(req: Request, res: Response) {
     try {
       const block = await blocks.$getBlock(req.params.hash);
diff --git a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html
index 5f11c2608..1d6038070 100644
--- a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html
+++ b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html
@@ -9,7 +9,7 @@
       </tr>
     </thead>
     <tbody *ngIf="(hashrateObservable$ | async) as data">
-      <tr *ngFor="let diffChange of data.difficulty">
+      <tr *ngFor="let diffChange of data">
         <td class="d-none d-md-block"><a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height
             }}</a></td>
         <td class="text-left">
@@ -17,7 +17,7 @@
         </td>
         <td class="text-right">{{ diffChange.difficultyShorten }}</td>
         <td class="text-right" [style]="diffChange.change >= 0 ? 'color: #42B747' : 'color: #B74242'">
-          {{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener }}%
+          {{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener: 2 }}%
         </td>
       </tr>
     </tbody>
diff --git a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts
index fb7d5c8f7..32ad907d3 100644
--- a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts
+++ b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts
@@ -30,27 +30,24 @@ export class DifficultyAdjustmentsTable implements OnInit {
   }
 
   ngOnInit(): void {
-    this.hashrateObservable$ = this.apiService.getHistoricalHashrate$('1y')
+    this.hashrateObservable$ = this.apiService.getDifficultyAdjustments$('3m')
       .pipe(
         map((response) => {
           const data = response.body;
           const tableData = [];
-          for (let i = data.difficulty.length - 1; i > 0; --i) {
-            const selectedPowerOfTen: any = selectPowerOfTen(data.difficulty[i].difficulty);
-            const change = (data.difficulty[i].difficulty / data.difficulty[i - 1].difficulty - 1) * 100;
-
-            tableData.push(Object.assign(data.difficulty[i], {
-              change: Math.round(change * 100) / 100,
+          for (const adjustment of data) {
+            const selectedPowerOfTen: any = selectPowerOfTen(adjustment[2]);
+            tableData.push({
+              height: adjustment[1],
+              timestamp: adjustment[0],
+              change: (adjustment[3] - 1) * 100,
               difficultyShorten: formatNumber(
-                data.difficulty[i].difficulty / selectedPowerOfTen.divider,
+                adjustment[2] / selectedPowerOfTen.divider,
                 this.locale, '1.2-2') + selectedPowerOfTen.unit
-            }));
+            });
           }
           this.isLoading = false;
-
-          return {
-            difficulty: tableData.slice(0, 6),
-          };
+          return tableData.slice(0, 6);
         }),
       );
   }
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts
index 900d6399d..1ee3c20c0 100644
--- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts
+++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts
@@ -95,6 +95,7 @@ export class HashrateChartComponent implements OnInit {
             .pipe(
               tap((response) => {
                 const data = response.body;
+
                 // We generate duplicated data point so the tooltip works nicely
                 const diffFixed = [];
                 let diffIndex = 1;
@@ -112,7 +113,7 @@ export class HashrateChartComponent implements OnInit {
                   }
 
                   while (hashIndex < data.hashrates.length && diffIndex < data.difficulty.length &&
-                    data.hashrates[hashIndex].timestamp <= data.difficulty[diffIndex].timestamp
+                    data.hashrates[hashIndex].timestamp <= data.difficulty[diffIndex].time
                   ) {
                     diffFixed.push({
                       timestamp: data.hashrates[hashIndex].timestamp,
diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts
index bd32685fd..5efa745d1 100644
--- a/frontend/src/app/services/api.service.ts
+++ b/frontend/src/app/services/api.service.ts
@@ -140,7 +140,7 @@ export class ApiService {
       this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/pools` +
       (interval !== undefined ? `/${interval}` : ''), { observe: 'response' }
     );
-  }
+  }  
 
   getPoolStats$(slug: string): Observable<PoolStat> {
     return this.httpClient.get<PoolStat>(this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/pool/${slug}`);
@@ -172,6 +172,13 @@ export class ApiService {
     return this.httpClient.get<TransactionStripped[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/block/' + hash + '/summary');
   }
 
+  getDifficultyAdjustments$(interval: string | undefined): Observable<any> {
+    return this.httpClient.get<any[]>(
+        this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/difficulty-adjustments` +
+        (interval !== undefined ? `/${interval}` : ''), { observe: 'response' }
+      );
+  }
+
   getHistoricalHashrate$(interval: string | undefined): Observable<any> {
     return this.httpClient.get<any[]>(
         this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/hashrate` +
diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer
index 366a07345..f6ec8473d 100755
--- a/production/nginx-cache-warmer
+++ b/production/nginx-cache-warmer
@@ -67,6 +67,16 @@ do for url in / \
 	'/api/v1/mining/blocks/fee-rates/2y' \
 	'/api/v1/mining/blocks/fee-rates/3y' \
 	'/api/v1/mining/blocks/fee-rates/all' \
+	'/api/v1/mining/difficulty-adjustments/24h' \
+	'/api/v1/mining/difficulty-adjustments/3d' \
+	'/api/v1/mining/difficulty-adjustments/1w' \
+	'/api/v1/mining/difficulty-adjustments/1m' \
+	'/api/v1/mining/difficulty-adjustments/3m' \
+	'/api/v1/mining/difficulty-adjustments/6m' \
+	'/api/v1/mining/difficulty-adjustments/1y' \
+	'/api/v1/mining/difficulty-adjustments/2y' \
+	'/api/v1/mining/difficulty-adjustments/3y' \
+	'/api/v1/mining/difficulty-adjustments/all' \
 
 	do
 		curl -s "https://${hostname}${url}" >/dev/null

From bbc9df486efc88c202a584793e2648c7fc1eb903 Mon Sep 17 00:00:00 2001
From: nymkappa <vialjoris@hotmail.com>
Date: Sat, 25 Jun 2022 19:50:39 +0200
Subject: [PATCH 4/9] If any indexing fails | chain of hash invalid, try again
 in 10 seconds

---
 backend/src/api/blocks.ts | 15 +++++----------
 backend/src/api/mining.ts |  3 +++
 backend/src/indexer.ts    |  8 ++++++--
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 4043a9276..631940dd2 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -295,7 +295,8 @@ class Blocks {
       }
       logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
     } catch (e) {
-      logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
     }
   }
 
@@ -367,18 +368,12 @@ class Blocks {
       logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
-      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
       loadingIndicators.setProgress('block-indexing', 100);
-      return false;
+      throw e;
     }
 
-    const chainValid = await BlocksRepository.$validateChain();
-    if (!chainValid) {
-      indexer.reindex();
-      return false;
-    }
-
-    return true;
+    return await BlocksRepository.$validateChain();
   }
 
   public async $updateBlocks() {
diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts
index 672f0970f..d69ff5cd9 100644
--- a/backend/src/api/mining.ts
+++ b/backend/src/api/mining.ts
@@ -7,6 +7,7 @@ import logger from '../logger';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import { escape } from 'mysql2';
+import indexer from '../indexer';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 
 class Mining {
@@ -263,6 +264,7 @@ class Mining {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -374,6 +376,7 @@ class Mining {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts
index 48bae00e4..8e4e7e87f 100644
--- a/backend/src/indexer.ts
+++ b/backend/src/indexer.ts
@@ -39,6 +39,8 @@ class Indexer {
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
+        setTimeout(() => this.reindex(), 10000);
         this.indexerRunning = false;
         return;
       }
@@ -49,8 +51,9 @@ class Indexer {
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
     } catch (e) {
-      this.reindex();
-      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
+      this.indexerRunning = false;
+      logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
+      setTimeout(() => this.reindex(), 10000);
     }
 
     this.indexerRunning = false;
@@ -62,6 +65,7 @@ class Indexer {
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
     } catch (e) {
       logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
     }
   }
 }

From 7154d755c1246f33eab1bdfb77d272b4234be291 Mon Sep 17 00:00:00 2001
From: Stephan Oeste <emzy@emzy.de>
Date: Wed, 6 Jul 2022 11:46:46 +0200
Subject: [PATCH 5/9] Disable confirmation prompt for rust install in prod
 install script

---
 production/install | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/production/install b/production/install
index 9f2533c61..ddb4f2a5d 100755
--- a/production/install
+++ b/production/install
@@ -1048,7 +1048,7 @@ case $OS in
     ;;
     Debian)
        echo "[*] Installing Rust from rustup.rs"
-       osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh"
+       osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"
     ;;
 esac
 
@@ -1095,15 +1095,6 @@ echo "[*] Cloning Liquid Asset Registry testnet repo from ${LIQUIDTESTNET_ASSET_
 osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
 osSudo "${ELEMENTS_USER}" git clone "${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUIDTESTNET_ASSET_REGISTRY_DB_NAME}"
 
-case $OS in
-    FreeBSD)
-    ;;
-    Debian)
-       echo "[*] Installing Rust from rustup.rs"
-       osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh"
-    ;;
-esac
-
 echo "[*] Building Liquid Electrs release binary"
 osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
 

From 4a6f3e189d60fddd2fedfc7b9544a24fecd78e10 Mon Sep 17 00:00:00 2001
From: Stephan Oeste <emzy@emzy.de>
Date: Wed, 6 Jul 2022 12:11:13 +0200
Subject: [PATCH 6/9] Change shebang in scripts to be universal for prod
 install

---
 production/electrs-start-liquid  | 2 +-
 production/electrs-start-mainnet | 2 +-
 production/electrs-start-signet  | 2 +-
 production/electrs-start-testnet | 2 +-
 production/mempool-build-all     | 2 +-
 production/mempool-start-all     | 2 +-
 production/test-nginx            | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/production/electrs-start-liquid b/production/electrs-start-liquid
index 64564e7f7..a59004478 100755
--- a/production/electrs-start-liquid
+++ b/production/electrs-start-liquid
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 cd "${HOME}/electrs"
 #source "${HOME}/.cargo/env"
 #export PATH="${HOME}/.cargo/bin:${PATH}"
diff --git a/production/electrs-start-mainnet b/production/electrs-start-mainnet
index d6f964f73..32227afd2 100755
--- a/production/electrs-start-mainnet
+++ b/production/electrs-start-mainnet
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 cd "${HOME}/electrs"
 #source "${HOME}/.cargo/env"
 #export PATH="${HOME}/.cargo/bin:${PATH}"
diff --git a/production/electrs-start-signet b/production/electrs-start-signet
index 6f7919f83..c37b670f6 100755
--- a/production/electrs-start-signet
+++ b/production/electrs-start-signet
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 cd "${HOME}/electrs"
 #source "${HOME}/.cargo/env"
 #export PATH="${HOME}/.cargo/bin:${PATH}"
diff --git a/production/electrs-start-testnet b/production/electrs-start-testnet
index 7096629dd..42e057a52 100755
--- a/production/electrs-start-testnet
+++ b/production/electrs-start-testnet
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 cd "${HOME}/electrs"
 #source $HOME/.cargo/env
 #export PATH=$HOME/.cargo/bin:$PATH
diff --git a/production/mempool-build-all b/production/mempool-build-all
index d44569fe2..503f9dbf7 100755
--- a/production/mempool-build-all
+++ b/production/mempool-build-all
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
 HOSTNAME=$(hostname)
 LOCATION=$(hostname|cut -d . -f2)
diff --git a/production/mempool-start-all b/production/mempool-start-all
index 5d5199166..d4a4ea6eb 100755
--- a/production/mempool-start-all
+++ b/production/mempool-start-all
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
 for site in mainnet liquid testnet bisq signet liquidtestnet
diff --git a/production/test-nginx b/production/test-nginx
index 2216689a4..c1cebd8d8 100755
--- a/production/test-nginx
+++ b/production/test-nginx
@@ -1,4 +1,4 @@
-#!/usr/local/bin/zsh
+#!/usr/bin/env zsh
 PROTO=https
 HOSTNAME=mempool.ninja
 URL_BASE=${PROTO}://${HOSTNAME}

From e8829e21e7a0bb8ce69e8d16f2aa44bb440230f9 Mon Sep 17 00:00:00 2001
From: Stephan Oeste <emzy@emzy.de>
Date: Wed, 6 Jul 2022 14:41:11 +0200
Subject: [PATCH 7/9] Limit make to max cpus in prod install

---
 production/install | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/production/install b/production/install
index 9f2533c61..74c6f1061 100755
--- a/production/install
+++ b/production/install
@@ -6,11 +6,13 @@ case `uname -s` in
 
     FreeBSD)
         OS=FreeBSD
+        NPROC=$(sysctl hw.ncpu | awk '{print $2}')
     ;;
 
     Linux)
         if [ "$(grep -Ei 'debian|buntu|mint' /etc/*release)" ]; then
             OS=Debian
+            NPROC=$(nproc --all)
         else
             echo "Your distribution of Linux is not yet supported by this installation script"
             exit 1
@@ -964,7 +966,7 @@ if [ "${BITCOIN_INSTALL}" = ON ];then
     echo "[*] Building Bitcoin from source repo"
     osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_REPO_NAME} && ./autogen.sh --quiet"
     osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_REPO_NAME} && MAKE=gmake CC=cc CXX=c++ CPPFLAGS=-I/usr/local/include ./configure --with-gui=no --disable-wallet --disable-tests"
-    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_REPO_NAME} && gmake -j48"
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_REPO_NAME} && gmake -j${NPROC}"
 
     echo "[*] Installing Bitcoin binaries into OS"
     osSudo "${ROOT_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_REPO_NAME} && gmake install"
@@ -1009,7 +1011,7 @@ if [ "${ELEMENTS_INSTALL}" = ON ];then
     echo "[*] Building Elements from source repo"
     osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_REPO_NAME} && ./autogen.sh --quiet"
     osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_REPO_NAME} && MAKE=gmake CC=cc CXX=c++ CPPFLAGS=-I/usr/local/include ./configure --with-gui=no --disable-wallet --disable-tests"
-    osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_REPO_NAME} && gmake -j48"
+    osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_REPO_NAME} && gmake -j${NPROC}"
 
     echo "[*] Installing Elements binaries into OS"
     osSudo "${ROOT_USER}" sh -c "cd ${ELEMENTS_HOME}/${ELEMENTS_REPO_NAME} && gmake install"

From fa92ba447810fd0d1ef1fbdfdc50b1599fc52881 Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Wed, 6 Jul 2022 14:01:18 +0200
Subject: [PATCH 8/9] Refactor nginx.conf and other ops scripts for lightning

---
 production/mempool-build-all                  |  40 ++++-
 production/mempool-start-all                  |   4 +-
 production/nginx/http-basic.conf              |   2 -
 .../nginx/location-api-v1-lightning.conf      |  20 +++
 production/nginx/location-api.conf            | 117 +++++++------
 production/nginx/location-liquid-api.conf     | 150 ++++++++++++++++-
 .../nginx/location-liquidtestnet-api.conf     | 154 +++++++++++++++++-
 .../location-signet-api-v1-lightning.conf     |  21 +++
 production/nginx/location-signet-api.conf     | 154 +++++++++++++++++-
 .../location-testnet-api-v1-lightning.conf    |  21 +++
 production/nginx/location-testnet-api.conf    | 154 +++++++++++++++++-
 production/nginx/nginx.conf                   |  46 ++++--
 production/nginx/server-bisq.conf             |  68 +++++---
 ...erver-electrs.conf => server-esplora.conf} |  10 +-
 production/nginx/server-mempool.conf          |   3 +
 ...eam-electrs.conf => upstream-esplora.conf} |  10 +-
 production/nginx/upstream-mempool.conf        |   9 +
 17 files changed, 844 insertions(+), 139 deletions(-)
 create mode 100644 production/nginx/location-api-v1-lightning.conf
 create mode 100644 production/nginx/location-signet-api-v1-lightning.conf
 create mode 100644 production/nginx/location-testnet-api-v1-lightning.conf
 rename production/nginx/{server-electrs.conf => server-esplora.conf} (62%)
 rename production/nginx/{upstream-electrs.conf => upstream-esplora.conf} (65%)

diff --git a/production/mempool-build-all b/production/mempool-build-all
index d44569fe2..ede944546 100755
--- a/production/mempool-build-all
+++ b/production/mempool-build-all
@@ -16,10 +16,13 @@ if [ -f "${LOCKFILE}" ];then
     exit 1
 fi
 
-trap "rv=\$?; rm -rf "${LOCKFILE}"; exit \$rv" INT TERM EXIT
+# on exit, remove lockfile but preserve exit code
+trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
 
+# create lockfile
 touch "${LOCKFILE}"
 
+# notify logged in users
 echo "Upgrading mempool to ${REF}" | wall
 
 update_repo()
@@ -84,25 +87,48 @@ ship_frontend()
     rsync -av "./dist/mempool/browser/" "${HOME}/public_html/${site}/" || exit 1
 }
 
+# load nvm if necessary
 export NVM_DIR="${HOME}/.nvm"
 source "${NVM_DIR}/nvm.sh"
 
-for target in mainnet testnet signet liquid liquidtestnet bisq;do
-    update_repo "${target}"
+# what to look for
+frontends=(mainnet liquid bisq)
+backends=(mainnet testnet signet liquid liquidtestnet bisq)
+frontend_repos=()
+backend_repos=()
+
+# find which frontend repos we have
+for repo in $frontends;do
+    [ -d "${repo}" ] && frontend_repos+="${repo}"
 done
 
-for target in mainnet testnet signet liquid liquidtestnet bisq;do
-    build_backend "${target}"
+# find which backend repos we have
+for repo in $backends;do
+    [ -d "${repo}" ] && backend_repos+="${repo}"
+    [ -d "${repo}-lightning" ] && backend_repos+="${repo}-lightning"
 done
 
-for target in mainnet liquid bisq;do
-    build_frontend "${target}"
+# update all repos
+for repo in $backend_repos;do
+    update_repo "${repo}"
 done
 
+# build backends
+for repo in $backend_repos;do
+    build_backend "${repo}"
+done
+
+# build frontends
+for repo in $frontend_repos;do
+    build_frontend "${repo}"
+done
+
+# ship frontend dist folders to public_html
 for target in mainnet liquid bisq;do
     ship_frontend "${target}"
 done
 
+# notify everyone
 echo "${HOSTNAME} updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
 echo "${HOSTNAME} updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
 
diff --git a/production/mempool-start-all b/production/mempool-start-all
index 5d5199166..ebb17f45b 100755
--- a/production/mempool-start-all
+++ b/production/mempool-start-all
@@ -1,8 +1,8 @@
 #!/usr/local/bin/zsh
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
-for site in mainnet liquid testnet bisq signet liquidtestnet
-do
+
+for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do
     cd "${HOME}/${site}/backend/" && \
     screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done'
 done
diff --git a/production/nginx/http-basic.conf b/production/nginx/http-basic.conf
index a79c73382..5d707ca42 100644
--- a/production/nginx/http-basic.conf
+++ b/production/nginx/http-basic.conf
@@ -4,8 +4,6 @@ tcp_nopush on;
 tcp_nodelay on;
 server_tokens off;
 server_name_in_redirect off;
-include /usr/local/etc/nginx/mime.types;
-default_type application/octet-stream;
 
 # default logs
 access_log /var/log/nginx/access.log;
diff --git a/production/nginx/location-api-v1-lightning.conf b/production/nginx/location-api-v1-lightning.conf
new file mode 100644
index 000000000..288fe4184
--- /dev/null
+++ b/production/nginx/location-api-v1-lightning.conf
@@ -0,0 +1,20 @@
+# route lightning API endpoints to lightning backend
+location /api/v1/lightning {
+	try_files /dev/null @mempool-api-v1-lightning;
+}
+location @mempool-api-v1-lightning {
+	proxy_pass $mempoolMainnetLightning;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf
index 886681f5c..2b5cb0152 100644
--- a/production/nginx/location-api.conf
+++ b/production/nginx/location-api.conf
@@ -1,26 +1,51 @@
-location /api/v1/statistics {
-	try_files /dev/null @mempool-api-v1-warmcache;
-}
-location /api/v1/mining {
-	try_files /dev/null @mempool-api-v1-warmcache;
-}
-location /api/v1/block/ {
-	try_files /dev/null @mempool-api-v1-forevercache;
-}
-location /api/v1 {
-	try_files /dev/null @mempool-api-v1-coldcache;
-}
-location /api/block/ {
-	rewrite ^/api/(.*) /$1 break;
-	try_files /dev/null @electrs-api-forevercache;
-}
-location /api/ {
-	rewrite ^/api/(.*) /$1 break;
-	try_files /dev/null @electrs-api-nocache;
+###########
+# mempool #
+###########
+
+# websocket has special HTTP headers
+location /api/v1/ws {
+	try_files /dev/null @mempool-api-v1-websocket;
 }
 
-location @mempool-api-v1-forevercache {
-	proxy_pass $mempoolBackend;
+# warm cache mining and mempool API responses
+location /api/v1/statistics {
+	try_files /dev/null @mempool-api-v1-cache-warm;
+}
+location /api/v1/mining {
+	try_files /dev/null @mempool-api-v1-cache-warm;
+}
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /api/v1/block/ {
+	try_files /dev/null @mempool-api-v1-cache-forever;
+}
+
+# everything else gets "normal" cache
+location /api/v1 {
+	try_files /dev/null @mempool-api-v1-cache-normal;
+}
+
+###########
+# esplora #
+###########
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /api/block/ {
+	rewrite ^/api/(.*) /$1 break;
+	try_files /dev/null @esplora-api-cache-forever;
+}
+# other API responses cannot be cached
+location /api/ {
+	rewrite ^/api/(.*) /$1 break;
+	try_files /dev/null @esplora-api-cache-disabled;
+}
+
+###########
+# routing #
+###########
+
+location @mempool-api-v1-websocket {
+	proxy_pass $mempoolMainnet;
 	proxy_http_version 1.1;
 
 	proxy_set_header Host $http_host;
@@ -29,8 +54,16 @@ location @mempool-api-v1-forevercache {
 	proxy_set_header Upgrade $http_upgrade;
 	proxy_set_header Connection "upgrade";
 	proxy_set_header X-Forwarded-Proto $scheme;
+}
+
+location @mempool-api-v1-cache-forever {
+	proxy_pass $mempoolMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
 
-	proxy_cache_bypass $http_upgrade;
 	proxy_cache_background_update on;
 	proxy_cache_use_stale updating;
 	proxy_cache api;
@@ -40,18 +73,14 @@ location @mempool-api-v1-forevercache {
 	expires 30d;
 }
 
-location @mempool-api-v1-warmcache {
-	proxy_pass $mempoolBackend;
-	proxy_http_version 1.1;
+location @mempool-api-v1-cache-warm {
+	proxy_pass $mempoolMainnet;
 
 	proxy_set_header Host $http_host;
 	proxy_set_header X-Real-IP $remote_addr;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
 	proxy_set_header X-Forwarded-Proto $scheme;
 
-	proxy_cache_bypass $http_upgrade;
 	proxy_cache_background_update on;
 	proxy_cache_use_stale updating;
 	proxy_cache api;
@@ -59,18 +88,14 @@ location @mempool-api-v1-warmcache {
 	proxy_redirect off;
 }
 
-location @mempool-api-v1-coldcache {
-	proxy_pass $mempoolBackend;
-	proxy_http_version 1.1;
+location @mempool-api-v1-cache-normal {
+	proxy_pass $mempoolMainnet;
 
 	proxy_set_header Host $http_host;
 	proxy_set_header X-Real-IP $remote_addr;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
 	proxy_set_header X-Forwarded-Proto $scheme;
 
-	proxy_cache_bypass $http_upgrade;
 	proxy_cache api;
 	proxy_cache_valid 200 10s;
 	proxy_redirect off;
@@ -78,54 +103,42 @@ location @mempool-api-v1-coldcache {
 	expires 10s;
 }
 
-location @mempool-api-v1-nocache {
-	proxy_pass $mempoolBackend;
-	proxy_http_version 1.1;
+location @mempool-api-v1-cache-disabled {
+	proxy_pass $mempoolMainnet;
 
 	proxy_set_header Host $http_host;
 	proxy_set_header X-Real-IP $remote_addr;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
 	proxy_set_header X-Forwarded-Proto $scheme;
 
-	proxy_cache_bypass $http_upgrade;
 	proxy_redirect off;
 	proxy_buffering off;
 
 	expires -1;
 }
 
-location @electrs-api-nocache {
-	proxy_pass $electrsBackend;
-	proxy_http_version 1.1;
+location @esplora-api-cache-disabled {
+	proxy_pass $esploraMainnet;
 
 	proxy_set_header Host $http_host;
 	proxy_set_header X-Real-IP $remote_addr;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
 	proxy_set_header X-Forwarded-Proto $scheme;
 
-	proxy_cache_bypass $http_upgrade;
 	proxy_redirect off;
 	proxy_buffering off;
 
 	expires -1;
 }
 
-location @electrs-api-forevercache {
-	proxy_pass $electrsBackend;
-	proxy_http_version 1.1;
+location @esplora-api-cache-forever {
+	proxy_pass $esploraMainnet;
 
 	proxy_set_header Host $http_host;
 	proxy_set_header X-Real-IP $remote_addr;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
 	proxy_set_header X-Forwarded-Proto $scheme;
 
-	proxy_cache_bypass $http_upgrade;
 	proxy_cache_background_update on;
 	proxy_cache_use_stale updating;
 	proxy_cache api;
diff --git a/production/nginx/location-liquid-api.conf b/production/nginx/location-liquid-api.conf
index 7c2f927a6..26ffffc70 100644
--- a/production/nginx/location-liquid-api.conf
+++ b/production/nginx/location-liquid-api.conf
@@ -1,12 +1,150 @@
+###########
+# mempool #
+###########
+
+# websocket has special HTTP headers
 location /liquid/api/v1/ws {
-	proxy_pass http://mempool-liquid-mainnet/;
-	proxy_http_version 1.1;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "Upgrade";
+	rewrite ^/liquid/(.*) /$1 break;
+	try_files /dev/null @mempool-liquid-api-v1-websocket;
 }
+
+# warm cache mempool API responses
+location /liquid/api/v1/statistics {
+	rewrite ^/liquid/(.*) /$1 break;
+	try_files /dev/null @mempool-liquid-api-v1-cache-warm;
+}
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /liquid/api/v1/block/ {
+	rewrite ^/liquid/(.*) /$1 break;
+	try_files /dev/null @mempool-liquid-api-v1-cache-forever;
+}
+
+# everything else gets "normal" cache
 location /liquid/api/v1 {
-	proxy_pass http://mempool-liquid-mainnet/api/v1;
+	rewrite ^/liquid/(.*) /$1 break;
+	try_files /dev/null @mempool-liquid-api-v1-cache-normal;
 }
+
+###########
+# esplora #
+###########
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /liquid/api/block/ {
+	rewrite ^/liquid/api/(.*) /$1 break;
+	try_files /dev/null @esplora-liquid-api-cache-forever;
+}
+# other API responses cannot be cached
 location /liquid/api/ {
-	proxy_pass http://electrs-liquid-mainnet/;
+	rewrite ^/liquid/api/(.*) /$1 break;
+	try_files /dev/null @esplora-liquid-api-cache-disabled;
+}
+
+###########
+# routing #
+###########
+
+location @mempool-liquid-api-v1-websocket {
+	proxy_pass $mempoolMainnet;
+	proxy_http_version 1.1;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "upgrade";
+	proxy_set_header X-Forwarded-Proto $scheme;
+}
+
+location @mempool-liquid-api-v1-cache-forever {
+	proxy_pass $mempoolMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
+}
+
+location @mempool-liquid-api-v1-cache-warm {
+	proxy_pass $mempoolMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+}
+
+location @mempool-liquid-api-v1-cache-normal {
+	proxy_pass $mempoolMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
+
+location @mempool-liquid-api-v1-cache-disabled {
+	proxy_pass $mempoolMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-liquid-api-cache-disabled {
+	proxy_pass $esploraMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-liquid-api-cache-forever {
+	proxy_pass $esploraMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
 }
diff --git a/production/nginx/location-liquidtestnet-api.conf b/production/nginx/location-liquidtestnet-api.conf
index 8b03c6fc4..311a6c317 100644
--- a/production/nginx/location-liquidtestnet-api.conf
+++ b/production/nginx/location-liquidtestnet-api.conf
@@ -1,12 +1,154 @@
+###########
+# mempool #
+###########
+
+# websocket has special HTTP headers
 location /liquidtestnet/api/v1/ws {
-	proxy_pass http://mempool-liquid-testnet/;
-	proxy_http_version 1.1;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "Upgrade";
+	rewrite ^/liquidtestnet/(.*) /$1 break;
+	try_files /dev/null @mempool-liquidtestnet-api-v1-websocket;
 }
+
+# warm cache mining and mempool API responses
+location /liquidtestnet/api/v1/statistics {
+	rewrite ^/liquidtestnet/(.*) /$1 break;
+	try_files /dev/null @mempool-liquidtestnet-api-v1-cache-warm;
+}
+location /liquidtestnet/api/v1/mining {
+	rewrite ^/liquidtestnet/(.*) /$1 break;
+	try_files /dev/null @mempool-liquidtestnet-api-v1-cache-warm;
+}
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /liquidtestnet/api/v1/block/ {
+	rewrite ^/liquidtestnet/(.*) /$1 break;
+	try_files /dev/null @mempool-liquidtestnet-api-v1-cache-forever;
+}
+
+# everything else gets "normal" cache
 location /liquidtestnet/api/v1 {
-	proxy_pass http://mempool-liquid-testnet/api/v1;
+	rewrite ^/liquidtestnet/(.*) /$1 break;
+	try_files /dev/null @mempool-liquidtestnet-api-v1-cache-normal;
 }
+
+###########
+# esplora #
+###########
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /liquidtestnet/api/block/ {
+	rewrite ^/liquidtestnet/api/(.*) /$1 break;
+	try_files /dev/null @esplora-liquidtestnet-api-cache-forever;
+}
+# other API responses cannot be cached
 location /liquidtestnet/api/ {
-	proxy_pass http://electrs-liquid-testnet/;
+	rewrite ^/liquidtestnet/api/(.*) /$1 break;
+	try_files /dev/null @esplora-liquidtestnet-api-cache-disabled;
+}
+
+###########
+# routing #
+###########
+
+location @mempool-liquidtestnet-api-v1-websocket {
+	proxy_pass $mempoolTestnet;
+	proxy_http_version 1.1;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "upgrade";
+	proxy_set_header X-Forwarded-Proto $scheme;
+}
+
+location @mempool-liquidtestnet-api-v1-cache-forever {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
+}
+
+location @mempool-liquidtestnet-api-v1-cache-warm {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+}
+
+location @mempool-liquidtestnet-api-v1-cache-normal {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
+
+location @mempool-liquidtestnet-api-v1-cache-disabled {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-liquidtestnet-api-cache-disabled {
+	proxy_pass $esploraTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-liquidtestnet-api-cache-forever {
+	proxy_pass $esploraTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
 }
diff --git a/production/nginx/location-signet-api-v1-lightning.conf b/production/nginx/location-signet-api-v1-lightning.conf
new file mode 100644
index 000000000..eb25c9820
--- /dev/null
+++ b/production/nginx/location-signet-api-v1-lightning.conf
@@ -0,0 +1,21 @@
+# route lightning API endpoints to lightning backend
+location /signet/api/v1/lightning {
+	rewrite ^/signet/(.*) /$1 break;
+	try_files /dev/null @mempool-signet-api-v1-lightning;
+}
+location @mempool-signet-api-v1-lightning {
+	proxy_pass $mempoolSignetLightning;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
diff --git a/production/nginx/location-signet-api.conf b/production/nginx/location-signet-api.conf
index 245979229..262dea48e 100644
--- a/production/nginx/location-signet-api.conf
+++ b/production/nginx/location-signet-api.conf
@@ -1,12 +1,154 @@
+###########
+# mempool #
+###########
+
+# websocket has special HTTP headers
 location /signet/api/v1/ws {
-	proxy_pass http://mempool-bitcoin-signet/;
-	proxy_http_version 1.1;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "Upgrade";
+	rewrite ^/signet/(.*) /$1 break;
+	try_files /dev/null @mempool-signet-api-v1-websocket;
 }
+
+# warm cache mining and mempool API responses
+location /signet/api/v1/statistics {
+	rewrite ^/signet/(.*) /$1 break;
+	try_files /dev/null @mempool-signet-api-v1-cache-warm;
+}
+location /signet/api/v1/mining {
+	rewrite ^/signet/(.*) /$1 break;
+	try_files /dev/null @mempool-signet-api-v1-cache-warm;
+}
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /signet/api/v1/block/ {
+	rewrite ^/signet/(.*) /$1 break;
+	try_files /dev/null @mempool-signet-api-v1-cache-forever;
+}
+
+# everything else gets "normal" cache
 location /signet/api/v1 {
-	proxy_pass http://mempool-bitcoin-signet/api/v1;
+	rewrite ^/signet/(.*) /$1 break;
+	try_files /dev/null @mempool-signet-api-v1-cache-normal;
 }
+
+###########
+# esplora #
+###########
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /signet/api/block/ {
+	rewrite ^/signet/api/(.*) /$1 break;
+	try_files /dev/null @esplora-signet-api-cache-forever;
+}
+# other API responses cannot be cached
 location /signet/api/ {
-	proxy_pass http://electrs-bitcoin-signet/;
+	rewrite ^/signet/api/(.*) /$1 break;
+	try_files /dev/null @esplora-signet-api-cache-disabled;
+}
+
+###########
+# routing #
+###########
+
+location @mempool-signet-api-v1-websocket {
+	proxy_pass $mempoolSignet;
+	proxy_http_version 1.1;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "upgrade";
+	proxy_set_header X-Forwarded-Proto $scheme;
+}
+
+location @mempool-signet-api-v1-cache-forever {
+	proxy_pass $mempoolSignet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
+}
+
+location @mempool-signet-api-v1-cache-warm {
+	proxy_pass $mempoolSignet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+}
+
+location @mempool-signet-api-v1-cache-normal {
+	proxy_pass $mempoolSignet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
+
+location @mempool-signet-api-v1-cache-disabled {
+	proxy_pass $mempoolSignet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-signet-api-cache-disabled {
+	proxy_pass $esploraSignet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-signet-api-cache-forever {
+	proxy_pass $esploraSignet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
 }
diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf
new file mode 100644
index 000000000..dbabc9aef
--- /dev/null
+++ b/production/nginx/location-testnet-api-v1-lightning.conf
@@ -0,0 +1,21 @@
+# route lightning API endpoints to lightning backend
+location /testnet/api/v1/lightning {
+	rewrite ^/testnet/(.*) /$1 break;
+	try_files /dev/null @mempool-testnet-api-v1-lightning;
+}
+location @mempool-testnet-api-v1-lightning {
+	proxy_pass $mempoolSignetLightning;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
diff --git a/production/nginx/location-testnet-api.conf b/production/nginx/location-testnet-api.conf
index 857b1b957..1f74aa533 100644
--- a/production/nginx/location-testnet-api.conf
+++ b/production/nginx/location-testnet-api.conf
@@ -1,12 +1,154 @@
+###########
+# mempool #
+###########
+
+# websocket has special HTTP headers
 location /testnet/api/v1/ws {
-	proxy_pass http://mempool-bitcoin-testnet/;
-	proxy_http_version 1.1;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "Upgrade";
+	rewrite ^/testnet/(.*) /$1 break;
+	try_files /dev/null @mempool-testnet-api-v1-websocket;
 }
+
+# warm cache mining and mempool API responses
+location /testnet/api/v1/statistics {
+	rewrite ^/testnet/(.*) /$1 break;
+	try_files /dev/null @mempool-testnet-api-v1-cache-warm;
+}
+location /testnet/api/v1/mining {
+	rewrite ^/testnet/(.*) /$1 break;
+	try_files /dev/null @mempool-testnet-api-v1-cache-warm;
+}
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /testnet/api/v1/block/ {
+	rewrite ^/testnet/(.*) /$1 break;
+	try_files /dev/null @mempool-testnet-api-v1-cache-forever;
+}
+
+# everything else gets "normal" cache
 location /testnet/api/v1 {
-	proxy_pass http://mempool-bitcoin-testnet/api/v1;
+	rewrite ^/testnet/(.*) /$1 break;
+	try_files /dev/null @mempool-testnet-api-v1-cache-normal;
 }
+
+###########
+# esplora #
+###########
+
+# it's ok to cache blockchain data "forever", so we do 30d
+location /testnet/api/block/ {
+	rewrite ^/testnet/api/(.*) /$1 break;
+	try_files /dev/null @esplora-testnet-api-cache-forever;
+}
+# other API responses cannot be cached
 location /testnet/api/ {
-	proxy_pass http://electrs-bitcoin-testnet/;
+	rewrite ^/testnet/api/(.*) /$1 break;
+	try_files /dev/null @esplora-testnet-api-cache-disabled;
+}
+
+###########
+# routing #
+###########
+
+location @mempool-testnet-api-v1-websocket {
+	proxy_pass $mempoolTestnet;
+	proxy_http_version 1.1;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "upgrade";
+	proxy_set_header X-Forwarded-Proto $scheme;
+}
+
+location @mempool-testnet-api-v1-cache-forever {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
+}
+
+location @mempool-testnet-api-v1-cache-warm {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+}
+
+location @mempool-testnet-api-v1-cache-normal {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache api;
+	proxy_cache_valid 200 10s;
+	proxy_redirect off;
+
+	expires 10s;
+}
+
+location @mempool-testnet-api-v1-cache-disabled {
+	proxy_pass $mempoolTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-testnet-api-cache-disabled {
+	proxy_pass $esploraTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
+}
+
+location @esplora-testnet-api-cache-forever {
+	proxy_pass $esploraTestnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache api;
+	proxy_cache_valid 200 30d;
+	proxy_redirect off;
+
+	expires 30d;
 }
diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf
index ed0d35e52..24f389533 100644
--- a/production/nginx/nginx.conf
+++ b/production/nginx/nginx.conf
@@ -1,4 +1,3 @@
-# FreeBSD configuration
 user nobody;
 pid /var/run/nginx.pid;
 
@@ -14,17 +13,21 @@ http {
 	# DNS servers for on-demand recursive resolver
 	resolver 8.8.8.8;
 
+	# include default mime types
+	include /usr/local/etc/nginx/mime.types;
+	default_type application/octet-stream;
+
 	# HTTP basic configuration
 	include mempool/production/nginx/http-basic.conf;
 	include mempool/production/nginx/http-proxy-cache.conf;
 	include mempool/production/nginx/http-language.conf;
 
-	# mempool backend configuration
+	# mempool configuration
 	include mempool/production/nginx/upstream-mempool.conf;
 
-	# electrs backend configuration
-	include mempool/production/nginx/upstream-electrs.conf;
-	include mempool/production/nginx/server-electrs.conf;
+	# esplora configuration
+	include mempool/production/nginx/upstream-esplora.conf;
+	include mempool/production/nginx/server-esplora.conf;
 
 	# MEMPOOL.NINJA
 	server {
@@ -36,11 +39,18 @@ http {
 		# for services from mempool.space like contributors on about page
 		set $mempoolSpaceServices "https://mempool.space";
 
-		# for mempool/backend daemon, see upstream-mempool.conf
-		set $mempoolBackend "http://mempool-bitcoin-mainnet";
+		# for mempool daemons, see upstream-mempool.conf
+		set $mempoolMainnet "http://mempool-bitcoin-mainnet";
+		set $mempoolMainnetLightning "http://mempool-bitcoin-mainnet-lightning";
+		set $mempoolTestnet "http://mempool-bitcoin-testnet";
+		set $mempoolTestnetLightning "http://mempool-bitcoin-testnet-lightning";
+		set $mempoolSignet "http://mempool-bitcoin-signet";
+		set $mempoolSignetLightning "http://mempool-bitcoin-signet-lightning";
 
-		# for blockstream/electrs daemon, see upstream-electrs.conf
-		set $electrsBackend "http://electrs-bitcoin-mainnet";
+		# for blockstream/esplora daemons, see upstream-esplora.conf
+		set $esploraMainnet "http://esplora-bitcoin-mainnet";
+		set $esploraTestnet "http://esplora-bitcoin-testnet";
+		set $esploraSignet "http://esplora-bitcoin-signet";
 
 		# tor v3
 		listen 127.0.0.1:81;
@@ -70,11 +80,11 @@ http {
 		# for services from mempool.space like contributors on about page
 		set $mempoolSpaceServices "https://mempool.space";
 
-		# for mempool/backend daemon, see upstream-mempool.conf
-		set $mempoolBackend "http://mempool-bisq-mainnet";
+		# for mempool daemons, see upstream-mempool.conf
+		set $mempoolBisq "http://mempool-bitcoin-bisq";
 
-		# for blockstream/electrs daemon, see upstream-electrs.conf
-		set $electrsBackend "http://electrs-bitcoin-mainnet";
+		# for blockstream/esplora daemon, see upstream-esplora.conf
+		set $esploraMainnet "http://esplora-bitcoin-mainnet";
 
 		# tor v3
 		listen 127.0.0.1:82;
@@ -104,11 +114,13 @@ http {
 		# for services from mempool.space like contributors on about page
 		set $mempoolSpaceServices "https://mempool.space";
 
-		# for mempool/backend daemon, see upstream-mempool.conf
-		set $mempoolBackend "http://mempool-liquid-mainnet";
+		# for mempool daemons, see upstream-mempool.conf
+		set $mempoolMainnet "http://mempool-liquid-mainnet";
+		set $mempoolTestnet "http://mempool-liquid-testnet";
 
-		# for blockstream/electrs daemon, see upstream-electrs.conf
-		set $electrsBackend "http://electrs-liquid-mainnet";
+		# for blockstream/esplora daemon, see upstream-esplora.conf
+		set $esploraMainnet "http://esplora-liquid-mainnet";
+		set $esploraTestnet "http://esplora-liquid-testnet";
 
 		# tor v3
 		listen 127.0.0.1:83;
diff --git a/production/nginx/server-bisq.conf b/production/nginx/server-bisq.conf
index 3933b6e4d..704ef5a6c 100644
--- a/production/nginx/server-bisq.conf
+++ b/production/nginx/server-bisq.conf
@@ -4,80 +4,98 @@ include mempool/production/nginx/location-api-v1-services.conf;
 proxy_cache markets;
 proxy_cache_valid 200 30s;
 
-# route electrs APIs to electrs
+# route esplora APIs to esplora
 location /api/tx/ {
-        proxy_pass http://electrs-bitcoin-mainnet/tx/;
+	rewrite ^/api/(.*) /$1 break;
+	try_files /dev/null @esplora-api-cache-disabled;
 }
 
 # rewrite APIs to match what backend expects
 location /api/currencies {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/depth {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/hloc {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/offers {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/ticker {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/trades {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/volumes {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/markets {
 	rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api/v1 {
 	rewrite ^/api/v1/(.*) /api/v1/bisq/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /api {
 	rewrite ^/api/(.*) /api/v1/bisq/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
+}
+location /api/v1/ws {
+	rewrite ^/api/(.*) /api/v1/bisq/$1 break;
+	try_files /dev/null @mempool-bisq-websocket;
+}
+location /bisq/api/v1/ws {
+	rewrite ^/bisq/api/v1/(.*) /api/v1/bisq/$1 break;
+	try_files /dev/null @mempool-bisq-websocket;
 }
 location /bisq/api/v1 {
 	rewrite ^/bisq/api/v1/(.*) /api/v1/bisq/$1 break;
-	try_files $uri $uri/ @mempool-bisq;
+	try_files /dev/null @mempool-bisq;
 }
 location /bisq/api {
-        rewrite ^/bisq/api/(.*) /api/v1/bisq/$1 break;
-        try_files $uri $uri/ @mempool-bisq;
+	rewrite ^/bisq/api/(.*) /api/v1/bisq/$1 break;
+	try_files /dev/null @mempool-bisq;
 }
 
-# special handling for websocket
-location /api/v1/ws {
-	proxy_pass http://mempool-bitcoin-bisq/;
+location @mempool-bisq-websocket {
+	proxy_pass $mempoolBisq;
 	proxy_http_version 1.1;
 	proxy_set_header Upgrade $http_upgrade;
 	proxy_set_header Connection "Upgrade";
 }
 
 location @mempool-bisq {
-	proxy_pass http://mempool-bitcoin-bisq;
-	proxy_http_version 1.1;
+	proxy_pass $mempoolBisq;
 
 	proxy_set_header Host $http_host;
 	proxy_set_header X-Real-IP $remote_addr;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
-
-	proxy_cache_bypass $http_upgrade;
-	proxy_redirect off;
 	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+}
+
+location @esplora-api-cache-disabled {
+	proxy_pass $esploraMainnet;
+
+	proxy_set_header Host $http_host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_redirect off;
+	proxy_buffering off;
+
+	expires -1;
 }
diff --git a/production/nginx/server-electrs.conf b/production/nginx/server-esplora.conf
similarity index 62%
rename from production/nginx/server-electrs.conf
rename to production/nginx/server-esplora.conf
index b817b8163..38cdb0bc2 100644
--- a/production/nginx/server-electrs.conf
+++ b/production/nginx/server-esplora.conf
@@ -2,34 +2,34 @@ server {
 	listen 127.0.0.1:4000;
 	access_log /dev/null;
 	location / {
-		proxy_pass http://electrs-bitcoin-mainnet;
+		proxy_pass http://esplora-bitcoin-mainnet;
 	}
 }
 server {
 	listen 127.0.0.1:4001;
 	access_log /dev/null;
 	location / {
-		proxy_pass http://electrs-liquid-mainnet;
+		proxy_pass http://esplora-liquid-mainnet;
 	}
 }
 server {
 	listen 127.0.0.1:4002;
 	access_log /dev/null;
 	location / {
-		proxy_pass http://electrs-bitcoin-testnet;
+		proxy_pass http://esplora-bitcoin-testnet;
 	}
 }
 server {
 	listen 127.0.0.1:4003;
 	access_log /dev/null;
 	location / {
-		proxy_pass http://electrs-bitcoin-signet;
+		proxy_pass http://esplora-bitcoin-signet;
 	}
 }
 server {
 	listen 127.0.0.1:4004;
 	access_log /dev/null;
 	location / {
-		proxy_pass http://electrs-liquid-testnet;
+		proxy_pass http://esplora-liquid-testnet;
 	}
 }
diff --git a/production/nginx/server-mempool.conf b/production/nginx/server-mempool.conf
index 5a84cce17..892fe50e1 100644
--- a/production/nginx/server-mempool.conf
+++ b/production/nginx/server-mempool.conf
@@ -1,6 +1,9 @@
 include mempool/production/nginx/server-common.conf;
 include mempool/production/nginx/location-redirects.conf;
 include mempool/production/nginx/location-api-v1-services.conf;
+include mempool/production/nginx/location-api-v1-lightning.conf;
 include mempool/production/nginx/location-api.conf;
 include mempool/production/nginx/location-testnet-api.conf;
+include mempool/production/nginx/location-testnet-api-v1-lightning.conf;
 include mempool/production/nginx/location-signet-api.conf;
+include mempool/production/nginx/location-signet-api-v1-lightning.conf;
diff --git a/production/nginx/upstream-electrs.conf b/production/nginx/upstream-esplora.conf
similarity index 65%
rename from production/nginx/upstream-electrs.conf
rename to production/nginx/upstream-esplora.conf
index 476b73054..6cad0730b 100644
--- a/production/nginx/upstream-electrs.conf
+++ b/production/nginx/upstream-esplora.conf
@@ -1,15 +1,15 @@
-upstream electrs-bitcoin-mainnet {
+upstream esplora-bitcoin-mainnet {
 	server [::1]:3000 fail_timeout=10s max_fails=10 weight=99999;
 }
-upstream electrs-liquid-mainnet {
+upstream esplora-liquid-mainnet {
 	server [::1]:3001 fail_timeout=10s max_fails=10 weight=99999;
 }
-upstream electrs-bitcoin-testnet {
+upstream esplora-bitcoin-testnet {
 	server [::1]:3002 fail_timeout=10s max_fails=10 weight=99999;
 }
-upstream electrs-bitcoin-signet {
+upstream esplora-bitcoin-signet {
 	server [::1]:3003 fail_timeout=10s max_fails=10 weight=99999;
 }
-upstream electrs-liquid-testnet {
+upstream esplora-liquid-testnet {
 	server [::1]:3004 fail_timeout=10s max_fails=10 weight=99999;
 }
diff --git a/production/nginx/upstream-mempool.conf b/production/nginx/upstream-mempool.conf
index 8e0f12ab6..c9cf9c2bb 100644
--- a/production/nginx/upstream-mempool.conf
+++ b/production/nginx/upstream-mempool.conf
@@ -16,3 +16,12 @@ upstream mempool-bitcoin-signet {
 upstream mempool-liquid-testnet {
 	server 127.0.0.1:8994 fail_timeout=10s max_fails=10 weight=99999;
 }
+upstream mempool-bitcoin-mainnet-lightning {
+	server 127.0.0.1:8993 fail_timeout=10s max_fails=10 weight=99999;
+}
+upstream mempool-bitcoin-testnet-lightning {
+	server 127.0.0.1:8992 fail_timeout=10s max_fails=10 weight=99999;
+}
+upstream mempool-bitcoin-signet-lightning {
+	server 127.0.0.1:8991 fail_timeout=10s max_fails=10 weight=99999;
+}

From 989c74699fe6c8c93fc9a78565d163e728604483 Mon Sep 17 00:00:00 2001
From: Stephan Oeste <emzy@emzy.de>
Date: Wed, 6 Jul 2022 18:31:31 +0200
Subject: [PATCH 9/9] Add crontab creation for linux and bitcoin-minfee.service
 in prod install

---
 production/install                      | 79 +++++++++++++++++++++----
 production/linux/bitcoin-minfee.service | 22 +++++++
 2 files changed, 91 insertions(+), 10 deletions(-)
 create mode 100644 production/linux/bitcoin-minfee.service

diff --git a/production/install b/production/install
index 9f2533c61..10bec7553 100755
--- a/production/install
+++ b/production/install
@@ -39,6 +39,7 @@ ELEMENTS_INSTALL=ON
 
 # configure 4 network instances
 BITCOIN_MAINNET_ENABLE=ON
+BITCOIN_MAINNET_MINFEE_ENABLE=ON
 BITCOIN_TESTNET_ENABLE=ON
 BITCOIN_SIGNET_ENABLE=ON
 BISQ_MAINNET_ENABLE=ON
@@ -682,6 +683,7 @@ $CUT >$input <<-EOF
 Tor:Enable Tor v3 HS Onion:ON
 Certbot:Enable HTTPS using Certbot:ON
 Mainnet:Enable Bitcoin Mainnet:ON
+Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON
 Testnet:Enable Bitcoin Testnet:ON
 Liquid:Enable Elements Liquid:ON
 Bisq:Enable Bisq:ON
@@ -725,6 +727,12 @@ else
     BITCOIN_MAINNET_ENABLE=OFF
 fi
 
+if grep Mainnet-Minfee $tempfile >/dev/null 2>&1;then
+    BITCOIN_MAINNET_MINFEE_ENABLE=ON
+else
+    BITCOIN_MAINNET_MINFEE_ENABLE=OFF
+fi
+
 if grep Testnet $tempfile >/dev/null 2>&1;then
     BITCOIN_TESTNET_ENABLE=ON
 else
@@ -1214,6 +1222,24 @@ if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
     esac
 fi
 
+#######################################
+# Bitcoin instance for Mainnet Minfee #
+#######################################
+
+if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
+    echo "[*] Installing Bitcoin Minfee service"
+    case $OS in
+
+        FreeBSD)
+            echo "[*] FIXME: Bitcoin Minfee service must be installed manually on FreeBSD"
+        ;;
+
+        Debian)
+            osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/linux/bitcoin-minfee.service" "${DEBIAN_SERVICE_HOME}"
+        ;;
+    esac
+fi
+
 ################################
 # Bitcoin instance for Testnet #
 ################################
@@ -1277,9 +1303,16 @@ if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
     osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-mainnet" "${BITCOIN_ELECTRS_HOME}"
 
     echo "[*] Installing Bitcoin crontab"
-    # FIXME: must only crontab enabled daemons
-    osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/bitcoin.crontab"
-    osSudo "${ROOT_USER}" crontab -u "${MINFEE_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/minfee.crontab"
+    case $OS in
+        FreeBSD)
+            echo [*] FIXME: must only crontab enabled daemons
+            osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/bitcoin.crontab"
+            osSudo "${ROOT_USER}" crontab -u "${MINFEE_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/minfee.crontab"
+        ;;
+        Debian)
+            (crontab -l ; echo "@reboot sleep 30 ; screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet") | osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" -
+        ;;
+    esac
 
     echo "[*] Configuring Bitcoin Mainnet RPC credentials in electrs start script"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet"
@@ -1295,6 +1328,13 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
     echo "[*] Installing Bitcoin Testnet electrs start script"
     osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-testnet" "${BITCOIN_ELECTRS_HOME}"
 
+    case $OS in
+        Debian)
+            echo "[*] Installing Bitcoin-testnet crontab"
+            (crontab -l ; echo "@reboot sleep 70 ; screen -dmS testnet /bitcoin/electrs/electrs-start-testnet") | osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" -
+        ;;
+    esac
+
     echo "[*] Configuring Bitcoin Testnet RPC credentials in electrs start script"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet"
@@ -1309,6 +1349,13 @@ if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
     echo "[*] Installing Bitcoin Signet electrs start script"
     osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-signet" "${BITCOIN_ELECTRS_HOME}"
 
+    case $OS in
+        Debian)
+            echo "[*] Installing Bitcoin-signet crontab"
+            (crontab -l ; echo "@reboot sleep 90 ; screen -dmS signet /bitcoin/electrs/electrs-start-signet") | osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" -
+        ;;
+    esac
+
     echo "[*] Configuring Bitcoin Signet RPC credentials in electrs start script"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet"
@@ -1324,8 +1371,15 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
     osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquid" "${ELEMENTS_ELECTRS_HOME}"
 
     echo "[*] Installing Elements crontab"
-    # FIXME: must only crontab enabled daemons
-    osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/elements.crontab"
+    case $OS in
+        FreeBSD)
+            echo [*] FIXME: must only crontab enabled daemons
+            osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/elements.crontab"
+        ;;
+        Debian)
+            (crontab -l ; echo "6 * * * * cd $HOME/asset_registry_db && git pull origin master >/dev/null 2>&1") | osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" -
+        ;;
+    esac
 
     echo "[*] Configuring Elements Liquid RPC credentials in electrs start script"
     osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid"
@@ -1341,6 +1395,13 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
     echo "[*] Installing Elements Liquid Testnet electrs start script"
     osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquidtestnet" "${ELEMENTS_ELECTRS_HOME}"
 
+    case $OS in
+        Debian)
+            echo "[*] Installing Elements-testnet crontab"
+            (crontab -l ; echo "6 * * * * cd $HOME/asset_registry_testnet_db && git pull origin master >/dev/null 2>&1") | osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" -
+        ;;
+    esac
+
     echo "[*] Installing Elements Liquid Testnet RPC credentials"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf"
     osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf"
@@ -1480,23 +1541,21 @@ case $OS in
         fi
         if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
             osSudo "${ROOT_USER}" systemctl enable bitcoin.service
-            osSudo "${ROOT_USER}" systemctl enable mempool.service
+        fi
+        if [ "${BITCOIN_MAINNET_MINFEE_ENABLE}" = ON ];then
+            osSudo "${ROOT_USER}" systemctl enable bitcoin-minfee.service
         fi
         if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
             osSudo "${ROOT_USER}" systemctl enable bitcoin-testnet.service
-            osSudo "${ROOT_USER}" systemctl enable mempool-testnet.service
         fi
         if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
             osSudo "${ROOT_USER}" systemctl enable bitcoin-signet.service
-            osSudo "${ROOT_USER}" systemctl enable mempool-signet.service
         fi
         if [ "${BISQ_MAINNET_ENABLE}" = ON ];then
             osSudo "${ROOT_USER}" systemctl enable bisq.service
-            osSudo "${ROOT_USER}" systemctl enable mempool-bisq.service
         fi
         if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
             osSudo "${ROOT_USER}" systemctl enable liquid.service
-            osSudo "${ROOT_USER}" systemctl enable mempool-liquid.service
         fi
     ;;
 esac
diff --git a/production/linux/bitcoin-minfee.service b/production/linux/bitcoin-minfee.service
new file mode 100644
index 000000000..fe70d99f0
--- /dev/null
+++ b/production/linux/bitcoin-minfee.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=Bitcoind-minfee
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/bitcoind -daemon -printtoconsole -pid=/minfee/bitcoind-minfee.pid
+ExecStop=/usr/local/bin/bitcoin-cli stop
+
+Type=forking
+PIDFile=/minfee/bitcoind.pid
+Restart=on-failure
+
+User=minfee
+Group=minfee
+
+PrivateTmp=true
+ProtectSystem=full
+NoNewPrivileges=true
+PrivateDevices=true
+
+[Install]
+WantedBy=multi-user.target