Merge branch 'master' into simon/i18n-additions

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -2,7 +2,7 @@ import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
-  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, blockHash?: string): Promise<IEsploraApi.Transaction>;
+  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
   $getBlockHeightTip(): Promise<number>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;

backend/src/api/bitcoin/bitcoin-api.ts

@@ -14,14 +14,31 @@ class BitcoinApi implements AbstractBitcoinApi {
     this.bitcoindClient = bitcoinClient;
   }
 
-  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, blockHash?: string): Promise<IEsploraApi.Transaction> {
+  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
+    return {
+      id: block.hash,
+      height: block.height,
+      version: block.version,
+      timestamp: block.time,
+      bits: parseInt(block.bits, 16),
+      nonce: block.nonce,
+      difficulty: block.difficulty,
+      merkle_root: block.merkleroot,
+      tx_count: block.nTx,
+      size: block.size,
+      weight: block.weight,
+      previousblockhash: block.previousblockhash,
+    };
+  }
+
+  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
     if (txInMempool && addPrevout) {
       return this.$addPrevouts(txInMempool);
     }
 
-    return this.bitcoindClient.getRawTransaction(txId, true, blockHash)
+    return this.bitcoindClient.getRawTransaction(txId, true)
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
           transaction.vout.forEach((vout) => {
@@ -174,35 +191,18 @@ class BitcoinApi implements AbstractBitcoinApi {
       };
     }
 
+    if (addPrevout) {
       if (transaction.confirmations) {
-      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
+        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction);
       } else {
         esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
-      if (addPrevout) {
+        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction);
-        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
       }
     }
 
     return esploraTransaction;
   }
 
-  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
-    return {
-      id: block.hash,
-      height: block.height,
-      version: block.version,
-      timestamp: block.time,
-      bits: parseInt(block.bits, 16),
-      nonce: block.nonce,
-      difficulty: block.difficulty,
-      merkle_root: block.merkleroot,
-      tx_count: block.nTx,
-      size: block.size,
-      weight: block.weight,
-      previousblockhash: block.previousblockhash,
-    };
-  }
-
   private translateScriptPubKeyType(outputType: string): string {
     const map = {
       'pubkey': 'p2pk',
@@ -245,7 +245,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       if (vin.prevout) {
         continue;
       }
-      const innerTx = await this.$getRawTransaction(vin.txid, false);
+      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
       vin.prevout = innerTx.vout[vin.vout];
       this.addInnerScriptsToVin(vin);
     }
@@ -271,18 +271,16 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawMemPool(true);
   }
 
-  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction): Promise<IEsploraApi.Transaction> {
     if (transaction.vin[0].is_coinbase) {
       transaction.fee = 0;
       return transaction;
     }
     let totalIn = 0;
     for (const vin of transaction.vin) {
-      const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout);
+      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
-      if (addPrevout) {
       vin.prevout = innerTx.vout[vin.vout];
       this.addInnerScriptsToVin(vin);
-      }
       totalIn += innerTx.vout[vin.vout].value;
     }
     const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);

backend/src/api/difficulty-adjustment.ts

@@ -32,7 +32,7 @@ class DifficultyAdjustmentApi {
       }
     }
 
-    let timeAvgMins = blocksInEpoch ? diff / blocksInEpoch / 60 : 10;
+    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
 
     // Testnet difficulty is set to 1 after 20 minutes of no blocks,
     // therefore the time between blocks will always be below 20 minutes (1200s).

backend/src/tasks/pools-updater.ts

@@ -1,8 +1,10 @@
-const https = require('https');
+import axios from 'axios';
 import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import * as https from 'https';
 
 /**
  * Maintain the most recent version of pools.json
@@ -28,6 +30,13 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
+    logger.info('Updating latest mining pools from Github');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info('List of public pools will be queried over the Tor network');
+    } else {
+      logger.info('List of public pools will be queried over clearnet');
+    }
+
     try {
       const dbSha = await this.getShaFromDb();
       const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
@@ -41,7 +50,10 @@ class PoolsUpdater {
       }
 
       logger.warn('Pools.json is outdated, fetch latest from github');
-      const poolsJson = await this.fetchPools();
+      const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
+      if (poolsJson === undefined) {
+        return;
+      }
       await poolsParser.migratePoolsJson(poolsJson);
       await this.updateDBSha(githubSha);
       logger.notice('PoolsUpdater completed');
@@ -52,14 +64,6 @@ class PoolsUpdater {
     }
   }
 
-  /**
-   * Fetch pools.json from github repo
-   */
-  private async fetchPools(): Promise<object> {
-    const response = await this.query('/repos/mempool/mining-pools/contents/pools.json');
-    return JSON.parse(Buffer.from(response['content'], 'base64').toString('utf8'));
-  }
-
   /**
    * Fetch our latest pools.json sha from the db
    */
@@ -90,13 +94,15 @@ class PoolsUpdater {
    * Fetch our latest pools.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
-    const response = await this.query('/repos/mempool/mining-pools/git/trees/master');
+    const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
 
+    if (response !== undefined) {
       for (const file of response['tree']) {
         if (file['path'] === 'pools.json') {
           return file['sha'];
         }
       }
+    }
 
     logger.err('Cannot to find latest pools.json sha from github api response');
     return undefined;
@@ -105,35 +111,45 @@ class PoolsUpdater {
   /**
    * Http request wrapper
    */
-  private query(path): Promise<string> {
+  private async query(path): Promise<object | undefined> {
-    return new Promise((resolve, reject) => {
+    type axiosOptions = {
-      const options = {
+      httpsAgent?: https.Agent;
-        host: 'api.github.com',
+    }
-        path: path,
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-        method: 'GET',
+    const axiosOptions: axiosOptions = {};
-        headers: { 'user-agent': 'node.js' }
+    let retry = 0;
+
+    if (config.SOCKS5PROXY.ENABLED) {
+      const socksOptions: any = {
+        agentOptions: {
+          keepAlive: true,
+        },
+        hostname: config.SOCKS5PROXY.HOST,
+        port: config.SOCKS5PROXY.PORT
       };
 
-      logger.debug('Querying: api.github.com' + path);
+      if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+        socksOptions.username = config.SOCKS5PROXY.USERNAME;
+        socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+      }
 
-      const request = https.get(options, (response) => {
+      axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-        const chunks_of_data: any[] = [];
+    }
-        response.on('data', (fragments) => {
-          chunks_of_data.push(fragments);
-        });
-        response.on('end', () => {
-          resolve(JSON.parse(Buffer.concat(chunks_of_data).toString()));
-        });
-        response.on('error', (error) => {
-          reject(error);
-        });
-      });
 
-      request.on('error', (error) => {
+    while(retry < 5) {
-        logger.err('Github API query failed. Reason: '  + error);
+      try {
-        reject(error);
+        const data = await axios.get(path, axiosOptions);
-      });
+        if (data.statusText !== 'OK' || !data.data) {
-    });
+          throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
+        }
+        return data.data;
+      } catch (e) {
+        logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
+        retry++;
+      }
+      await setDelay();
+    }
+    return undefined;
   }
 }
 

contributors/ayanamidev.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 15, 2022.
+
+Signed: ayanamidev

frontend/src/app/components/block/block.component.html

@@ -195,7 +195,7 @@
     </div>
     <div class="clearfix"></div>
 
-    <app-transactions-list [transactions]="transactions"></app-transactions-list>
+    <app-transactions-list [transactions]="transactions" [paginated]="true"></app-transactions-list>
 
     <ng-template [ngIf]="isLoadingTransactions">
       <div class="text-center mb-4" class="tx-skeleton">

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -22,6 +22,7 @@ export class TransactionsListComponent implements OnInit, OnChanges {
   @Input() showConfirmations = false;
   @Input() transactionPage = false;
   @Input() errorUnblinded = false;
+  @Input() paginated = false;
   @Input() outputIndex: number;
   @Input() address: string = '';
 
@@ -84,6 +85,9 @@ export class TransactionsListComponent implements OnInit, OnChanges {
     if (!this.transactions || !this.transactions.length) {
       return;
     }
+    if (this.paginated) {
+      this.outspends = [];
+    }
     if (this.outputIndex) {
       setTimeout(() => {
         const assetBoxElements = document.getElementsByClassName('assetBox');

frontend/src/app/services/websocket.service.ts

@@ -9,9 +9,9 @@ import { take } from 'rxjs/operators';
 import { TransferState, makeStateKey } from '@angular/platform-browser';
 import { BlockExtended } from '../interfaces/node-api.interface';
 
-const OFFLINE_RETRY_AFTER_MS = 10000;
+const OFFLINE_RETRY_AFTER_MS = 1000;
-const OFFLINE_PING_CHECK_AFTER_MS = 30000;
+const OFFLINE_PING_CHECK_AFTER_MS = 10000;
-const EXPECT_PING_RESPONSE_AFTER_MS = 4000;
+const EXPECT_PING_RESPONSE_AFTER_MS = 5000;
 
 const initData = makeStateKey('/api/v1/init-data');
 

nginx-mempool.conf

@@ -44,25 +44,6 @@
 		try_files $uri $uri/ /en-US/index.html =404;
 	}
 
-	# mainnet API
-	location /api/v1/donations {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/donations/images {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/contributors {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/contributors/images {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/translators {
-		proxy_pass https://mempool.space;
-	}
-	location /api/v1/translators/images {
-		proxy_pass https://mempool.space;
-	}
 	location /api/v1/ws {
 		proxy_pass http://127.0.0.1:8999/;
 		proxy_http_version 1.1;

production/install

@@ -276,7 +276,7 @@ MINFEE_HOME=/minfee
 MEMPOOL_REPO_URL=https://github.com/mempool/mempool
 MEMPOOL_REPO_NAME=mempool
 MEMPOOL_REPO_BRANCH=master
-MEMPOOL_LATEST_RELEASE=v2.3.1
+MEMPOOL_LATEST_RELEASE=master
 
 BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin
 BITCOIN_REPO_NAME=bitcoin
@@ -1016,7 +1016,7 @@ osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --releas
 echo "[*] Patching Bitcoin Electrs code for FreeBSD"
 osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
 osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i .bak -e s/Snappy/None/ db.rs && rm db.rs.bak"
-osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i .bak -e s/from_secs(5)/from_secs(1)/ electrs.rs && rm electrs.rs.bak"
+osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i .bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak"
 
 echo "[*] Building Bitcoin Electrs release binary"
 osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
@@ -1299,7 +1299,7 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
     osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
-if [ "${BISQ_ENABLE}" = ON ];then
+if [ "${BISQ_INSTALL}" = ON ];then
     echo "[*] Creating Mempool instance for Bisq"
     osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/bisq"

production/syslog.conf

@@ -1,4 +1,4 @@
 local7.>=notice |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops alerts
-local7.info |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops node100
+local7.>=info |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops node100
-local7.info /var/log/mempool
+local7.>=info /var/log/mempool
 local7.* /var/log/mempool.debug