Merge branch 'master' into knorrium/backend_unit_tests

backend/src/api/pools-parser.ts

@@ -1,6 +1,7 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
+import BlocksRepository from '../repositories/BlocksRepository';
 
 interface Pool {
   name: string;
@@ -32,7 +33,6 @@ class PoolsParser {
     // First we save every entries without paying attention to pool duplication
     const poolsDuplicated: Pool[] = [];
 
-    logger.debug('Parse coinbase_tags');
     const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
     for (let i = 0; i < coinbaseTags.length; ++i) {
       poolsDuplicated.push({
@@ -43,7 +43,6 @@ class PoolsParser {
         'slug': ''
       });
     }
-    logger.debug('Parse payout_addresses');
     const addressesTags = Object.entries(poolsJson['payout_addresses']);
     for (let i = 0; i < addressesTags.length; ++i) {
       poolsDuplicated.push({
@@ -56,7 +55,6 @@ class PoolsParser {
     }
 
     // Then, we find unique mining pool names
-    logger.debug('Identify unique mining pools');
     const poolNames: string[] = [];
     for (let i = 0; i < poolsDuplicated.length; ++i) {
       if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
@@ -119,8 +117,15 @@ class PoolsParser {
         'slug': slug
       };
 
-      if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
+      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
+      if (existingPool !== undefined) {
+        // Check if any data was actually updated
+        const equals = (a, b) =>
+          a.length === b.length &&
+          a.every((v, i) => v === b[i]);
+        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
           finalPoolDataUpdate.push(poolObj);
+        }
       } else {
         logger.debug(`Add '${finalPoolName}' mining pool`);
         finalPoolDataAdd.push(poolObj);
@@ -140,6 +145,7 @@ class PoolsParser {
       return;
     }
 
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0) {    
       logger.debug(`Update pools table now`);
 
       // Add new mining pools into the database
@@ -164,6 +170,8 @@ class PoolsParser {
       }
 
       try {
+        await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+
         if (finalPoolDataAdd.length > 0) {
           await DB.query({ sql: queryAdd, timeout: 120000 });
         }
@@ -178,6 +186,14 @@ class PoolsParser {
       }
     }
 
+    try {
+      await this.insertUnknownPool();
+    } catch (e) {
+      logger.err(`Cannot insert unknown pool in the database`);
+      throw e;
+    }
+  }
+
   /**
    * Manually add the 'unknown pool'
    */
@@ -201,6 +217,32 @@ class PoolsParser {
       logger.err('Unable to insert "Unknown" mining pool');
     }
   }
+
+  /**
+   * Delete blocks which needs to be reindexed
+   */
+   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+    const blockCount = await BlocksRepository.$blockCount(null, null);
+    if (blockCount === 0) {
+      return;
+    }
+
+    for (const updatedPool of finalPoolDataUpdate) {
+      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
+      if (pool.length > 0) {
+        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
+        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
+      }
+    }
+
+    // Ignore early days of Bitcoin as there were not mining pool yet
+    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+
+    logger.notice('Truncating hashrates for future re-indexing');
+    await DB.query(`DELETE FROM hashrates`);
+  }
 }
 
 export default new PoolsParser();

production/install

@@ -183,6 +183,9 @@ case $OS in
         TOR_PKG=tor
         TOR_USER=_tor
         TOR_GROUP=_tor
+        NGINX_USER=www
+        NGINX_ETC_FOLDER=/usr/local/etc/nginx
+        NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf
         CERTBOT_PKG=py39-certbot
     ;;
 
@@ -197,6 +200,7 @@ case $OS in
         TOR_USER=debian-tor
         TOR_GROUP=debian-tor
         CERTBOT_PKG=python3-certbot-nginx
+        NGINX_USER=www-data
         NGINX_ETC_FOLDER=/etc/nginx
         NGINX_CONFIGURATION=/etc/nginx/nginx.conf
     ;;
@@ -1534,26 +1538,17 @@ NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
 NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
 
 echo "[*] Adding Nginx configuration"
-case $OS in
+osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
-
+mkdir -p /var/cache/nginx/services /var/cache/nginx/api
-    FreeBSD)
+chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api
-        echo "[*] FIXME: nginx must be configured manually on FreeBSD"
+ln -s /mempool/mempool /etc/nginx/mempool
-    ;;
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
-
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"
-    Debian)
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-        osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-        mkdir -p /var/cache/nginx/services /var/cache/nginx/api
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
-        chown www-data: /var/cache/nginx/services /var/cache/nginx/api
+echo "[*] Restarting Nginx"
-        ln -s /mempool/mempool /etc/nginx/mempool
+osSudo "${ROOT_USER}" service nginx restart
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!www-data!" "${NGINX_CONFIGURATION}"
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
-        echo "[*] Restarting Nginx"
-        osSudo "${ROOT_USER}" service nginx restart
-    ;;
-esac
 
 ##### OS systemd