From e0952a4c1d7fd1fd41ccd2360c1ba34e6eb07a05 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 16 Jul 2022 09:22:45 +0200 Subject: [PATCH 1/5] Wait for the price updater to complete before saving blocks prices --- backend/src/api/blocks.ts | 15 ++++++++++ backend/src/api/mining/mining.ts | 14 +++++----- backend/src/indexer.ts | 29 +++++++++++++++++--- backend/src/repositories/PricesRepository.ts | 5 ++++ backend/src/tasks/price-updater.ts | 9 +++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 30f9fbf78..45ffd6079 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,6 +22,8 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import PricesRepository from '../repositories/PricesRepository'; +import priceUpdater from '../tasks/price-updater'; class Blocks { private blocks: BlockExtended[] = []; @@ -457,6 +459,19 @@ class Blocks { } await blocksRepository.$saveBlockInDatabase(blockExtended); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === true && lastestPriceId !== null) { + await blocksRepository.$saveBlockPrices([{ + height: blockExtended.height, + priceId: lastestPriceId, + }]); + } else { + logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`) + setTimeout(() => { + indexer.runSingleTask('blocksPrices'); + }, 10000); + } + // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 55e749596..55cd33bd3 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -473,7 +473,7 @@ class Mining { for (const block of blocksWithoutPrices) { // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks - if (block.height < 68951) { + if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) { blocksPrices.push({ height: block.height, priceId: prices[0].id, @@ -492,11 +492,11 @@ class Mining { if (blocksPrices.length >= 100000) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); blocksPrices.length = 0; } @@ -504,11 +504,11 @@ class Mining { if (blocksPrices.length > 0) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); } } catch (e) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index e452a42f4..26a407291 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -6,13 +6,12 @@ import logger from './logger'; import HashratesRepository from './repositories/HashratesRepository'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; +import PricesRepository from './repositories/PricesRepository'; class Indexer { runIndexer = true; indexerRunning = false; - - constructor() { - } + tasksRunning: string[] = []; public reindex() { if (Common.indexingEnabled()) { @@ -20,6 +19,28 @@ class Indexer { } } + public async runSingleTask(task: 'blocksPrices') { + if (!Common.indexingEnabled()) { + return; + } + + if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === false || lastestPriceId === null) { + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + setTimeout(() => { + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.runSingleTask('blocksPrices'); + }, 10000); + } else { + logger.debug(`Blocks prices indexer will run now`) + await mining.$indexBlockPrices(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + } + } + } + public async $run() { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() @@ -50,7 +71,7 @@ class Indexer { return; } - await mining.$indexBlockPrices(); + this.runSingleTask('blocksPrices'); await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await mining.$generateNetworkHashrateHistory(); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 92fb4860f..cc79ff2a6 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -27,6 +27,11 @@ class PricesRepository { return oldestRow[0] ? oldestRow[0].time : 0; } + public async $getLatestPriceId(): Promise { + const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + return oldestRow[0] ? oldestRow[0].id : null; + } + public async $getLatestPriceTime(): Promise { const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index a5901d7f7..81066efb2 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import { Common } from '../api/common'; import config from '../config'; import logger from '../logger'; import PricesRepository from '../repositories/PricesRepository'; @@ -34,10 +35,10 @@ export interface Prices { } class PriceUpdater { - historyInserted: boolean = false; - lastRun: number = 0; - lastHistoricalRun: number = 0; - running: boolean = false; + public historyInserted = false; + lastRun = 0; + lastHistoricalRun = 0; + running = false; feeds: PriceFeed[] = []; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; latestPrices: Prices; From cacd4abd9d8f5153454ba2ef2ccde20712ecc30b Mon Sep 17 00:00:00 2001 From: Stephan Oeste Date: Fri, 5 Aug 2022 16:41:00 +0200 Subject: [PATCH 2/5] Add Core Lighting for FreeBSD in prod installer --- production/install | 52 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/production/install b/production/install index 729ff33e0..db48abaab 100755 --- a/production/install +++ b/production/install @@ -34,10 +34,11 @@ esac TOR_INSTALL=ON CERTBOT_INSTALL=ON -# install 3 network daemons +# install 4 network daemons BITCOIN_INSTALL=ON BISQ_INSTALL=ON ELEMENTS_INSTALL=ON +CLN_INSTALL=ON # configure 4 network instances BITCOIN_MAINNET_ENABLE=ON @@ -187,6 +188,7 @@ case $OS in NGINX_ETC_FOLDER=/usr/local/etc/nginx NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf CERTBOT_PKG=py39-certbot + CLN_PKG=c-lightning ;; Debian) @@ -272,6 +274,12 @@ ELECTRS_LIQUID_DATA=${ELECTRS_DATA_ROOT}/liquid ELECTRS_LIQUIDTESTNET_ZPOOL=${ZPOOL} ELECTRS_LIQUIDTESTNET_DATA=${ELECTRS_DATA_ROOT}/liquidtestnet +# Core Lightning user/group +CLN_USER=cln +CLN_GROUP=cln +# Core Lightning home folder +CLN_HOME=/cln + # bisq user/group BISQ_USER=bisq BISQ_GROUP=bisq @@ -572,6 +580,10 @@ zfsCreateFilesystems() done fi + if [ "${CLN_INSTALL}" = ON ];then + zfs create -o "mountpoint=${CLN_HOME}" "${ZPOOL}/cln" + fi + if [ "${BISQ_INSTALL}" = ON ];then zfs create -o "mountpoint=${BISQ_HOME}" "${ZPOOL}/bisq" fi @@ -651,6 +663,10 @@ ext4CreateDir() done fi + if [ "${CLN_INSTALL}" = ON ];then + mkdir -p "${CLN_HOME}" + fi + if [ "${BISQ_INSTALL}" = ON ];then mkdir -p "${BISQ_HOME}" fi @@ -711,6 +727,7 @@ Testnet:Enable Bitcoin Testnet:ON Signet:Enable Bitcoin Signet:ON Liquid:Enable Elements Liquid:ON Liquidtestnet:Enable Elements Liquidtestnet:ON +CoreLN:Enable Core Lightning:ON Bisq:Enable Bisq:ON EOF @@ -785,6 +802,12 @@ else ELEMENTS_INSTALL=OFF fi +if grep CoreLN $tempfile >/dev/null 2>&1;then + CLN_INSTALL=ON +else + CLN_INSTALL=OFF +fi + if grep Bisq $tempfile >/dev/null 2>&1;then BISQ_INSTALL=ON BISQ_MAINNET_ENABLE=ON @@ -1165,6 +1188,33 @@ if [ "${ELEMENTS_INSTALL}" = ON ;then osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true fi +##################################### +# Core Lightning for Bitcoin Mainnet # +##################################### + +echo "[*] Installing Core Lightning" +case $OS in + FreeBSD) + echo "[*] Creating Core Lightning user" + osGroupCreate "${CLN_GROUP}" + osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" + osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" + osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" + osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc" + + echo "[*] Installing Core Lightning package" + osPackageInstall ${CLN_PKG} + + echo "[*] Installing Core Lightning mainnet Cronjob" + crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n' + crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n' + crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n' + echo "${crontab_cln}" | crontab -u "${CLN_USER}" - + ;; + Debian) + ;; +esac + ##################### # Bisq installation # ##################### From a64cb4bbad104da0f2f7240702785ca6ba0d8ba2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 15:52:24 +0200 Subject: [PATCH 3/5] Make mining pools url configurable --- backend/src/config.ts | 4 ++++ backend/src/tasks/pools-updater.ts | 13 +++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/src/config.ts b/backend/src/config.ts index d4dfc9edd..3d7a05242 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -24,6 +24,8 @@ interface IConfig { USER_AGENT: string; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; AUTOMATIC_BLOCK_REINDEXING: boolean; + POOLS_JSON_URL: string, + POOLS_JSON_TREE_URL: string, }; ESPLORA: { REST_API_URL: string; @@ -135,6 +137,8 @@ const defaults: IConfig = { 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', 'AUTOMATIC_BLOCK_REINDEXING': false, + 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', + 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 04d9d5d07..11bb8060f 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -12,14 +12,11 @@ import * as https from 'https'; */ class PoolsUpdater { lastRun: number = 0; - currentSha: any = undefined; - poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'; - treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master'; + currentSha: string | undefined = undefined; + poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL; + treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL; - constructor() { - } - - public async updatePoolsJson() { + public async updatePoolsJson(): Promise { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } @@ -77,7 +74,7 @@ class PoolsUpdater { /** * Fetch our latest pools.json sha from the db */ - private async updateDBSha(githubSha: string) { + private async updateDBSha(githubSha: string): Promise { this.currentSha = githubSha; if (config.DATABASE.ENABLED === true) { try { From aed37afb3e3bbc5cf1ddbb1578984b8bfc74c4b3 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 9 Aug 2022 17:26:14 +0200 Subject: [PATCH 4/5] Add missing config in docker script and sample --- backend/mempool-config.sample.json | 4 +++- docker/README.md | 6 +++++- docker/backend/start.sh | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 312d9d18d..e636c9e2d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -21,7 +21,9 @@ "EXTERNAL_RETRY_INTERVAL": 0, "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug", - "AUTOMATIC_BLOCK_REINDEXING": false + "AUTOMATIC_BLOCK_REINDEXING": false, + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/docker/README.md b/docker/README.md index 14b80c19e..dd42462a7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,7 +102,9 @@ Below we list all settings from `mempool-config.json` and the corresponding over "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], - "STDOUT_LOG_MIN_PRIORITY": "info" + "STDOUT_LOG_MIN_PRIORITY": "info", + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" }, ``` @@ -126,6 +128,8 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: "" MEMPOOL_EXTERNAL_ASSETS: "" MEMPOOL_STDOUT_LOG_MIN_PRIORITY: "" + MEMPOOL_POOLS_JSON_URL: "" + MEMPOOL_POOLS_JSON_TREE_URL: "" ... ``` diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c31273bb6..3d754c979 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -24,6 +24,8 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} +__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=false} +__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=false} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -114,6 +116,8 @@ sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.jso sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json +sed -i "s/__MEMPOOL_POOLS_JSON_URL__/${__MEMPOOL_POOLS_JSON_URL__}/g" mempool-config.json +sed -i "s/__MEMPOOL_POOLS_JSON_TREE_URL__/${__MEMPOOL_POOLS_JSON_TREE_URL__}/g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json From 2b2c40f65a9c81fbf257502a3ecc7f69e12c3e3a Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 10 Aug 2022 22:37:22 +0900 Subject: [PATCH 5/5] [ops] Fix minor issue for /cln/.zshrc in prod installer --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index db48abaab..f2ecc5dd1 100755 --- a/production/install +++ b/production/install @@ -1199,8 +1199,8 @@ case $OS in osGroupCreate "${CLN_GROUP}" osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}" osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}" - osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc" + osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}" echo "[*] Installing Core Lightning package" osPackageInstall ${CLN_PKG}