diff --git a/README.md b/README.md index a6032a18a..db921d86d 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,10 @@ You will need [Bitcoin Core](https://github.com/bitcoin/bitcoin), [Electrum Serv Clone the Mempool repo, and checkout the latest release tag: ```bash -$ git clone https://github.com/mempool/mempool -$ cd mempool -$ latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4) -$ git checkout $latestrelease +git clone https://github.com/mempool/mempool +cd mempool +latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4) +git checkout $latestrelease ``` ### 2. Configure Bitcoin Core @@ -63,11 +63,11 @@ Install MariaDB from your OS package manager: ```bash # Debian, Ubuntu, etc. -$ apt-get install mariadb-server mariadb-client +apt-get install mariadb-server mariadb-client # macOS -$ brew install mariadb -$ mysql.server start +brew install mariadb +mysql.server start ``` Create a database and grant privileges: @@ -88,15 +88,15 @@ Query OK, 0 rows affected (0.00 sec) Install Mempool dependencies with npm and build the backend: ```bash -$ cd backend -$ npm install --prod -$ npm run build +cd backend +npm install --prod +npm run build ``` In the `backend` folder, make a copy of the sample config: ```bash -$ cp mempool-config.sample.json mempool-config.json +cp mempool-config.sample.json mempool-config.json ``` Edit `mempool-config.json` with your Bitcoin Core node RPC credentials: @@ -133,7 +133,7 @@ Edit `mempool-config.json` with your Bitcoin Core node RPC credentials: Start the backend: ```bash -$ npm run start +npm run start ``` When it's running, you should see output like this: @@ -164,15 +164,15 @@ Updating mempool Install the Mempool dependencies with npm and build the frontend: ```bash -$ cd frontend -$ npm install --prod -$ npm run build +cd frontend +npm install --prod +npm run build ``` Install the output into the nginx webroot folder: ```bash -$ sudo rsync -av --delete dist/ /var/www/ +sudo rsync -av --delete dist/ /var/www/ ``` ### 6. `nginx` + `certbot` @@ -181,13 +181,13 @@ Install the supplied `nginx.conf` and `nginx-mempool.conf` in `/etc/nginx`: ```bash # install nginx and certbot -$ apt-get install -y nginx python3-certbot-nginx +apt-get install -y nginx python3-certbot-nginx # install the mempool configuration for nginx -$ cp nginx.conf nginx-mempool.conf /etc/nginx/ +cp nginx.conf nginx-mempool.conf /etc/nginx/ # replace example.com with your domain name -$ certbot --nginx -d example.com +certbot --nginx -d example.com ``` If everything went well, you should see the beautiful mempool :grin: diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 633e797d1..6af631382 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -221,9 +221,10 @@ class Blocks { const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`); + loadingIndicators.setProgress('block-indexing', 0); const chunkSize = 10000; - let totaIndexed = await blocksRepository.$blockCount(null, null); + let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; const startedAt = new Date().getTime() / 1000; @@ -246,16 +247,17 @@ class Blocks { break; } ++indexedThisRun; - ++totaIndexed; + ++totalIndexed; const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); - const progress = Math.round(totaIndexed / indexingBlockAmount * 100); - const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds); - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; + const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds); + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; + loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); @@ -269,9 +271,11 @@ class Blocks { currentBlockHeight -= chunkSize; } logger.info(`Indexed ${newlyIndexed} blocks`); + loadingIndicators.setProgress('block-indexing', 100); } catch (e) { logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e)); this.blockIndexingStarted = false; + loadingIndicators.setProgress('block-indexing', 100); return; } diff --git a/backend/src/api/loading-indicators.ts b/backend/src/api/loading-indicators.ts index c2d682d1c..38d5aea96 100644 --- a/backend/src/api/loading-indicators.ts +++ b/backend/src/api/loading-indicators.ts @@ -12,8 +12,8 @@ class LoadingIndicators { this.progressChangedCallback = fn; } - public setProgress(name: string, progressPercent: number) { - const newProgress = Math.round(progressPercent); + public setProgress(name: string, progressPercent: number, rounded: boolean = true) { + const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent; if (newProgress >= 100) { delete this.loadingIndicators[name]; } else { diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 2c7530643..0909d6100 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -6,6 +6,7 @@ import bitcoinClient from './bitcoin/bitcoin-client'; import logger from '../logger'; import blocks from './blocks'; import { Common } from './common'; +import loadingIndicators from './loading-indicators'; class Mining { hashrateIndexingStarted = false; @@ -131,7 +132,7 @@ class Mining { * [INDEXING] Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { - if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) { + if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) { return; } @@ -167,7 +168,10 @@ class Mining { let indexedThisRun = 0; let totalIndexed = 0; let newlyIndexed = 0; - let startedAt = new Date().getTime(); + const startedAt = new Date().getTime() / 1000; + let timer = new Date().getTime() / 1000; + + loadingIndicators.setProgress('weekly-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 604800000; @@ -214,14 +218,17 @@ class Mining { await HashratesRepository.$saveHashrates(hashrates); hashrates.length = 0; - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime()) - startedAt)) / 1000; + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 1) { - const weeksPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2); + const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); + const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100; + const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds); const formattedDate = new Date(fromTimestamp).toUTCString(); - const weeksLeft = Math.round(totalWeekIndexed - totalIndexed); - logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds} weeks/sec | ~${weeksLeft} weeks left to index`); - startedAt = new Date().getTime(); + logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + timer = new Date().getTime() / 1000; indexedThisRun = 0; + loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false); } toTimestamp -= 604800000; @@ -233,7 +240,9 @@ class Mining { if (newlyIndexed > 0) { logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`); } + loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { + loadingIndicators.setProgress('weekly-hashrate-indexing', 100); this.weeklyHashrateIndexingStarted = false; throw e; } @@ -273,7 +282,10 @@ class Mining { let indexedThisRun = 0; let totalIndexed = 0; let newlyIndexed = 0; - let startedAt = new Date().getTime(); + const startedAt = new Date().getTime() / 1000; + let timer = new Date().getTime() / 1000; + + loadingIndicators.setProgress('daily-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 86400000; @@ -312,15 +324,17 @@ class Mining { hashrates.length = 0; } - const elapsedSeconds = Math.max(1, Math.round(new Date().getTime() - startedAt)) / 1000; + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 1) { - const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2); + const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); + const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100; + const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds); const formattedDate = new Date(fromTimestamp).toUTCString(); - const daysLeft = Math.round(totalDayIndexed - totalIndexed); - logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ` + - `~${daysLeft} days left to index`); - startedAt = new Date().getTime(); + logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + timer = new Date().getTime() / 1000; indexedThisRun = 0; + loadingIndicators.setProgress('daily-hashrate-indexing', progress); } toTimestamp -= 86400000; @@ -346,7 +360,9 @@ class Mining { if (newlyIndexed > 0) { logger.info(`Indexed ${newlyIndexed} day of network hashrate`); } + loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { + loadingIndicators.setProgress('daily-hashrate-indexing', 100); this.hashrateIndexingStarted = false; throw e; } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index c5267c461..e04080a9c 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -188,6 +188,24 @@ class BlocksRepository { } } + /** + * Get blocks count for a period + */ + public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise { + const params: any[] = []; + let query = `SELECT count(height) as blockCount + FROM blocks + WHERE height <= ${startHeight} AND height >= ${endHeight}`; + + try { + const [rows] = await DB.query(query, params); + return rows[0].blockCount; + } catch (e) { + logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Get the oldest indexed block */ diff --git a/docker/README.md b/docker/README.md index da7bf20e3..07590d0dd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -34,7 +34,7 @@ The IP address in the example above refers to Docker's default gateway IP addres Now, run: ```bash -$ docker-compose up +docker-compose up ``` Your Mempool instance should be running at http://localhost. The graphs will be populated as new transactions are detected. @@ -59,7 +59,7 @@ Of course, if your Docker host IP address is different, update accordingly. With `bitcoind` and Electrum Server set up, run Mempool with: ```bash -$ docker-compose up +docker-compose up ``` ## Further Configuration diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 8738c37f1..09e26e906 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -48,7 +48,8 @@ import { DashboardComponent } from './dashboard/dashboard.component'; import { DifficultyComponent } from './components/difficulty/difficulty.component'; import { FontAwesomeModule, FaIconLibrary } from '@fortawesome/angular-fontawesome'; import { faFilter, faAngleDown, faAngleUp, faAngleRight, faAngleLeft, faBolt, faChartArea, faCogs, faCubes, faHammer, faDatabase, faExchangeAlt, faInfoCircle, - faLink, faList, faSearch, faCaretUp, faCaretDown, faTachometerAlt, faThList, faTint, faTv, faAngleDoubleDown, faSortUp, faAngleDoubleUp, faChevronDown, faFileAlt, faRedoAlt, faArrowAltCircleRight, faExternalLinkAlt, faBook, faListUl } from '@fortawesome/free-solid-svg-icons'; + faLink, faList, faSearch, faCaretUp, faCaretDown, faTachometerAlt, faThList, faTint, faTv, faAngleDoubleDown, faSortUp, faAngleDoubleUp, faChevronDown, + faFileAlt, faRedoAlt, faArrowAltCircleRight, faExternalLinkAlt, faBook, faListUl, faDownload } from '@fortawesome/free-solid-svg-icons'; import { TermsOfServiceComponent } from './components/terms-of-service/terms-of-service.component'; import { PrivacyPolicyComponent } from './components/privacy-policy/privacy-policy.component'; import { TrademarkPolicyComponent } from './components/trademark-policy/trademark-policy.component'; @@ -75,6 +76,8 @@ import { DataCyDirective } from './data-cy.directive'; import { BlockFeesGraphComponent } from './components/block-fees-graph/block-fees-graph.component'; import { BlockRewardsGraphComponent } from './components/block-rewards-graph/block-rewards-graph.component'; import { BlockFeeRatesGraphComponent } from './components/block-fee-rates-graph/block-fee-rates-graph.component'; +import { LoadingIndicatorComponent } from './components/loading-indicator/loading-indicator.component'; +import { IndexingProgressComponent } from './components/indexing-progress/indexing-progress.component'; @NgModule({ declarations: [ @@ -131,6 +134,8 @@ import { BlockFeeRatesGraphComponent } from './components/block-fee-rates-graph/ BlockFeesGraphComponent, BlockRewardsGraphComponent, BlockFeeRatesGraphComponent, + LoadingIndicatorComponent, + IndexingProgressComponent, ], imports: [ BrowserModule.withServerTransition({ appId: 'serverApp' }), @@ -195,5 +200,6 @@ export class AppModule { library.addIcons(faAngleLeft); library.addIcons(faBook); library.addIcons(faListUl); + library.addIcons(faDownload); } } diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index 0ac64f86d..fa9e14651 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -134,7 +134,7 @@ ({{ error.error }}) - The number of transactions on this address exceeds the Electrum server limit + There many transactions on this address, more than your backend can handle. See more on setting up a stronger backend.

Consider viewing this address on the official Mempool website instead: diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html index 1fb97407b..f15356c2f 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html @@ -1,6 +1,12 @@ + +
Block fee rates + +