Merge branch 'master' into simon/angular-13.3-update

README.md

@@ -10,14 +10,7 @@ It is an open-source project developed and operated for the benefit of the Bitco
 
 Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
 
-We support the following installation methods, ranked in order from simple to advanced:
+**Most people should use a one-click install method.** Other install methods are meant for developers and others with experience managing servers. 
-
-1) [One-click installation on full-node distros](#one-click-installation)
-2) [Docker installation on Linux using docker-compose](./docker)
-3) [Manual installation on Linux or FreeBSD](#manual-installation)
-4) [Production installation on a powerful FreeBSD server](./production)
-
-This doc offers install notes on the one-click method and manual install method. Follow the links above for install notes on Docker and production installations.
 
 <a id="one-click-installation"></a>
 ## One-Click Installation
@@ -29,167 +22,12 @@ Mempool can be conveniently installed on the following full-node distros:
 - [myNode](https://github.com/mynodebtc/mynode)
 - [Start9](https://github.com/Start9Labs/embassy-os)
 
-<a id="manual-installation"></a>
+**We highly recommend you deploy your own Mempool instance this way.** No matter which option you pick, you'll be able to get your own fully-sovereign instance of Mempool up quickly without needing to fiddle with any settings.
-## Manual Installation
 
-The following instructions are for a manual installation on Linux or FreeBSD. You may need to change file and directory paths to match your OS.
+## Advanced Installation Methods
 
-You will need [Bitcoin Core](https://github.com/bitcoin/bitcoin), [Electrum Server](https://github.com/romanz/electrs), [Node.js](https://github.com/nodejs/node), [MariaDB](https://github.com/mariadb/server), and [Nginx](https://github.com/nginx/nginx). Below, we walk through how to configure each of these.
+Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
-### 1. Get Latest Mempool Release
+- See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
-
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers and small-scale deployments.
-Clone the Mempool repo, and checkout the latest release tag:
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.
-
-```bash
-git clone https://github.com/mempool/mempool
-cd mempool
-latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-git checkout $latestrelease
-```
-
-### 2. Configure Bitcoin Core
-
-Enable RPC and txindex in `bitcoin.conf`:
-
-```bash
-rpcuser=mempool
-rpcpassword=mempool
-txindex=1
-```
-
-### 3. Get & Configure MySQL
-
-Install MariaDB from your OS package manager:
-
-```bash
-# Debian, Ubuntu, etc.
-apt-get install mariadb-server mariadb-client
-
-# macOS
-brew install mariadb
-mysql.server start
-```
-
-Create a database and grant privileges:
-
-```bash
-MariaDB [(none)]> drop database mempool;
-Query OK, 0 rows affected (0.00 sec)
-
-MariaDB [(none)]> create database mempool;
-Query OK, 1 row affected (0.00 sec)
-
-MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
-Query OK, 0 rows affected (0.00 sec)
-```
-
-### 4. Build Mempool Backend
-
-Install Mempool dependencies with npm and build the backend:
-
-```bash
-cd backend
-npm install --prod
-npm run build
-```
-
-In the `backend` folder, make a copy of the sample config:
-
-```bash
-cp mempool-config.sample.json mempool-config.json
-```
-
-Edit `mempool-config.json` with your Bitcoin Core node RPC credentials:
-
-```bash
-{
-  "MEMPOOL": {
-    "NETWORK": "mainnet",
-    "BACKEND": "electrum",
-    "HTTP_PORT": 8999
-  },
-  "CORE_RPC": {
-    "HOST": "127.0.0.1",
-    "PORT": 8332,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-  "ELECTRUM": {
-    "HOST": "127.0.0.1",
-    "PORT": 50002,
-    "TLS_ENABLED": true
-  },
-  "DATABASE": {
-    "ENABLED": true,
-    "HOST": "127.0.0.1",
-    "PORT": 3306,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "DATABASE": "mempool"
-  }
-}
-```
-
-Start the backend:
-
-```bash
-npm run start
-```
-
-When it's running, you should see output like this:
-
-```bash
-Mempool updated in 0.189 seconds
-Updating mempool
-Mempool updated in 0.096 seconds
-Updating mempool
-Mempool updated in 0.099 seconds
-Updating mempool
-Calculated fee for transaction 1 / 10
-Calculated fee for transaction 2 / 10
-Calculated fee for transaction 3 / 10
-Calculated fee for transaction 4 / 10
-Calculated fee for transaction 5 / 10
-Calculated fee for transaction 6 / 10
-Calculated fee for transaction 7 / 10
-Calculated fee for transaction 8 / 10
-Calculated fee for transaction 9 / 10
-Calculated fee for transaction 10 / 10
-Mempool updated in 0.243 seconds
-Updating mempool
-```
-
-### 5. Build Mempool Frontend
-
-Install the Mempool dependencies with npm and build the frontend:
-
-```bash
-cd frontend
-npm install --prod
-npm run build
-```
-
-Install the output into the nginx webroot folder:
-
-```bash
-sudo rsync -av --delete dist/ /var/www/
-```
-
-### 6. `nginx` + `certbot`
-
-Install the supplied `nginx.conf` and `nginx-mempool.conf` in `/etc/nginx`:
-
-```bash
-# install nginx and certbot
-apt-get install -y nginx python3-certbot-nginx
-
-# install the mempool configuration for nginx
-cp nginx.conf nginx-mempool.conf /etc/nginx/
-
-# replace example.com with your domain name
-certbot --nginx -d example.com
-```
-
-If everything went well, you should see the beautiful mempool :grin:
-
-If you get stuck on "loading blocks", this means the websocket can't connect. Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.

backend/README.md

@@ -1,22 +1,161 @@
-# Setup backend watchers
+# Mempool Backend
 
-The backend is static. Typescript scripts are compiled into the `dist` folder and served through a node web server.
+These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 
 
-You can avoid the manual shutdown/recompile/restart command line cycle by using a watcher.
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
 
-Make sure you are in the `backend` directory `cd backend`.
+See other ways to set up Mempool on [the main README](/../../#installation-methods).
 
-1. Install nodemon and ts-node
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
 
 ```
-sudo npm install -g ts-node nodemon
+git clone https://github.com/mempool/mempool
+cd mempool
 ```
 
-2. Run the watcher
+Check out the latest release:
 
-> Note: You can find your npm global binary folder using `npm -g bin`, where nodemon will be installed.
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Node.js 16 and npm 7 are recommended._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install # add --prod for production
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:
 
 ```
 nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```
 
+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.

docker/README.md

@@ -1,18 +1,23 @@
 # Docker Installation
 
-This directory contains the Dockerfiles used to build and release the official images and a `docker-compose.yml` for end users to run a Mempool instance with minimal effort.
+This directory contains the Dockerfiles used to build and release the official images, as well as a `docker-compose.yml` to configure environment variables and other settings.
 
-You can choose to configure Mempool to run with a basic backend powered by just `bitcoind`, or with `bitcoind` along with an Electrum-compatible server for full functionality.
+If you are looking to use these Docker images to deploy your own instance of Mempool, note that they only containerize Mempool's frontend and backend. You will still need to deploy and configure Bitcoin Core and an Electrum Server separately, along with any other utilities specific to your use case (e.g., a reverse proxy, etc). Such configuration is mostly beyond the scope of the Mempool project, so please only proceed if you know what you're doing.
 
-## `bitcoind`-only Configuration
+Jump to a section in this doc:
+- [Configure with Bitcoin Core Only](#configure-with-bitcoin-core-only)
+- [Configure with Bitcoin Core + Electrum Server](#configure-with-bitcoin-core--electrum-server)
+- [Further Configuration](#further-configuration)
 
-_Note: address lookups require an Electrum server and will not work with this configuration._
+## Configure with Bitcoin Core Only
 
-Make sure `bitcoind` is running and synced.
+_Note: address lookups require an Electrum Server and will not work with this configuration. [Add an Electrum Server](#configure-with-bitcoin-core--electrum-server) to your backend for full functionality._
 
-The default Docker configuration assumes you have added RPC credentials for a `mempool` user with a `mempool` password in your `bitcoin.conf` file, like so:
+The default Docker configuration assumes you have the following configuration in your `bitcoin.conf` file:
 
 ```
+txindex=1
+server=1
 rpcuser=mempool
 rpcpassword=mempool
 ```
@@ -31,6 +36,8 @@ If you want to use different credentials, specify them in the `docker-compose.ym
 
 The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
 
+Make sure `bitcoind` is running and synced.
+
 Now, run:
 
 ```bash
@@ -39,11 +46,11 @@ docker-compose up
 
 Your Mempool instance should be running at http://localhost. The graphs will be populated as new transactions are detected.
 
-## `bitcoind` + Electrum Server Configuration
+## Configure with Bitcoin Core + Electrum Server
 
-First, configure `bitcoind` as specified above, and make sure your Electrum server is running and synced.
+First, configure `bitcoind` as specified above, and make sure your Electrum Server is running and synced. See [this FAQ](https://mempool.space/docs/faq#address-lookup-issues) if you need help picking an Electrum Server implementation.
 
-Then, make sure the following variables are set in `docker-compose.yml`, as shown below, so Mempool can connect to your Electrum server:
+Then, set the following variables in `docker-compose.yml` so Mempool can connect to your Electrum Server:
 
 ```
   api:
@@ -54,6 +61,11 @@ Then, make sure the following variables are set in `docker-compose.yml`, as show
       ELECTRUM_TLS_ENABLED: "false"
 ```
 
+Eligible values for `MEMPOOL_BACKEND`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
 Of course, if your Docker host IP address is different, update accordingly.
 
 With `bitcoind` and Electrum Server set up, run Mempool with:

frontend/README.md

@@ -1,8 +1,30 @@
-# mempool-frontend
+# Mempool Frontend
 
-## Contributing
+You can build and run the Mempool frontend and proxy to the production Mempool backend (for easier frontend development), or you can connect it to your own backend for a full Mempool development instance, custom deployment, etc.
 
-This package is used for the https://mempool.space, https://liquid.network and https://bisq.markets websites - there are npm scripts to setup all three, which effectively change how BASE_MODULE is  configured:
+Jump to a section in this doc:
+- [Quick Setup for Frontend Development](#quick-setup-for-frontend-development)
+- [Manual Frontend Setup](#manual-setup)
+- [Translations](#translations-transifex-project)
+
+## Quick Setup for Frontend Development
+
+If you want to quickly improve the UI, fix typos, or make other updates that don't require any backend changes, you don't need to set up an entire backend—you can simply run the Mempool frontend locally and proxy to the mempool.space backend.
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
+
+```
+git clone https://github.com/mempool/mempool
+cd mempool
+```
+
+### 2. Specify Website
+
+The same frontend codebase is used for https://mempool.space, https://liquid.network and https://bisq.markets.
+
+Configure the frontend for the site you want by running the corresponding command:
 
 ```
 $ npm run config:defaults:mempool
@@ -10,18 +32,22 @@ $ npm run config:defaults:liquid
 $ npm run config:defaults:bisq
 ```
 
-Changes that affect the frontend codebase only can be done using the production backend so you don't need to spin up the entire Mempool infrastructure. This is very convenient in case you want to quickly improve the UI, fix typos or implement new features that don't require any backend changes.
+### 3. Run the Frontend
 
-Make your changes, install the project dependencies and run the frontend server as follows:
+_Node.js 16 and npm 7 are recommended._
+
+Install project dependencies and run the frontend server:
 
 ```
 $ npm install
 $ npm run serve:local-prod
 ```
 
-The frontend will be available at http://localhost:4200/ and all API requests will be proxied to the production server at https://mempool.space
+The frontend will be available at http://localhost:4200/ and all API requests will be proxied to the production server at https://mempool.space.
 
-After making your changes, you can run our end-to-end automation suite and check for possible regressions:
+### 4. Test
+
+After making your changes, you can run our end-to-end automation suite and check for possible regressions.
 
 Headless:
 
@@ -37,11 +63,43 @@ $ npm run config:defaults:mempool && npm run cypress:open
 
 This will open the Cypress test runner, where you can select any of the test files to run.
 
-If all tests are green, submit your PR and it will be reviewed by someone on the team as soon as possible.
+If all tests are green, submit your PR, and it will be reviewed by someone on the team as soon as possible.
+
+## Manual Setup
+
+Set up the [Mempool backend](../backend/) first, if you haven't already.
+
+### 1. Build the Frontend
+
+_Node.js 16 and npm 7 are recommended._
+
+Build the frontend:
+
+```
+cd frontend
+npm install # add --prod for production
+npm run build
+```
+
+### 2. Run the Frontend
+
+#### Development
+
+To run your local Mempool frontend with your local Mempool backend:
+
+```
+npm run serve
+```
+
+#### Production
+
+The `npm run build` command from step 1 above should have generated a `dist` directory. Put the contents of `dist/` onto your web server.
+
+You will probably want to set up a reverse proxy, TLS, etc. There are sample nginx configuration files in the top level of the repository for reference, but note that support for such tasks is outside the scope of this project.
 
 ## Translations: Transifex Project
 
-The mempool frontend strings are localized into 20+ locales:
+The Mempool frontend strings are localized into 20+ locales:
 https://www.transifex.com/mempool/mempool/dashboard/
 
 ### Translators

frontend/src/app/bisq/bisq.module.ts

@@ -7,7 +7,6 @@ import { LightweightChartsComponent } from './lightweight-charts/lightweight-cha
 import { LightweightChartsAreaComponent } from './lightweight-charts-area/lightweight-charts-area.component';
 import { BisqMarketComponent } from './bisq-market/bisq-market.component';
 import { BisqTransactionsComponent } from './bisq-transactions/bisq-transactions.component';
-import { NgbPaginationModule } from '@ng-bootstrap/ng-bootstrap';
 import { BisqTransactionComponent } from './bisq-transaction/bisq-transaction.component';
 import { BisqBlockComponent } from './bisq-block/bisq-block.component';
 import { BisqDashboardComponent } from './bisq-dashboard/bisq-dashboard.component';
@@ -50,7 +49,6 @@ import { CommonModule } from '@angular/common';
     CommonModule,
     BisqRoutingModule,
     SharedModule,
-    NgbPaginationModule,
     FontAwesomeModule,
     NgxBootstrapMultiselectModule,
   ],

frontend/src/app/components/block/block.component.html

@@ -114,7 +114,7 @@
                   <td><span class="skeleton-loader"></span></td>
                 </tr>
               </ng-template>
-              <tr>
+              <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
                 <td i18n="block.miner">Miner</td>
                 <td *ngIf="stateService.env.MINING_DASHBOARD">
                   <a placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block.extras.pool.slug]" class="badge"

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -2,6 +2,7 @@
 
 <div class="container-xl" style="min-height: 335px" [ngClass]="{'widget': widget, 'full-height': !widget, 'legacy': !indexingAvailable}">
   <h1 *ngIf="!widget" class="float-left" i18n="master-page.blocks">Blocks</h1>
+  <div *ngIf="!widget && isLoading" class="spinner-border ml-3" role="status"></div>
 
   <div class="clearfix"></div>
 

frontend/src/app/components/blocks-list/blocks-list.component.scss

@@ -1,3 +1,9 @@
+.spinner-border {
+  height: 25px;
+  width: 25px;
+  margin-top: 13px;
+}
+
 .container-xl {
   max-width: 1400px;
   padding-bottom: 100px;

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -5989,6 +5989,14 @@ export const faqData = [
     title: "What does it mean for the mempool to be \"full\"?",
     answer: "<p>When a Bitcoin transaction is made, it is stored in a Bitcoin node's mempool before it is confirmed into a block. When the rate of incoming transactions exceeds the rate transactions are confirmed, the mempool grows in size.</p><p>The default maximum size of a Bitcoin node's mempool is 300MB, so when there are 300MB of transactions in the mempool, we say it's \"full\".</p>"
   },
+  {
+    type: "endpoint",
+    category: "basics",
+    showConditions: bitcoinNetworks,
+    fragment: "why-empty-blocks",
+    title: "Why are there empty blocks?",
+    answer: "<p>When a new block is found, mining pools send miners a block template with no transactions so they can start searching for the next block as soon as possible. They send a block template full of transactions right afterward, but a full block template is a bigger data transfer and takes slightly longer to reach miners.</p><p>In this intervening time, which is usually no more than 1-2 seconds, miners sometimes get lucky and find a new block using the empty block template.</p>"
+  },
   {
     type: "category",
     category: "help",

frontend/src/app/shared/shared.module.ts

@@ -1,7 +1,6 @@
 import { NgModule } from '@angular/core';
 import { CommonModule } from '@angular/common';
-import { NgbModule } from '@ng-bootstrap/ng-bootstrap';
+import { NgbCollapse, NgbCollapseModule, NgbRadioGroup, NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap';
-import { NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap';
 import { FontAwesomeModule, FaIconLibrary } from '@fortawesome/angular-fontawesome';
 import { faFilter, faAngleDown, faAngleUp, faAngleRight, faAngleLeft, faBolt, faChartArea, faCogs, faCubes, faHammer, faDatabase, faExchangeAlt, faInfoCircle,
   faLink, faList, faSearch, faCaretUp, faCaretDown, faTachometerAlt, faThList, faTint, faTv, faAngleDoubleDown, faSortUp, faAngleDoubleUp, faChevronDown,
@@ -146,11 +145,10 @@ import { IndexingProgressComponent } from '../components/indexing-progress/index
     NgbTooltipModule,
     NgbButtonsModule,
     NgbPaginationModule,
-    NgbDropdownModule,
-    NgbAccordionModule,
-    InfiniteScrollModule,
     NgbTypeaheadModule,
-    NgbModule,
+    NgbDropdownModule,
+    NgbCollapseModule,
+    InfiniteScrollModule,
     FontAwesomeModule,
   ],
   providers: [
@@ -162,17 +160,16 @@ import { IndexingProgressComponent } from '../components/indexing-progress/index
   ],
   exports: [
     RouterModule,
-    InfiniteScrollModule,
-    NgbTypeaheadModule,
-    NgbModule,
-    FontAwesomeModule,
-    NgbAccordionModule,
-    NgbNavModule,
     ReactiveFormsModule,
+    NgbNavModule,
     NgbTooltipModule,
     NgbButtonsModule,
     NgbPaginationModule,
+    NgbTypeaheadModule,
     NgbDropdownModule,
+    NgbCollapseModule,
+    InfiniteScrollModule,
+    FontAwesomeModule,
     TimeSinceComponent,
     TimeUntilComponent,
     ClipboardComponent,