Merge pull request #317 from bguillaumat/add-cache-dir

Add Dockerization of Mempool v2 for Umbrel App Store

backend/mempool-config.sample.json

@@ -5,7 +5,8 @@
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 2000
+    "POLL_RATE_MS": 2000,
+    "CACHE_DIR": "./"
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/package.json

@@ -36,6 +36,7 @@
     "locutus": "^2.0.12",
     "mysql2": "2.2.5",
     "node-worker-threads-pool": "^1.4.2",
+    "typescript": "~3.9.7",
     "ws": "^7.3.1"
   },
   "devDependencies": {

backend/src/api/disk-cache.ts

@@ -4,10 +4,11 @@ import * as cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
+import config from '../config';
 
 class DiskCache {
-  private static FILE_NAME = './cache.json';
+  private static FILE_NAME = config.MEMPOOL.CACHE_DIR + 'cache.json';
-  private static FILE_NAMES = './cache{number}.json';
+  private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + 'cache{number}.json';
   private static CHUNK_SIZE = 10000;
   constructor() { }
 

backend/src/config.ts

@@ -8,6 +8,7 @@ interface IConfig {
     SPAWN_CLUSTER_PROCS: number;
     API_URL_PREFIX: string;
     POLL_RATE_MS: number;
+    CACHE_DIR: string;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -66,7 +67,8 @@ const defaults: IConfig = {
     'HTTP_PORT': 8999,
     'SPAWN_CLUSTER_PROCS': 0,
     'API_URL_PREFIX': '/api/v1/',
-    'POLL_RATE_MS': 2000
+    'POLL_RATE_MS': 2000,
+    'CACHE_DIR': './'
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

docker/README.md

@@ -0,0 +1,5 @@
+## Init
+./docker/init.sh
+
+## Build
+./docker/build.sh YOUR_DOCKER_HUB_NAME VERSION

docker/backend/Dockerfile

@@ -0,0 +1,26 @@
+FROM node:12-buster-slim AS builder
+
+WORKDIR /build
+COPY . .
+RUN apt-get update
+RUN apt-get install -y build-essential python3 pkg-config
+RUN npm ci --production
+RUN npm i typescript
+RUN npm run build
+
+FROM node:12-buster-slim
+
+WORKDIR /backend
+
+COPY --from=builder /build/ .
+
+RUN chmod +x /backend/start.sh
+RUN chmod +x /backend/wait-for-it.sh
+
+RUN chown -R 1000:1000 /backend && chmod -R 755 /backend
+
+USER 1000
+
+EXPOSE 8999
+
+CMD ["/backend/start.sh"]

docker/backend/mempool-config.json

@@ -0,0 +1,38 @@
+{
+  "MEMPOOL": {
+    "NETWORK": "mainnet",
+    "BACKEND": "electrum",
+    "HTTP_PORT": __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__,
+    "SPAWN_CLUSTER_PROCS": 0,
+    "API_URL_PREFIX": "/api/v1/",
+    "POLL_RATE_MS": 2000,
+    "CACHE_DIR": "__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__"
+  },
+  "CORE_RPC": {
+    "HOST": "__BITCOIN_MAINNET_RPC_HOST__",
+    "PORT": __BITCOIN_MAINNET_RPC_PORT__,
+    "USERNAME": "__BITCOIN_MAINNET_RPC_USER__",
+    "PASSWORD": "__BITCOIN_MAINNET_RPC_PASS__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRS_MAINNET_HTTP_HOST__",
+    "PORT": __ELECTRS_MAINNET_HTTP_PORT__,
+    "TLS_ENABLED": false,
+    "TX_LOOKUPS": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "http://127.0.0.1:3000"
+  },
+  "DATABASE": {
+    "ENABLED": true,
+    "HOST": "__MYSQL_HOST__",
+    "PORT": __MYSQL_PORT__,
+    "DATABASE": "mempool",
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+  "STATISTICS": {
+    "ENABLED": true,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 150
+  }
+}

docker/backend/start.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+#MEMPOOL
+__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
+__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__=${CACHE_DIR:=./cache}
+# BITCOIN
+__BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST:=127.0.0.1}
+__BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT:=8332}
+__BITCOIN_MAINNET_RPC_USER__=${RPC_USER:=mempool}
+__BITCOIN_MAINNET_RPC_PASS__=${RPC_PASS:=mempool}
+# ELECTRUM
+__ELECTRS_MAINNET_HTTP_HOST__=${ELECTRS_HOST:=127.0.0.1}
+__ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002}
+# MYSQL
+__MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1}
+__MYSQL_PORT__=${MYSQL_PORT:=3306}
+
+mkdir -p "${__MEMPOOL_BACKEND_CACHE_DIR__}"
+
+sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json
+sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json
+sed -i "s/__BITCOIN_MAINNET_RPC_USER__/${__BITCOIN_MAINNET_RPC_USER__}/g" mempool-config.json
+sed -i "s/__BITCOIN_MAINNET_RPC_PASS__/${__BITCOIN_MAINNET_RPC_PASS__}/g" mempool-config.json
+sed -i "s/__ELECTRS_MAINNET_HTTP_HOST__/${__ELECTRS_MAINNET_HTTP_HOST__}/g" mempool-config.json
+sed -i "s/__ELECTRS_MAINNET_HTTP_PORT__/${__ELECTRS_MAINNET_HTTP_PORT__}/g" mempool-config.json
+sed -i "s/__MYSQL_HOST__/${__MYSQL_HOST__}/g" mempool-config.json
+sed -i "s/__MYSQL_PORT__/${__MYSQL_PORT__}/g" mempool-config.json
+sed -i "s!__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__!${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}!g" mempool-config.json
+sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" mempool-config.json
+
+node /backend/dist/index.js

docker/backend/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

docker/build.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+cd ./backend
+docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-backend:$2 --push .
+
+cd ../frontend
+docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-frontend:$2 --push .
+
+cd ../mysql
+docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-db:$2 --push .

docker/frontend/Dockerfile

@@ -0,0 +1,35 @@
+FROM node:12-buster-slim AS builder
+
+WORKDIR /build
+COPY . .
+RUN apt-get update
+RUN apt-get install -y build-essential rsync
+RUN npm i
+RUN npm run build
+
+FROM nginx:1.17.8-alpine
+
+WORKDIR /patch
+
+COPY --from=builder /build/entrypoint.sh .
+COPY --from=builder /build/wait-for .
+COPY --from=builder /build/dist/mempool /var/www/mempool
+COPY --from=builder /build/nginx.conf /etc/nginx/
+COPY --from=builder /build/nginx-mempool.conf /etc/nginx/conf.d/
+
+RUN chmod +x /patch/entrypoint.sh
+RUN chmod +x /patch/wait-for
+
+RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \
+        chown -R 1000:1000 /var/cache/nginx && \
+        chown -R 1000:1000 /var/log/nginx && \
+        chown -R 1000:1000 /etc/nginx/conf.d
+RUN touch /var/run/nginx.pid && \
+        chown -R 1000:1000 /var/run/nginx.pid
+
+USER 1000
+
+EXPOSE 8080
+
+ENTRYPOINT ["/patch/entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

docker/frontend/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1}
+
+sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf
+
+exec "$@"

docker/frontend/wait-for

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+TIMEOUT=15
+QUIET=0
+
+echoerr() {
+  if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi
+}
+
+usage() {
+  exitcode="$1"
+  cat << USAGE >&2
+Usage:
+  $cmdname host:port [-t timeout] [-- command args]
+  -q | --quiet                        Do not output any status messages
+  -t TIMEOUT | --timeout=timeout      Timeout in seconds, zero for no timeout
+  -- COMMAND ARGS                     Execute command with args after the test finishes
+USAGE
+  exit "$exitcode"
+}
+
+wait_for() {
+ if ! command -v nc >/dev/null; then
+    echoerr 'nc command is missing!'
+    exit 1
+  fi
+
+  for i in `seq $TIMEOUT` ; do
+    nc -z "$HOST" "$PORT" > /dev/null 2>&1
+    
+    result=$?
+    if [ $result -eq 0 ] ; then
+      if [ $# -gt 0 ] ; then
+        exec "$@"
+      fi
+      exit 0
+    fi
+    sleep 1
+  done
+  echo "Operation timed out" >&2
+  exit 1
+}
+
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    *:* )
+    HOST=$(printf "%s\n" "$1"| cut -d : -f 1)
+    PORT=$(printf "%s\n" "$1"| cut -d : -f 2)
+    shift 1
+    ;;
+    -q | --quiet)
+    QUIET=1
+    shift 1
+    ;;
+    -t)
+    TIMEOUT="$2"
+    if [ "$TIMEOUT" = "" ]; then break; fi
+    shift 2
+    ;;
+    --timeout=*)
+    TIMEOUT="${1#*=}"
+    shift 1
+    ;;
+    --)
+    shift
+    break
+    ;;
+    --help)
+    usage 0
+    ;;
+    *)
+    echoerr "Unknown argument: $1"
+    usage 1
+    ;;
+  esac
+done
+
+if [ "$HOST" = "" -o "$PORT" = "" ]; then
+  echoerr "Error: you need to provide a host and port to test."
+  usage 2
+fi
+
+wait_for "$@"

docker/init.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+#backend
+gitMaster="\.\.\/\.git\/refs\/heads\/master"
+cp .git/refs/heads/master ./backend
+cp ./docker/backend/* ./backend/
+sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
+
+#frontend
+localhostIP="127.0.0.1"
+cp ./docker/frontend/* ./frontend
+cp ./nginx.conf ./frontend/
+cp ./nginx-mempool.conf ./frontend/
+sed -i "s/${localhostIP}:/0.0.0.0:80/g" ./frontend/nginx.conf
+sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
+sed -i "s/user nobody;//g" ./frontend/nginx.conf
+sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
+sed -i "s/${localhostIP}:/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:/g" ./frontend/nginx-mempool.conf
+
+#db
+cp -fr docker/mysql .
+cp ./mariadb-structure.sql ./mysql/setup.sql

docker/mysql/Dockerfile

@@ -0,0 +1,7 @@
+FROM mariadb:10.5.8
+
+WORKDIR /db
+COPY . .
+
+COPY setup.sql /docker-entrypoint-initdb.d/
+COPY mysql-mempool.cnf /etc/mysql/conf.d/

docker/mysql/mysql-mempool.cnf

@@ -0,0 +1,2 @@
+[mysqld]
+sql_mode="NO_AUTO_VALUE_ON_ZERO"