BTCpay and NBXplorer update, use postgres for new installs (#3641)

* use postgres for new btcpay installs
keep sqlite if the database is present
clean and format the script with shellcheck
* reinstall the service on update and check postgres
* run userdel -rf btcpay when off
* btcpay update to v1.7.5
* add systemctl daemon-reload when changing service
* exit on failed builds
* Restart=always
* nbxplorer update to v2.3.58
* prevent the git error 'detected dubious ownership in repository'

CHANGES.md

@@ -20,7 +20,7 @@
 - Update: RTL v0.13.0 with update option [details](https://github.com/Ride-The-Lightning/RTL/releases/tag/v0.12.3)
 - Update: Thunderhub v0.13.16 with balance sharing disabled [details](https://github.com/apotdevin/thunderhub/releases/tag/v0.13.16)
 - Update: LNbits 0.9.6 [details](https://github.com/lnbits/lnbits-legend/releases/tag/0.9.6)
-- Update: BTCPayServer 1.7.2 [details](https://github.com/btcpayserver/btcpayserver/releases/tag/v1.7.2)
+- Update: BTCPayServer 1.7.5 (using postgres for new installs) [details](https://github.com/btcpayserver/btcpayserver/releases/tag/v1.7.5)
 - Update: ItchySats 0.7.0 [details](https://github.com/itchysats/itchysats/releases/tag/0.7.0)
 - Update: Channel Tools (chantools) v0.10.5 [details](https://github.com/guggero/chantools/releases/tag/v0.10.5)
 - Update: JoinMarket v0.9.8 [details](https://github.com/JoinMarket-Org/joinmarket-clientserver/releases/tag/v0.9.8)

home.admin/config.scripts/bonus.btcpayserver.sh

@@ -3,9 +3,9 @@
 # Based on: https://gist.github.com/normandmickey/3f10fc077d15345fb469034e3697d0d0
 
 # https://github.com/dgarage/NBXplorer/tags
-NBXplorerVersion="v2.3.49"
+NBXplorerVersion="v2.3.58"
 # https://github.com/btcpayserver/btcpayserver/releases
-BTCPayVersion="v1.7.2"
+BTCPayVersion="v1.7.5"
 
 # command info
 if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "-help" ]; then
@@ -25,27 +25,19 @@ source /mnt/hdd/raspiblitz.conf
 source /home/admin/raspiblitz.info
 source <(/home/admin/_cache.sh get state)
 
-function postgresConfig() {
+function NBXplorerConfig() {
-
+  # check the postgres database
-  echo "# Generate the database"
+  if sudo -u postgres psql -c '\l' | grep nbxplorermainnet; then
+    echo "# nbxplorermainnet database already exists"
+  else
+    echo "# Generate the database for nbxplorer"
     sudo -u postgres psql -c "create database nbxplorermainnet;"
     sudo -u postgres psql -c "create user nbxplorer with encrypted password 'raspiblitz';"
-  # change to ${newPassword} or use Passfile=
-  # sudo -u postgres psql -c "alter user btcpay with encrypted password '${newPassword}';"
-  # sudo -u btcpay sed -i "s/Password=*/Password='${newPassword}';/g" /home/btcpay/.nbxplorer/Main/settings.config
-  # sudo -u btcpay sed -i "s/Password=*/Password='${newPassword}';/g" /home/btcpay/.btcpayserver/Main/settings.config
     sudo -u postgres psql -c "grant all privileges on database nbxplorermainnet to nbxplorer;"
+  fi
   echo "# List databases with: sudo -u postgres psql -c '\l'"
   sudo -u postgres psql -c '\l'
 
-  ## clean postgresql:
-  # sudo su - postgres -c "/usr/lib/postgresql/${PGVERSION}/bin/pg_ctl stop --wait --pgdata=/var/lib/postgresql/${PGVERSION}/main"
-  # sudo pg_dropcluster ${PGVERSION} main
-  # sudo apt remove postgresql -y --purge
-  # sudo apt remove  postgresql-${PGVERSION} -y --purge
-}
-
-function NBXplorerConfig() {
   # https://docs.btcpayserver.org/Deployment/ManualDeploymentExtended/#4-create-a-configuration-file
   echo
   echo "# Getting RPC credentials from the bitcoin.conf"
@@ -66,7 +58,24 @@ nomigrateevts=1
 function BtcPayConfig() {
   # set thumbprint
   FINGERPRINT=$(openssl x509 -noout -fingerprint -sha256 -inform pem -in /home/btcpay/.lnd/tls.cert | cut -d"=" -f2)
-  echo "# setting the LND TLS thumbprint for BTCPay"
+  if sudo ls /mnt/hdd/app-data/.btcpayserver/Main/sqllite.db 1>/dev/null 2>&1; then
+    echo "# sqlite database exists"
+    databaseOption="# keep using sqlite as /mnt/hdd/app-data/.btcpayserver/Main/sqllite.db exists (configured in the btcpayserver.service)"
+  else
+    echo "# sqlite database does not exist, using postgresql"
+    databaseOption="postgres=User ID=btcpay;Host=localhost;Port=5432;Application Name=btcpay;MaxPoolSize=20;Database=btcpaymainnet;Password='raspiblitz';"
+    if sudo -u postgres psql -c '\l' | grep btcpaymainnet; then
+      echo "# btcpaymainnet database already exists"
+    else
+      echo "# Generate the database for btcpay"
+      sudo -u postgres psql -c "create database btcpaymainnet;"
+      sudo -u postgres psql -c "create user btcpay with encrypted password 'raspiblitz';"
+      sudo -u postgres psql -c "grant all privileges on database btcpaymainnet to btcpay;"
+    fi
+    echo "# List databases with: sudo -u postgres psql -c '\l'"
+    sudo -u postgres psql -c '\l'
+  fi
+  echo "# Regenerate the btcpayserver settings (includes the LND TLS thumbprint)"
   # https://docs.btcpayserver.org/Deployment/ManualDeploymentExtended/#3-create-a-configuration-file
   echo "
 ### Global settings ###
@@ -82,14 +91,50 @@ BTC.explorer.url=http://127.0.0.1:24444/
 BTC.lightning=type=lnd-rest;server=https://127.0.0.1:8080/;macaroonfilepath=/home/btcpay/admin.macaroon;certthumbprint=$FINGERPRINT
 
 ### Database ###
-# keep sqlite for now as configured in the btcpayserver.service
+${databaseOption}
-# postgres=User ID=btcpay;Password=urpassword;Application Name=btcpayserver;Host=localhost;Port=5432;Database=btcpay;
 explorer.postgres=User ID=nbxplorer;Host=localhost;Port=5432;Application Name=nbxplorer;MaxPoolSize=20;Database=nbxplorermainnet;Password='raspiblitz';
 " | sudo -u btcpay tee /home/btcpay/.btcpayserver/Main/settings.config
-  #doesNetworkEntryAlreadyExists=$(sudo cat /home/btcpay/.btcpayserver/Main/settings.config | grep -c '^network=')
+}
-  #echo "# setting new LND TLS thumbprint for BTCPay"
+
-  #s="BTC.lightning=type=lnd-rest\;server=https\://127.0.0.1:8080/\;macaroonfilepath=/home/btcpay/admin.macaroon\;"
+function BtcPayService() {
-  #sudo -u btcpay sed -i "s|^${s}certthumbprint=.*|${s}certthumbprint=$FINGERPRINT|g" /home/btcpay/.btcpayserver/Main/settings.config
+  if sudo ls /mnt/hdd/app-data/.btcpayserver/Main/sqllite.db 1>/dev/null 2>&1; then
+    echo "# sqlite database exists"
+    databaseOption=" -- --sqlitefile=sqllite.db"
+  else
+    echo "# sqlite database does not exist, using postgresql"
+    databaseOption=""
+  fi
+  # see the configuration options with:
+  # sudo -u btcpay /home/btcpay/dotnet/dotnet run --no-launch-profile --no-build -c Release --project "/home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj" -- -h
+  # run manually to debug:
+  # sudo -u btcpay /home/btcpay/dotnet/dotnet run --no-launch-profile --no-build -c Release --project "/home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj" -- --sqlitefile=sqllite.db
+  echo "# create the btcpayserver.service"
+  echo "
+[Unit]
+Description=BtcPayServer daemon
+Requires=nbxplorer.service
+After=nbxplorer.service
+
+[Service]
+ExecStart=/home/btcpay/dotnet/dotnet run --no-launch-profile --no-build \
+ -c Release --project \"/home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj\" ${databaseOption}
+User=btcpay
+Group=btcpay
+Type=simple
+PIDFile=/run/btcpayserver/btcpayserver.pid
+Restart=always
+RestartSec=10
+
+# Hardening measures
+PrivateTmp=true
+ProtectSystem=full
+NoNewPrivileges=true
+PrivateDevices=true
+
+[Install]
+WantedBy=multi-user.target
+" | sudo tee /etc/systemd/system/btcpayserver.service
+  sudo systemctl daemon-reload
 }
 
 if [ "$1" = "status" ]; then
@@ -341,9 +386,9 @@ if [ "$1" = "install" ]; then
   echo "DOTNET_CLI_TELEMETRY_OPTOUT=1" | sudo tee -a /etc/environment
 
   # NBXplorer
-  echo "# Install NBXplorer"
+  echo "# Install NBXplorer $NBXplorerVersion"
   cd /home/btcpay || exit 1
-  echo "# Download the NBXplorer source code ..."
+  echo "# Download the NBXplorer source code $NBXplorerVersion"
   sudo -u btcpay git clone https://github.com/dgarage/NBXplorer.git 2>/dev/null
   cd NBXplorer || exit 1
   sudo -u btcpay git reset --hard $NBXplorerVersion
@@ -352,14 +397,17 @@ if [ "$1" = "install" ]; then
   PGPpubkeyLink="https://keybase.io/nicolasdorier/pgp_keys.asc"
   PGPpubkeyFingerprint="AB4CFA9895ACA0DBE27F6B346618763EF09186FE"
   sudo -u btcpay /home/admin/config.scripts/blitz.git-verify.sh "${PGPsigner}" "${PGPpubkeyLink}" "${PGPpubkeyFingerprint}" || exit 1
-  echo "# Build NBXplorer ..."
+  echo "# Build NBXplorer $NBXplorerVersion"
   # from the build.sh with path
-  sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release NBXplorer/NBXplorer.csproj
+  sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release NBXplorer/NBXplorer.csproj || (
+    echo "# Build failed"
+    exit 1
+  )
 
   # BTCPayServer
   echo "# Install BTCPayServer"
   cd /home/btcpay || exit 1
-  echo "# Download the BTCPayServer source code ..."
+  echo "# Download the BTCPayServer source code $BTCPayVersion"
   sudo -u btcpay git clone https://github.com/btcpayserver/btcpayserver.git 2>/dev/null
   cd btcpayserver || exit 1
   sudo -u btcpay git reset --hard $BTCPayVersion
@@ -372,9 +420,13 @@ if [ "$1" = "install" ]; then
   #PGPpubkeyFingerprint="8E5530D9D1C93097"
   sudo -u btcpay /home/admin/config.scripts/blitz.git-verify.sh "${PGPsigner}" "${PGPpubkeyLink}" "${PGPpubkeyFingerprint}" || exit 1
 
-  echo "# Build BTCPayServer ..."
+  echo "# Build BTCPayServer $BTCPayVersion"
   # from the build.sh with path
-  sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release /home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj
+  sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release \
+    /home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj || (
+    echo "# Build failed"
+    exit 1
+  )
 
   exit 0
 fi
@@ -522,8 +574,7 @@ WantedBy=multi-user.target
     echo "# Starting nbxplorer"
     sudo systemctl start nbxplorer
     echo "# Checking for nbxplorer config"
-    while [ ! -f "/home/btcpay/.nbxplorer/Main/settings.config" ]
+    while [ ! -f "/home/btcpay/.nbxplorer/Main/settings.config" ]; do
-      do
       echo "# Waiting for nbxplorer to start - CTRL+C to abort"
       sleep 10
       hasFailed=$(sudo systemctl status nbxplorer | grep -c "Active: failed")
@@ -536,8 +587,6 @@ WantedBy=multi-user.target
     echo "# Because the system is not 'ready' the service 'nbxplorer' will not be started at this point .. its enabled and will start on next reboot"
   fi
 
-  postgresConfig
-
   NBXplorerConfig
 
   # whitelist localhost in bitcoind
@@ -554,38 +603,11 @@ WantedBy=multi-user.target
     sudo systemctl restart nbxplorer
   fi
 
-  # see the configuration options with:
+  BtcPayConfig
-  # sudo -u btcpay /home/btcpay/dotnet/dotnet run --no-launch-profile --no-build -c Release -p "/home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj" -- -h
-  # run manually to debug:
-  # sudo -u btcpay /home/btcpay/dotnet/dotnet run --no-launch-profile --no-build -c Release --project "/home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj" -- --sqlitefile=sqllite.db
-  echo "# create the btcpayserver.service"
-  echo "
-[Unit]
-Description=BtcPayServer daemon
-Requires=nbxplorer.service
-After=nbxplorer.service
 
-[Service]
+  BtcPayService
-ExecStart=/home/btcpay/dotnet/dotnet run --no-launch-profile --no-build \
- -c Release --project \"/home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj\" \
- -- --sqlitefile=sqllite.db
-User=btcpay
-Group=btcpay
-Type=simple
-PIDFile=/run/btcpayserver/btcpayserver.pid
-Restart=on-failure
 
-# Hardening measures
-PrivateTmp=true
-ProtectSystem=full
-NoNewPrivileges=true
-PrivateDevices=true
-
-[Install]
-WantedBy=multi-user.target
-" | sudo tee /etc/systemd/system/btcpayserver.service
   sudo systemctl enable btcpayserver
-
   if [ "${state}" == "ready" ]; then
     echo "# Starting btcpayserver"
     sudo systemctl start btcpayserver
@@ -600,7 +622,7 @@ WantedBy=multi-user.target
       fi
     done
   else
-    echo "# Because the system is not 'ready' the service 'btcpayserver' will not be started at this point .. its enabled and will start on next reboot"
+    echo "# Because the system is not 'ready' the service 'btcpayserver' will not be started at this point .. it is enabled and will start on next reboot"
   fi
 
   sudo -u btcpay mkdir -p /home/btcpay/.btcpayserver/Main/
@@ -684,7 +706,10 @@ if [ "$1" = "0" ] || [ "$1" = "off" ]; then
   else
     echo "# keeping data"
   fi
-  echo "# OK BTCPayServer deactivaed."
+  echo "# OK BTCPayServer deactivated."
+
+  echo "# delete the btcpay user home directory"
+  sudo userdel -rf btcpay 2>/dev/null
 
   # needed for API/WebUI as signal that install ran thru
   echo "result='OK'"
@@ -693,6 +718,11 @@ if [ "$1" = "0" ] || [ "$1" = "off" ]; then
 fi
 
 if [ "$1" = "update" ]; then
+
+  # prevent the git error 'detected dubious ownership in repository'
+  git config --global --add safe.directory /home/btcpay/NBXplorer
+  git config --global --add safe.directory /home/btcpay/btcpayserver
+
   echo "# Update NBXplorer"
   cd /home/btcpay || exit 1
   cd NBXplorer || exit 1
@@ -712,7 +742,7 @@ if [ "$1" = "update" ]; then
     TAG=$(git tag | sort -V | tail -1)
     echo "# Up-to-date on version $TAG"
   else
-    echo "# Pulling latest changes..."
+    echo "# Pulling the latest changes..."
     sudo -u btcpay git pull -p
     TAG=$(git tag | sort -V | tail -1)
     echo "# Reset to the latest release tag: $TAG"
@@ -720,12 +750,17 @@ if [ "$1" = "update" ]; then
     PGPsigner="nicolasdorier"
     PGPpubkeyLink="https://keybase.io/nicolasdorier/pgp_keys.asc"
     PGPpubkeyFingerprint="AB4CFA9895ACA0DBE27F6B346618763EF09186FE"
+
     sudo -u btcpay /home/admin/config.scripts/blitz.git-verify.sh \
       "${PGPsigner}" "${PGPpubkeyLink}" "${PGPpubkeyFingerprint}" || exit 1
-    echo "# Build NBXplorer ..."
+
+    echo "# Build NBXplorer $TAG"
     # from the build.sh with path
     sudo systemctl stop nbxplorer
-    sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release NBXplorer/NBXplorer.csproj
+    sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release NBXplorer/NBXplorer.csproj || (
+      echo "# Build failed"
+      exit 1
+    )
     # whitelist localhost in bitcoind
     if ! sudo grep -Eq "^whitelist=127.0.0.1" /mnt/hdd/bitcoin/bitcoin.conf; then
       echo "whitelist=127.0.0.1" | sudo tee -a /mnt/hdd/bitcoin/bitcoin.conf
@@ -733,9 +768,6 @@ if [ "$1" = "update" ]; then
       sudo systemctl restart bitcoind
     fi
 
-    # POSTGRES
-    postgresConfig
-
     NBXplorerConfig
 
     sudo systemctl start nbxplorer
@@ -747,6 +779,9 @@ if [ "$1" = "update" ]; then
 
   BtcPayConfig
 
+  # always update the btcpayserver.service
+  BtcPayService
+
   echo "# Update BTCPayServer"
   cd /home/btcpay || exit 1
   cd btcpayserver || exit 1
@@ -775,10 +810,13 @@ if [ "$1" = "update" ]; then
     # https://github.com/rootzoll/raspiblitz/issues/3025
     # sudo -u btcpay /home/admin/config.scripts/blitz.git-verify.sh \
     #  "${PGPsigner}" "${PGPpubkeyLink}" "${PGPpubkeyFingerprint}" || exit 1
-    echo "# Build BTCPayServer ..."
+    echo "# Build BTCPayServer $TAG"
     # from the build.sh with path
     sudo systemctl stop btcpayserver
-    sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release /home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj
+    sudo -u btcpay /home/btcpay/dotnet/dotnet build -c Release /home/btcpay/btcpayserver/BTCPayServer/BTCPayServer.csproj || (
+      echo "# Build failed"
+      exit 1
+    )
     sudo systemctl start btcpayserver
     echo "# Updated BTCPayServer to $TAG"
   fi

home.admin/config.scripts/bonus.postgresql.sh

@@ -51,7 +51,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
     fix_postgres=1
   fi
 
-  if [ fix_postgres = 1 ] || [ ! -d /mnt/hdd/app-data/postgresql ]; then
+  if [ ${fix_postgres} = 1 ] || [ ! -d /mnt/hdd/app-data/postgresql ]; then
     echo "# Move the PostgreSQL data to /mnt/hdd/app-data/postgresql"
     sudo systemctl stop postgresql 2>/dev/null
     sudo rsync -av $postgres_datadir /mnt/hdd/app-data
@@ -67,9 +67,8 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
     # wait for the postgres server to start
     count=0
     count_max=30
-    while ! nc -zv 127.0.0.1 5432 2>/dev/null;
+    while ! nc -zv 127.0.0.1 5432 2>/dev/null; do
-    do
+      count=$((count + 1))
-      count=`expr $count + 1`
       echo "sleep $count/$count_max"
       sleep 1
       if [ $count = $count_max ]; then
@@ -103,7 +102,7 @@ fi
 
 # backup
 backup_target="/mnt/hdd/app-data/backup/$db_name"
-backup_file="${db_name}_`date +%d`-`date +%m`-`date +%Y`_`date +%H`-`date +%M`_dump"
+backup_file="${db_name}_$(date +%d)-$(date +%m)-$(date +%Y)_$(date +%H)-$(date +%M)_dump"
 if [ ! -d $backup_target ]; then
   sudo mkdir -p $backup_target 1>&2
 fi