Improve VM install & add Bitcoin Testnet Option (#4579)

build_sdcard.sh

@@ -369,7 +369,7 @@ echo -e "\n*** SOFTWARE UPDATE ***"
 # sqlite3 -> database
 # fdisk -> create partitions
 # lsb-release -> needed to know which distro version we're running to add APT sources
-general_utils="policykit-1 htop git curl bash-completion vim jq dphys-swapfile bsdmainutils autossh telnet vnstat parted dosfstools fbi sysbench build-essential dialog bc python3-dialog unzip whois fdisk lsb-release smartmontools rsyslog resolvconf"
+general_utils="sudo policykit-1 htop git curl bash-completion vim jq dphys-swapfile bsdmainutils autossh telnet vnstat parted dosfstools fbi sysbench build-essential dialog bc python3-dialog unzip whois fdisk lsb-release smartmontools rsyslog resolvconf"
 # add btrfs-progs if not bookworm on aarch64
 [ "${architecture}" = "aarch64" ] && ! grep "12 (bookworm)" < /etc/os-release && general_utils="${general_utils} btrfs-progs"
 # python3-mako --> https://github.com/rootzoll/raspiblitz/issues/3441
@@ -664,18 +664,19 @@ echo -e "\n*** ADDING GROUPS FOR CREDENTIALS STORE ***"
 echo -e "\n*** SHELL SCRIPTS & ASSETS ***"
 # copy raspiblitz repo from github
 cd /home/admin/ || exit 1
-sudo -u admin git config --global user.name "${github_user}"
+sudo -u admin git config --global user.name "${github_user}" || exit 1
-sudo -u admin git config --global user.email "johndoe@example.com"
+sudo -u admin git config --global user.email "johndoe@example.com" || exit 1
+sudo -u admin git config --global http.postBuffer 524288000 || exit 1
 sudo -u admin rm -rf /home/admin/raspiblitz
-sudo -u admin git clone -b "${branch}" https://github.com/${github_user}/raspiblitz.git
+sudo -u admin git clone -b "${branch}" https://github.com/${github_user}/raspiblitz.git || exit 1
-sudo -u admin cp -r /home/admin/raspiblitz/home.admin/*.* /home/admin
+sudo -u admin cp -r /home/admin/raspiblitz/home.admin/*.* /home/admin || exit 1
-sudo -u admin cp /home/admin/raspiblitz/home.admin/.tmux.conf /home/admin
+sudo -u admin cp /home/admin/raspiblitz/home.admin/.tmux.conf /home/admin || exit 1
-sudo -u admin cp -r /home/admin/raspiblitz/home.admin/assets /home/admin/
+sudo -u admin cp -r /home/admin/raspiblitz/home.admin/assets /home/admin/ || exit 1
-sudo -u admin chmod +x *.sh
+sudo -u admin chmod +x *.sh || exit 1
-sudo -u admin cp -r /home/admin/raspiblitz/home.admin/config.scripts /home/admin/
+sudo -u admin cp -r /home/admin/raspiblitz/home.admin/config.scripts /home/admin/ || exit 1
-sudo -u admin chmod +x /home/admin/config.scripts/*.sh
+sudo -u admin chmod +x /home/admin/config.scripts/*.sh || exit 1
-sudo -u admin cp -r /home/admin/raspiblitz/home.admin/setup.scripts /home/admin/
+sudo -u admin cp -r /home/admin/raspiblitz/home.admin/setup.scripts /home/admin/ || exit 1
-sudo -u admin chmod +x /home/admin/setup.scripts/*.sh
+sudo -u admin chmod +x /home/admin/setup.scripts/*.sh || exit 1
 
 # install newest version of BlitzPy
 blitzpy_wheel=$(ls -tR /home/admin/raspiblitz/home.admin/BlitzPy/dist | grep -E "any.whl" | tail -n 1)

home.admin/00raspiblitz.sh

@@ -98,8 +98,7 @@ do
     network \
     chain \
     lightning \
-    internet_localip \
+    internet_localip
-    system_vm_vagrant \
   )
 
   # background.scan is not ready yet
@@ -281,25 +280,6 @@ MAINMENU > REPAIR > REPAIR-LND > RETRYSCB
 
     #echo "# DURING SETUP: Handle System State (${state})"
 
-    # when no HDD on Vagrant - just print info & exit (admin info & exit)
-    if [ "${state}" == "noHDD" ] && [ ${system_vm_vagrant} != "0" ]; then
-      echo "***********************************************************"
-      echo "VAGRANT INFO"
-      echo "***********************************************************"
-      echo "To connect a HDD data disk to your VagrantVM:"
-      echo "- shutdown VM with command: off"
-      echo "- open your VirtualBox GUI and select RaspiBlitzVM"
-      echo "- change the 'mass storage' settings"
-      echo "- add a second 'Primary Slave' drive to the already existing controller"
-      echo "- close VirtualBox GUI and run: vagrant up & vagrant ssh"
-      echo "***********************************************************"
-      echo "You can either create a new dynamic VDI with around 900GB or download"
-      echo "a VDI with a presynced blockchain to speed up setup. If you dont have 900GB"
-      echo "space on your laptop you can store the VDI file on an external drive."
-      echo "***********************************************************"
-      exit 1
-    fi
-
     # for all critical errors (admin info & exit)
     if [ "${state}" == "error" ] || [ "${state}" == "errorHDD" ]; then
       clear

home.admin/_background.scan.sh

@@ -105,25 +105,11 @@ fi
 echo "importing: _version.info"
 /home/admin/_cache.sh import /home/admin/_version.info
 
-# basic hardware info (will not change)
+# get hardware info
-source <(/home/admin/_cache.sh valid \
+source <(/home/admin/config.scripts/blitz.hardware.sh status)
-  system_board \
+/home/admin/_cache.sh set system_board "${board}"
-  system_ram_mb \
+/home/admin/_cache.sh set system_ram_mb "${ramMB}"
-  system_ram_gb \
+/home/admin/_cache.sh set system_ram_gb "${ramGB}"
-)
-if [ "${stillvalid}" == "0" ]; then
-  source <(/home/admin/config.scripts/blitz.hardware.sh status)
-  /home/admin/_cache.sh set system_board "${board}"
-  /home/admin/_cache.sh set system_ram_mb "${ramMB}"
-  /home/admin/_cache.sh set system_ram_gb "${ramGB}"
-fi
-
-# VM detect vagrant
-source <(/home/admin/_cache.sh valid system_vm_vagrant)
-if [ "${stillvalid}" == "0" ]; then
-  vagrant=$(df | grep -c "/vagrant")
-  /home/admin/_cache.sh set system_vm_vagrant "${vagrant}"
-fi
 
 # flag that init was done (will be checked on each loop)
 /home/admin/_cache.sh set system_init_time "$(date +%s)"
@@ -171,9 +157,7 @@ do
   fi
 
   # ram
-  ram=$(free -m | grep Mem | awk '{ print $2 }')
+  ram_avail=$(free -m | grep -E 'Mem|Speicher' | awk '{ print $7 }')
-  ram_avail=$(free -m | grep Mem | awk '{ print $7 }')
-  /home/admin/_cache.sh set system_ram_mb "${ram}"
   /home/admin/_cache.sh set system_ram_available_mb "${ram_avail}"
 
   # undervoltage

home.admin/_commands.sh

@@ -55,6 +55,11 @@ function blitzhelp() {
   echo "  cache        check on chache system state"
   echo "  github       jumping directly into the options to change branch/repo/pr"
   echo
+  echo "Development with VM:"
+  echo "  sync         sync all repos from shared folder"
+  echo "  sync code    sync only main raspiblitz repo from shared folder"
+  echo "  sync api     sync only blitz api repo from shared folder"
+  echo  
   echo "Power:"
   echo "  restart      restart the node"
   echo "  off          shutdown the node"
@@ -215,6 +220,13 @@ function patch() {
   echo
 }
 
+# command: sync
+# sync VM with shared folder
+function sync() {
+  sudo /home/admin/config.scripts/blitz.vm.sh sync ${1}
+  echo
+}
+
 # command: off
 function off() {
   echo "Command to power off your RaspiBlitz"

home.admin/config.scripts/bitcoin.monitor.sh

@@ -148,7 +148,12 @@ if [ "$2" = "info" ]; then
   if [ "$1" == "signet" ]; then
     subfolder="signet/"
   fi
-  btc_blocks_data_kb=$(sudo du -s /mnt/hdd/bitcoin/${subfolder}blocks | cut -f1)
+  btc_blocks_data_kb=$(sudo du -s /mnt/hdd/bitcoin/${subfolder}blocks 2>/dev/null | cut -f1)
+  if [ "${btc_blocks_data_kb}" == "" ]; then
+   btc_blocks_data_kb="0"
+  fi
+
+  # print data
 
   # parse data
   btc_blocks_headers=$(echo "${blockchaininfo}" | jq -r '.headers')
@@ -156,9 +161,14 @@ if [ "$2" = "info" ]; then
   btc_blocks_behind=$((${btc_blocks_headers} - ${btc_blocks_verified}))
   btc_sync_initialblockdownload=$(echo "${blockchaininfo}" | jq -r '.initialblockdownload' | grep -c 'true')
   btc_sync_progress=$(echo "${blockchaininfo}" | jq -r '.verificationprogress')
-  if (( $(awk 'BEGIN { print( '${btc_sync_progress}'<0.99995 ) }') )); then
+  if [[ "${btc_sync_progress}" == *"e-"* ]]; then
+    # is still very small - round up to 0.01%
+    btc_sync_percentage="0.01"
+  elif (( $(awk 'BEGIN { print( '${btc_sync_progress}'<0.99995 ) }') )); then
     # #3620 prevent displaying 100.00%, although incorrect because of rounding
-    btc_sync_percentage=$(awk 'BEGIN { printf( "%.2f%%", 100 * '${btc_sync_progress}') }')
+    btc_sync_percentage="${btc_sync_progress:2:2}.${btc_sync_progress:4:2}"
+    # remove trailing zero if present (just first one)
+    btc_sync_percentage="${btc_sync_percentage#0}"
   elif [ "${btc_blocks_headers}" != "" ] && [ "${btc_blocks_headers}" == "${btc_blocks_verified}" ]; then
     btc_sync_percentage="100.00"
   else

home.admin/config.scripts/bitcoin.testnet.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# command info
+if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "-help" ]; then
+  echo "# Switches on bitcoind mainnet behind the scenes to testnet."
+  echo "# !!! JUST USE FOR DEVELOPEMNT - NOT FOR PRODUCTION !!!"
+  echo "# "
+  echo "# bitcoin.testnet.sh [activate|revert]"
+  echo
+  exit 1
+fi
+
+# make sure user is root
+if [ $UID -ne 0 ]; then
+  echo "error='run this script with sudo'"
+  exit 1
+fi
+
+echo "# Running: bitcoin.testnet.sh $*"
+
+if [ "$1" == "activate" ]; then
+
+  # check if bitcoin testnet is already activated
+  testnetSet=$(cat /mnt/hdd/bitcoin/bitcoin.conf | grep -c "^testnet=1")
+  if [ $testnetSet -gt 0 ]; then
+   echo "error='testnet is already activated'"
+   exit 1
+  fi
+
+  echo "# SWITCHING TO TESTNET .."
+
+  # make changes to bitcoin.conf
+  sed -i 's|^testnet=0|testnet=1|' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's/^\(main.debuglogfile=.*\)/#\1/' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's|^test.debuglogfile=/mnt/hdd/bitcoin/testnet3/debug.log|test.debuglogfile=/mnt/hdd/bitcoin/debug.log|' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's/^\(main.rpcbind=.*\)/#\1/' /mnt/hdd/bitcoin/bitcoin.conf 
+  sed -i 's|^test.rpcbind=127.0.0.1:18332|test.rpcbind=127.0.0.1:8332|' /mnt/hdd/bitcoin/bitcoin.conf
+
+  # restart bitcoind service
+  systemctl restart bitcoind.service
+
+  echo "# OK bitcoind should now run testnet on mainnet ports"
+  echo "# If you want to save space you can delete old mainnet blockchain with:"
+  echo "# rm -rf /mnt/hdd/bitcoin/blocks"
+  echo "# rm -rf /mnt/hdd/bitcoin/chainstate"
+
+  exit 0
+fi
+
+if [ "$1" == "revert" ]; then
+
+  # check if bitcoin testnet is already activated
+  testnetSet=$(cat /mnt/hdd/bitcoin/bitcoin.conf | grep -c "^testnet=1")
+  if [ $testnetSet -eq 0 ]; then
+   echo "error='testnet is not activated'"
+   exit 1
+  fi
+
+  echo "# SWITCHING BACK TO MAINNET .."
+
+  # make changes to bitcoin.conf
+  sed -i 's|^testnet=1|testnet=0|' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's|^#main.debuglogfile=.*|main.debuglogfile=/mnt/hdd/bitcoin/debug.log|' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's|^test.debuglogfile=/mnt/hdd/bitcoin/debug.log|test.debuglogfile=/mnt/hdd/bitcoin/testnet3/debug.log|' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's|^#main.rpcbind=.*|main.rpcbind=127.0.0.1:8332|' /mnt/hdd/bitcoin/bitcoin.conf
+  sed -i 's|^test.rpcbind=127.0.0.1:8332|test.rpcbind=127.0.0.1:18332|' /mnt/hdd/bitcoin/bitcoin.conf
+
+  # restart bitcoind service
+  systemctl restart bitcoind.service
+
+  echo "# OK bitcoind should now run normal mainnet again"
+  echo "# If you want to save space you can delete old testnet blockchain with:"
+  echo "# rm -rf /mnt/hdd/bitcoin/testnet3/blocks"
+  echo "# rm -rf /mnt/hdd/bitcoin/testnet3/chainstate"
+
+  exit 0
+fi
+
+echo "error='unkown parameter'"
+exit 1

home.admin/config.scripts/blitz.debug.sh

@@ -464,7 +464,7 @@ sudo /home/admin/config.scripts/blitz.zram.sh status
 echo
 
 echo "*** HARDWARE TEST RESULTS ***"
-sudo vcgencmd get_throttled
+sudo vcgencmd get_throttled 2>/dev/null
 source <(/home/admin/_cache.sh get system_count_undervoltage)
 showImproveInfo=0
 if [ ${#system_count_undervoltage} -gt 0 ]; then

home.admin/config.scripts/blitz.hardware.sh

@@ -13,11 +13,31 @@ fi
 
 # detect known SBCs
 board=""
-isRaspberryPi4=$(cat /proc/device-tree/model | grep -c "Raspberry Pi 4")
+
+# detect RaspberryPi 3
+isRaspberryPi3=$(cat /proc/device-tree/model 2>/dev/null | grep -c "Raspberry Pi 3")
+if [ "${isRaspberryPi3}" == "1" ]; then
+    board="rp3"
+fi
+
+# detect RaspberryPi 4
+isRaspberryPi4=$(cat /proc/device-tree/model 2>/dev/null | grep -c "Raspberry Pi 4")
 if [ "${isRaspberryPi4}" == "1" ]; then
     board="rp4"
 fi
 
+# detect RaspberryPi 5
+isRaspberryPi5=$(cat /proc/device-tree/model 2>/dev/null | grep -c "Raspberry Pi 5")
+if [ "${isRaspberryPi5}" == "1" ]; then
+    board="rp5"
+fi
+
+# detect VM
+isVM=$(grep -c 'hypervisor' /proc/cpuinfo)
+if [ ${isVM} -gt 0 ]; then
+    board="vm"
+fi
+
 # get how many RAM (in MB)
 ramMB=$(awk '/MemTotal/ {printf( "%d\n", $2 / 1024 )}' /proc/meminfo)
 

home.admin/config.scripts/blitz.vm.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+
+# This is for developing on your RaspiBlitz VM
+
+# command info
+if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ] || [ "$1" = "-help" ]; then
+  echo "FOR DEVELOPMENT USE ONLY!"
+  echo "RaspiBlitzVM Sync with repos in /mnt/vm_shared_folder"
+  echo "blitz.vm.sh sync      -> syncs all available repos in shared folder"
+  echo "blitz.vm.sh sync code -> syncs only the raspiblitz repo"
+  echo "blitz.vm.sh sync api  -> syncs only the raspiblitz API repo"
+  echo ""
+  exit 1
+fi
+
+# check runnig as sudo
+if [ "$EUID" -ne 0 ]; then
+  echo "error='please run as root'"
+  exit 1
+fi
+
+# check if running in vm
+isVM=$(grep -c 'hypervisor' /proc/cpuinfo)
+if [ ${isVM} -eq 0 ]; then
+  echo "# This script is only for RaspiBlitz running in a VM"
+  echo "error='not a VM'"
+  exit 1
+fi
+
+# check if shared folder exists
+if [ ! -d "/mnt/vm_shared_folder" ]; then
+  echo "# Creating shared folder /mnt/vm_shared_folder"
+  mkdir /mnt/vm_shared_folder
+  chmod 777 /mnt/vm_shared_folder
+fi
+
+# check if shared folder is mounted
+isMounted=$(mount | grep '/mnt/vm_shared_folder')
+if [ ${#isMounted} -eq 0 ]; then
+  echo "# Mounting shared folder /mnt/vm_shared_folder"
+  mount -t 9p -o trans=virtio share /mnt/vm_shared_folder
+  if [ $? -eq 0 ]; then
+    echo "# OK - shared folder mounted"
+  else
+    echo "# make sure to activate shared folder in VM settings (VirtFS)"
+    echo "error='mount failed'"
+    exit 1
+  fi
+fi
+
+# RASPIBLITZ MAIN REPO
+if [ "$2" == "code" ] || [ "$2" == "" ]; then
+
+  echo
+  echo  "# ##### RASPIBLITZ REPO"
+
+  #check if contains a raspiblitz MAIN repo
+  containsRaspiBlitzRepo=$(ls /mnt/vm_shared_folder | grep -wc 'raspiblitz')
+  if [ ${containsRaspiBlitzRepo} -eq 0 ]; then
+ 
+    echo "# /mnt/vm_shared_folder does not contain a raspiblitz repo"
+    echo "# make sure to share the directory that contains the raspiblitz repo - not the repo itself"
+    echo "# make sure its named 'raspiblitz' and not 'raspiblitz-main' or 'raspiblitz-v1.7'"
+ 
+    if [ "$2" != "" ]; then
+      echo "error='no raspiblitz main repo'"
+      exit 1
+    fi
+ 
+  else
+
+    cd /home/admin
+    echo "# COPYING from VM SHARED FOLDER to /home/admin/"
+    echo "# - basic admin files"
+    rm -f *.sh
+    su - admin -c 'cp /mnt/vm_shared_folder/raspiblitz/home.admin/.tmux.conf /home/admin'
+    su - admin -c 'cp /mnt/vm_shared_folder/raspiblitz/home.admin/*.* /home/admin 2>/dev/null'
+    su - admin -c 'chmod 755 *.sh'
+    echo "# - asset directory"
+    rm -rf assets
+    su - admin -c 'cp -R /mnt/vm_shared_folder/raspiblitz/home.admin/assets /home/admin/assets'
+    echo "# - config.scripts directory"
+    rm -rf /home/admin/config.scripts
+    su - admin -c 'cp -R /mnt/vm_shared_folder/raspiblitz/home.admin/config.scripts /home/admin/config.scripts'
+    su - admin -c 'chmod 755 /home/admin/config.scripts/*.sh'
+    su - admin -c 'chmod 755 /home/admin/config.scripts/*.py'
+    echo "# - setup.scripts directory"
+    rm -rf /home/admin/setup.scripts
+    su - admin -c 'cp -R /mnt/vm_shared_folder/raspiblitz/home.admin/setup.scripts /home/admin/setup.scripts'
+    su - admin -c 'chmod 755 /home/admin/setup.scripts/*.sh'
+    su - admin -c 'chmod 755 /home/admin/config.scripts/*.py'
+    echo "# ******************************************"
+
+    if [ "$2" != "" ]; then
+      exit 0
+    fi
+
+  fi
+fi  
+
+# RASPIBLITZ API REPO
+if [ "$2" == "api" ] || [ "$2" == "" ]; then
+
+  echo
+  echo  "# ##### RASPIBLITZ API REPO"
+
+  # check if blitzapi service is enabled
+  systemctl is-enabled blitzapi 2>/dev/null
+  notInstalled=$?
+
+  #check if contains a raspiblitz API repo
+  containsApiRepo=$(ls /mnt/vm_shared_folder | grep -wc 'blitz_api')
+  if [ ${containsApiRepo} -eq 0 ]; then
+ 
+    echo "# /mnt/vm_shared_folder does not contain a api repo"
+    echo "# make sure to share the directory that contains the api repo - not the repo itself"
+    echo "# make sure its named 'blitz_api'"
+ 
+    if [ "$2" != "" ]; then
+      echo "error='no raspiblitz api repo'"
+      exit 1
+    fi
+
+  elif [ ${notInstalled} -gt 0 ]; then
+  
+    echo "# blitzapi service is not installed or enabled - skipping"
+    if [ "$2" != "" ]; then
+      echo "error='blitzapi service not enabled'"
+      exit 1
+    fi
+ 
+  else
+
+  
+    echo "# TODO: Not implemented yet - use /script/updateBlitzAPI.sh instead to sync from host to VM"
+
+    #echo "# Stopping blitzapi service"
+    #systemctl stop blitzapi
+    #echo "# COPYING from VM SHARED FOLDER to /home/blitzapi/"
+    #rm -rf /home/blitzapi/blitz_api
+    #cp -R /mnt/vm_shared_folder/blitz_api /home/blitzapi
+    #chown -R blitzapi:blitzapi /home/blitzapi/blitz_api
+    #cd /home/blitzapi/blitz_api || exit 1
+    #su - blitzapi -c './venv/bin/pip install -r requirements.txt'
+    #echo "# Starting blitzapi service"
+    #systemctl start blitzapi
+
+    if [ "$2" != "" ]; then
+      exit 0
+    fi
+
+  fi
+
+fi
+
+if [ "$1" == "sync" ]; then
+  exit 0
+fi
+
+# in case of unknown command
+echo "error='unkown command'"
+exit 1

home.admin/setup.scripts/controlFinalDialog.sh

@@ -44,12 +44,17 @@ if [ "${btc_default_blocks_data_kb}" != "" ] && [ ${btc_default_blocks_data_kb}
   OPTIONS=()
   OPTIONS+=(SELFSYNC "Run full self sync/validation (takes long)")
   OPTIONS+=(COPY "Copy from Computer/RaspiBlitz over LAN (3-10h)")
+  OPTIONS+=(TESTNET "Sync smaller Testnet (ONLY DEVELOPER)")
   CHOICESUB=$(dialog --backtitle "RaspiBlitz" --clear --title " Blockchain Sync/Validation " --menu "\nYour Blockchain is not fully synced yet.\nThe full validation might take multiple days to finish.\n\nHow do you want to proceed:" 13 66 7 "${OPTIONS[@]}" 2>&1 >/dev/tty)
 
   if [ "${CHOICESUB}" == "COPY" ]; then
     /home/admin/config.scripts/blitz.copychain.sh target
   fi
 
+  if [ "${CHOICESUB}" == "TESTNET" ]; then
+    sudo /home/admin/config.scripts/bitcoin.testnet.sh activate
+  fi
+
 fi
 
 ############################################

home.admin/setup.scripts/eventBlockchainSync.sh

@@ -28,7 +28,7 @@ actionString="Please wait - this can take some time"
 
 # formatting BLOCKCHAIN SYNC PROGRESS
 if [ "${btc_default_ready}" == "0" ] || [ "${btc_default_peers}" == "" ]; then
-    if [ ${system_count_start_blockchain} -gt 1 ]; then
+    if [ "${system_count_start_blockchain}" != "" ] && [ ${system_count_start_blockchain} -gt 1 ]; then
         syncProgress="${system_count_start_blockchain} restarts"
     else
         syncProgress="waiting for start"