2018-12-11 13:15:15 +01:00
#!/bin/bash
# This script runs on after start in background
# as a service and gets restarted on failure
# it runs ALMOST every seconds
# INFOFILE - state data from bootstrap
infoFile = "/home/admin/raspiblitz.info"
# CONFIGFILE - configuration of RaspiBlitz
configFile = "/mnt/hdd/raspiblitz.conf"
2018-12-23 14:20:13 +01:00
# LOGS see: sudo journalctl -f -u background
2018-12-23 14:31:58 +01:00
echo "_background.sh STARTED"
2024-03-15 11:28:07 +01:00
echo "INFO: _background.sh loop started - sudo journalctl -f -u background" >> /home/admin/raspiblitz.log
2018-12-23 14:31:58 +01:00
2020-01-25 23:28:33 +01:00
# global vars
blitzTUIHeartBeatLine = ""
2021-12-14 23:34:35 +01:00
/home/admin/_cache.sh set blitzTUIRestarts "0"
2020-01-25 23:28:33 +01:00
2024-04-08 11:48:12 +02:00
# determine correct raspberrypi boot drive path (that easy to access when sd card is insert into laptop)
raspi_bootdir = ""
if [ -d /boot/firmware ] ; then
raspi_bootdir = "/boot/firmware"
elif [ -d /boot ] ; then
raspi_bootdir = "/boot"
fi
echo " # raspi_bootdir( ${ raspi_bootdir } ) "
2018-12-11 13:15:15 +01:00
counter = 0
while [ 1 ]
do
###############################
# Prepare this loop
###############################
# count up
counter = $(( $counter + 1 ))
2021-08-04 00:18:30 +02:00
# limit counter to max seconds per week:
# 604800 = 60sec * 60min * 24hours * 7days
if [ ${ counter } -gt 604800 ] ; then
counter = 0
echo "counter zero reset"
fi
2018-12-22 16:44:15 +01:00
# gather the uptime seconds
upSeconds = $( cat /proc/uptime | grep -o '^[0-9]\+' )
2021-12-14 23:34:35 +01:00
# source info & config file fresh on every loop
2021-08-04 00:18:30 +02:00
source ${ infoFile } 2>/dev/null
2021-12-14 23:34:35 +01:00
source ${ configFile } 2>/dev/null
2022-12-07 19:39:28 +00:00
2021-08-04 00:18:30 +02:00
####################################################
# SKIP BACKGROUND TASK LOOP ON CERTAIN SYSTEM STATES
# https://github.com/rootzoll/raspiblitz/issues/160
####################################################
if [ " ${ state } " = = "" ] || [ " ${ state } " = = "copysource" ] || [ " ${ state } " = = "copytarget" ] ; then
echo " skipping background loop ( ${ counter } ) - state( ${ state } ) "
sleep 1
continue
fi
2022-05-05 10:59:50 +02:00
####################################################
# Trigger Final Setup
####################################################
if [ " ${ setupPhase } " != "done" ] && [ " ${ state } " = = "donefinal" ] ; then
echo "trigger final setup tasks & reboot ..."
/home/admin/_provision.xfinal.sh
sleep 120
exit
fi
2021-08-04 00:18:30 +02:00
####################################################
# SKIP REST OF THE TASKS IF STILL IN SETUP PHASE
####################################################
if [ " ${ setupPhase } " != "done" ] ; then
echo " skipping rest of tasks because still in setupPhase( ${ setupPhase } ) "
sleep 1
2020-10-01 17:30:17 +02:00
continue
fi
2022-12-15 16:05:31 +01:00
####################################################
# MONITOR LOG SIZES
# https://github.com/rootzoll/raspiblitz/issues/2659
####################################################
2022-12-15 17:58:41 +01:00
# once a day
2022-12-15 16:05:31 +01:00
recheckLogs = $(( ( $counter % 86400 ) + 2 ))
if [ ${ recheckLogs } -eq 1 ] ; then
echo "*** MONITOR LOG SIZES ***"
journalctl --vacuum-size= 100M
fi
2018-12-11 13:15:15 +01:00
####################################################
2021-09-14 12:30:22 +02:00
# RECHECK DHCP-SERVER
2018-12-11 13:15:15 +01:00
# https://github.com/rootzoll/raspiblitz/issues/160
####################################################
# every 5 minutes
2018-12-11 13:33:22 +01:00
recheckDHCP = $(( ( $counter % 300 ) + 1 ))
2018-12-11 13:35:00 +01:00
if [ ${ recheckDHCP } -eq 1 ] ; then
2018-12-11 13:15:15 +01:00
echo "*** RECHECK DHCP-SERVER ***"
# get the local network IP
2021-08-04 00:18:30 +02:00
localip = $( hostname -I | awk '{print $1}' )
2018-12-11 13:15:15 +01:00
echo " localip( ${ localip } ) "
2021-09-14 12:30:22 +02:00
# detect a missing DHCP config
2018-12-11 13:15:15 +01:00
if [ " ${ localip : 0 : 4 } " = "169." ] ; then
2024-03-15 11:28:07 +01:00
echo "Missing DHCP detected ..."
2018-12-11 13:15:15 +01:00
else
echo "DHCP OK"
fi
fi
####################################################
# RECHECK PUBLIC IP
2020-10-05 22:45:34 +02:00
#
# when public IP changes
# - restart bitcoind with new IP
# - restart LND with new IP (if autounlock is enabled)
# - restart BTCRPCexplorer if enabled in config or running)
2018-12-11 13:15:15 +01:00
####################################################
# every 15min - not too often
# because its a ping to external service
2022-06-30 16:09:54 +02:00
recheckPublicIP = $(( ( $counter % 1800 ) + 1 ))
2021-12-14 23:34:35 +01:00
2019-04-03 02:58:10 +01:00
# prevent when lndAddress is set
if [ ${# lndAddress } -gt 3 ] ; then
recheckPublicIP = 0
fi
2020-01-07 20:15:35 +00:00
# prevent also when runBehindTor is on
if [ " ${ runBehindTor } " = "1" ] || [ " ${ runBehindTor } " = "on" ] ; then
recheckPublicIP = 0
fi
2018-12-13 21:59:58 +01:00
updateDynDomain = 0
2018-12-11 13:35:00 +01:00
if [ ${ recheckPublicIP } -eq 1 ] ; then
2018-12-11 13:15:15 +01:00
echo "*** RECHECK PUBLIC IP ***"
# execute only after setup when config exists
2021-12-14 23:34:35 +01:00
publicIPChanged = $( /home/admin/config.scripts/internet.sh update-publicip | grep -c 'ip_changed=1' )
2020-09-28 21:40:41 +02:00
2020-09-24 21:01:16 +02:00
# check if changed
2020-09-28 21:40:41 +02:00
if [ ${ publicIPChanged } -gt 0 ] ; then
2019-01-21 21:40:48 +01:00
2020-10-05 22:45:34 +02:00
echo "*** change of public IP detected ***"
2021-09-14 12:30:22 +02:00
# store the old IP address
publicIP_Old = " ${ publicIP } "
2020-09-24 21:01:16 +02:00
# refresh data
source /mnt/hdd/raspiblitz.conf
2021-09-14 12:30:22 +02:00
# store the new IP address
publicIP_New = " ${ publicIP } "
# some log output
echo " old: ${ publicIP_Old } "
echo " new: ${ publicIP_New } "
2020-10-05 22:45:34 +02:00
# if we run on IPv6 only, the global IPv6 address at the current network device (e.g: eth0) is the public IP
if [ " ${ ipv6 } " = "on" ] ; then
2021-09-14 12:30:22 +02:00
# if the old or the new IPv6 address is "::1" something has gone wrong in "internet.sh update-publicip" => no need to restart services
if [ " ${ publicIP_Old } " != "::1" ] && [ " ${ publicIP_New } " != "::1" ] ; then
# restart bitcoind as the global IP is stored in the node configuration
# and we will get more connections if this matches our real IP address
# otherwise the bitcoin-node connections will slowly decline
echo "IPv6 only is enabled => restart bitcoind to pickup up new publicIP as local IP"
2021-12-14 23:34:35 +01:00
systemctl stop bitcoind
2021-09-14 12:30:22 +02:00
sleep 3
2021-12-14 23:34:35 +01:00
systemctl start bitcoind
2021-09-14 12:30:22 +02:00
# if BTCRPCexplorer is currently running
# it needs to be restarted to pickup the new IP for its "Node Status Page"
# but this is only needed in IPv6 only mode
2021-12-14 23:34:35 +01:00
breIsRunning = $( systemctl status btc-rpc-explorer 2>/dev/null | grep -c 'active (running)' )
2021-09-14 12:30:22 +02:00
if [ ${ breIsRunning } -eq 1 ] ; then
echo "BTCRPCexplorer is running => restart BTCRPCexplorer to pickup up new publicIP for the bitcoin node"
2021-12-14 23:34:35 +01:00
systemctl stop btc-rpc-explorer
systemctl start btc-rpc-explorer
2021-09-14 12:30:22 +02:00
else
echo "new publicIP but no BTCRPCexplorer restart because not running"
fi
else
echo "IPv6 only is ON, but publicIP_Old OR publicIP_New is equal ::1 => no need to restart bitcoind nor BTCRPCexplorer"
fi
2020-10-05 22:45:34 +02:00
else
echo "IPv6 only is OFF => no need to restart bitcoind nor BTCRPCexplorer"
2021-09-14 12:30:22 +02:00
fi
2018-12-13 21:59:58 +01:00
2020-09-24 21:01:16 +02:00
# only restart LND if auto-unlock is activated
2021-09-14 12:30:22 +02:00
# AND neither the old nor the new IPv6 address is "::1"
2020-09-24 21:01:16 +02:00
if [ " ${ autoUnlock } " = "on" ] ; then
2021-09-14 12:30:22 +02:00
if [ " ${ publicIP_Old } " != "::1" ] && [ " ${ publicIP_New } " != "::1" ] ; then
echo "restart LND to pickup up new publicIP"
2021-12-14 23:34:35 +01:00
systemctl stop lnd
systemctl start lnd
2021-09-14 12:30:22 +02:00
else
echo "publicIP_Old OR publicIP_New is equal ::1 => no need to restart LND"
fi
2018-12-11 13:15:15 +01:00
else
2020-09-24 21:01:16 +02:00
echo "new publicIP but no LND restart because no auto-unlock"
2018-12-11 13:15:15 +01:00
fi
2021-08-27 03:59:21 -04:00
# trigger update if dynamic domain (if set)
2020-09-24 21:01:16 +02:00
updateDynDomain = 1
2018-12-11 13:15:15 +01:00
else
2020-09-24 21:01:16 +02:00
echo "public IP has not changed"
2018-12-11 13:15:15 +01:00
fi
fi
2021-12-14 23:34:35 +01:00
###############################
# UPDATE DYNAMIC DOMAIN
# like afraid.org
# ! experimental
###############################
# if not activated above, update every 6 hours
if [ ${ updateDynDomain } -eq 0 ] ; then
# dont +1 so that it gets executed on first loop
updateDynDomain = $(( $counter % 21600 ))
fi
if [ ${ updateDynDomain } -eq 1 ] ; then
echo "*** UPDATE DYNAMIC DOMAIN ***"
# check if update URL for dyn Domain is set
if [ ${# dynUpdateUrl } -gt 0 ] ; then
/home/admin/config.scripts/internet.dyndomain.sh update
else
echo " 'dynUpdateUrl' not set in ${ configFile } "
fi
fi
2020-12-21 00:27:20 +01:00
###############################
# Blockchain Sync Monitor
###############################
# check every 1min
recheckSync = $(( $counter % 60 ))
2021-12-14 23:34:35 +01:00
if [ ${ recheckSync } -eq 1 ] && [ " ${ chain } " = = "main" ] ; then
source <( /home/admin/config.scripts/bitcoin.monitor.sh mainnet network)
2021-12-17 22:28:35 +01:00
echo " Blockchain Sync Monitoring: peers= ${ btc_peers } "
2021-12-14 23:34:35 +01:00
if [ " ${ btc_peers } " = = "0" ] && [ " ${ btc_running } " = = "1" ] ; then
2020-12-21 00:27:20 +01:00
echo "Blockchain Sync Monitoring: ZERO PEERS DETECTED .. doing out-of-band kickstart"
2021-12-14 23:34:35 +01:00
/home/admin/config.scripts/bitcoin.monitor.sh mainnet peer-kickstart
2020-12-21 00:27:20 +01:00
fi
2022-12-07 19:26:10 +00:00
if [ " ${ i2pd } " = = "on" ] && [ " ${ btc_peers_i2p } " = = "0" ] && [ " ${ btc_running } " = = "1" ] ; then
echo "Blockchain Sync Monitoring: IP2TOR 0 peers .. doing out-of-band kickstart"
/home/admin/config.scripts/bitcoin.monitor.sh mainnet peer-kickstart i2p
fi
2020-12-21 00:27:20 +01:00
fi
2023-09-10 17:14:56 +02:00
###############################
# SYSTEM LOG FILE SIZES
###############################
# check every 15min
recheckSync = $(( $counter % 900 ))
if [ ${ recheckSync } -eq 1 ] ; then
echo "*** CHECK LOG FILE SIZES ***"
# check if log file is getting too big
logsMegaByte = $( sudo du -c -m /var/log | grep "total" | awk '{print $1;}' )
if [ ${ logsMegaByte } -gt 5000 ] ; then
echo "WARN # Logs /var/log in are bigger then 5GB .. starting repair"
debuginfo = $( ls -la /var/log/ 2>/dev/null)
# dont delete directories - can make services crash
sudo rm /var/log/*
2023-12-01 16:25:34 +01:00
sudo touch /var/log/auth.log
sudo chown root:adm /var/log/auth.log
2023-09-10 17:14:56 +02:00
sudo service rsyslog restart
/home/admin/config.scripts/blitz.error.sh _background.sh "log-delete" "REPAIR: /var/log/ >5GB" "Logs in /var/log in were bigger then 5GB and got emergency delete to prevent fillup." " ${ debuginfo } "
sleep 10
else
echo " OK - logs are at ${ logsMegaByte } MB - within safety limit "
fi
echo ""
fi
2022-01-25 12:07:11 +01:00
####################################################
# MONITOR Initial Syncing of Bitcoin & Lightning
# - turn off recovery mode
####################################################
recheckIBD = $(( ( $counter % 10 ) + 1 ))
if [ ${ recheckIBD } -eq 1 ] ; then
# loop thru mainet, testnet & signet
networks = ( "main" "test" "sig" )
for CHAIN in " ${ networks [@] } "
do
# gat values from cache
source <( /home/admin/_cache.sh meta btc_${ CHAIN } net_sync_initial_started)
flagBtcStarted = " ${ value } "
source <( /home/admin/_cache.sh meta btc_${ CHAIN } net_sync_initialblockdownload)
flagBtcActive = " ${ value } "
source <( /home/admin/_cache.sh meta btc_${ CHAIN } net_synced)
flagBtcSynced = " ${ value } "
source <( /home/admin/_cache.sh meta btc_${ CHAIN } net_online)
flagBtcOnline = " ${ value } "
source <( /home/admin/_cache.sh meta btc_${ CHAIN } net_sync_initial_done)
flagBtcDone = " ${ value } "
#echo "CHAIN(${CHAIN}) flagBtcStarted(${flagBtcStarted}) flagBtcActive(${flagBtcActive}) flagBtcSynced(${flagBtcSynced}) flagBtcOnline(${flagBtcOnline}) flagBtcDone(${flagBtcDone})"
# first check if flags need to be reset (manually delete of blockchain)
if [ " ${ flagBtcDone } " = = "1" ] && [ " ${ flagBtcActive } " = = "1" ] ; then
flagBtcDone = 0
2022-12-07 19:39:28 +00:00
/home/admin/config.scripts/blitz.conf.sh set btc_${ CHAIN } net_sync_initial_done ${ flagBtcDone } /home/admin/raspiblitz.info
2022-01-25 12:07:11 +01:00
echo " EVENT --> btc_ ${ CHAIN } net_sync_initial_done changed to ${ flagBtcDone } "
fi
# when started flag not set yet - but is now active --> set flag
if [ " ${ flagBtcStarted } " != "1" ] && [ " ${ flagBtcActive } " = = "1" ] ; then
flagBtcStarted = 1
/home/admin/_cache.sh set btc_${ CHAIN } net_sync_initial_started ${ flagBtcStarted }
echo " EVENT --> btc_ ${ CHAIN } net_sync_initial_started changed to ${ flagBtcStarted } "
fi
# when started done is set - but not not active anymore --> end of IDB event detected
if [ " ${ flagBtcDone } " = = "0" ] && [ " ${ flagBtcOnline } " = = "1" ] && [ " ${ flagBtcSynced } " = = "1" ] ; then
flagBtcDone = 1
2022-12-07 19:39:28 +00:00
/home/admin/config.scripts/blitz.conf.sh set btc_${ CHAIN } net_sync_initial_done ${ flagBtcDone } /home/admin/raspiblitz.info
2022-01-25 12:07:11 +01:00
echo " EVENT --> btc_ ${ CHAIN } net_sync_initial_done changed to ${ flagBtcDone } "
fi
# loop thru all second layers
sedondLayers = ( "lnd" "cl" )
for LN in " ${ sedondLayers [@] } "
do
source <( /home/admin/_cache.sh meta ln_${ LN } _${ CHAIN } net_sync_chain)
flagLnSyncChain = " ${ value } "
source <( /home/admin/_cache.sh meta ln_${ LN } _${ CHAIN } net_online)
flagLnOnline = " ${ value } "
source <( /home/admin/_cache.sh meta ln_${ LN } _${ CHAIN } net_recovery_mode)
flagLNRecoveryMode = " ${ value } "
source <( /home/admin/_cache.sh meta ln_${ LN } _${ CHAIN } net_recovery_done)
flagLNRecoveryDone = " ${ value } "
source <( /home/admin/_cache.sh meta ln_${ LN } _${ CHAIN } net_sync_initial_done)
flagLNSyncDone = " ${ value } "
#echo "LN(${LN}) flagLnSyncChain(${flagLnSyncChain}) flagLnOnline(${flagLnOnline}) flagLNRecoveryMode(${flagLNRecoveryMode}) flagLNRecoveryDone(${flagLNRecoveryDone}) flagLNSyncDone(${flagLNSyncDone})"
# first check if flags need to be reset (manually a rescan was triggered)
if [ " ${ flagLNSyncDone } " = = "1" ] && [ " ${ flagLNRecoveryMode } " = = "1" ] ; then
flagLNSyncDone = 0
/home/admin/config.scripts/blitz.conf.sh set ln_${ LN } _${ CHAIN } net_sync_initial_done ${ flagLNSyncDone } /home/admin/raspiblitz.info
echo " EVENT --> ln_ ${ LN } _ ${ CHAIN } net_sync_initial_done to ${ flagLNSyncDone } "
fi
# when flag initial sync not done yet - but all chains are in sync with network
if [ " ${ flagLNSyncDone } " = = "0" ] && [ " ${ flagBtcDone } " = = "1" ] && [ " ${ flagLnOnline } " = = "1" ] && [ " ${ flagLnSyncChain } " = = "1" ] ; then
# then only finished if no LNRecoveryMode or LNRecoveryDone
if [ " ${ flagLNRecoveryMode } " = = "0" ] || [ " ${ flagLNRecoveryDone } " = = "1" ] ; then
# write event
flagLNSyncDone = 1
/home/admin/config.scripts/blitz.conf.sh set ln_${ LN } _${ CHAIN } net_sync_initial_done ${ flagLNSyncDone } /home/admin/raspiblitz.info
echo " EVENT --> ln_ ${ LN } _ ${ CHAIN } net_sync_initial_done to ${ flagLNSyncDone } "
# LND if recovery mode was on - deactivate now
if [ " ${ LN } " = = "lnd" ] && [ " ${ flagLNRecoveryMode } " = = "1" ] ; then
/home/admin/_cache.sh set ln_lnd_mainnet_recovery_mode 0
/home/admin/config.scripts/lnd.backup.sh mainnet recoverymode off
fi
2023-05-15 21:15:55 +01:00
# CLN if recovery mode was on - deactivate now
if [ " ${ LN } " = = "cl" ] && [ " ${ flagLNRecoveryMode } " = = "1" ] ; then
/home/admin/_cache.sh set ln_cl_mainnet_recovery_mode 0
/home/admin/config.scripts/cl.backup.sh mainnet recoverymode off
fi
2022-01-25 12:07:11 +01:00
fi
fi
done
done
fi
####################################################
2022-12-07 19:39:28 +00:00
# Check for end of Initial Blockhain & Lightning Sync
2022-01-25 12:07:11 +01:00
# bitcoin mainnet only / special on dbcache size
####################################################
# check every 60secs
recheckIBD = $(( ( $counter % 60 ) + 1 ))
if [ ${ recheckIBD } -eq 1 ] ; then
# check if flag exists (gets created on setup)
# this flag signals that an initial blockchain sync/chatchup was happening
flagExists = $( ls /mnt/hdd/bitcoin/blocks/selfsync.flag 2>/dev/null | grep -c "selfsync.flag" )
if [ ${ flagExists } -eq 1 ] ; then
2022-12-07 19:39:28 +00:00
2022-01-25 12:07:11 +01:00
source <( /home/admin/_cache.sh get btc_default_sync_initialblockdownload)
if [ " ${ btc_default_sync_initialblockdownload } " = = "0" ] ; then
echo "CHECK FOR END OF IBD --> reduce RAM for next reboot"
# remove flag
rm /mnt/hdd/bitcoin/blocks/selfsync.flag
# set dbcache back to normal (to give room for other apps after reboot in the future)
kbSizeRAM = $( cat /proc/meminfo | grep "MemTotal" | sed 's/[^0-9]*//g' )
# RP4 4GB
if [ ${ kbSizeRAM } -gt 3500000 ] ; then
echo "Detected RAM >=4GB --> normalizing bitcoin.conf"
sed -i "s/^dbcache=.*/dbcache=512/g" /mnt/hdd/bitcoin/bitcoin.conf
# RP4 2GB
elif [ ${ kbSizeRAM } -gt 1500000 ] ; then
echo "Detected RAM >=2GB --> normalizing bitcoin.conf"
sed -i "s/^dbcache=.*/dbcache=256/g" /mnt/hdd/bitcoin/bitcoin.conf
#RP3/4 1GB
else
echo "Detected RAM <=1GB --> normalizing bitcoin.conf"
sed -i "s/^dbcache=.*/dbcache=128/g" /mnt/hdd/bitcoin/bitcoin.conf
fi
# relax sanning on sync progress (after 30 more secs)
/home/admin/_cache.sh focus btc_default_sync_progress 10 30
fi
fi
fi
2020-01-25 23:28:33 +01:00
###############################
# BlitzTUI Monitoring
###############################
2019-04-15 12:31:28 +01:00
2020-01-25 23:28:33 +01:00
# check every 30sec
recheckBlitzTUI = $(( $counter % 30 ))
2020-01-25 23:52:57 +01:00
if [ " ${ touchscreen } " = = "1" ] && [ ${ recheckBlitzTUI } -eq 1 ] ; then
2022-12-07 19:39:28 +00:00
2020-01-25 23:52:57 +01:00
echo "BlitzTUI Monitoring Check"
2020-10-01 17:30:17 +02:00
if [ -d "/var/cache/raspiblitz" ] ; then
2021-12-14 23:34:35 +01:00
latestHeartBeatLine = $( tail -n 300 /var/cache/raspiblitz/pi/blitz-tui.log | grep beat | tail -n 1)
2020-05-30 12:56:24 +01:00
else
2021-12-14 23:34:35 +01:00
latestHeartBeatLine = $( tail -n 300 /home/pi/blitz-tui.log | grep beat | tail -n 1)
2020-10-01 17:30:17 +02:00
fi
if [ ${# blitzTUIHeartBeatLine } -gt 0 ] ; then
#echo "blitzTUIHeartBeatLine(${blitzTUIHeartBeatLine})"
#echo "latestHeartBeatLine(${latestHeartBeatLine})"
if [ " ${ blitzTUIHeartBeatLine } " = = " ${ latestHeartBeatLine } " ] ; then
echo "FAIL - still no new heart beat .. restarting BlitzTUI"
2021-12-14 23:34:35 +01:00
source <( /home/admin/_cache.sh increment system_count_start_tui)
init 3 ; sleep 2 ; init 5
2020-01-25 23:28:33 +01:00
fi
2020-10-01 17:30:17 +02:00
else
echo "blitzTUIHeartBeatLine is empty - skipping check"
2020-01-25 23:28:33 +01:00
fi
2020-10-01 17:30:17 +02:00
blitzTUIHeartBeatLine = " ${ latestHeartBeatLine } "
2020-01-25 23:28:33 +01:00
fi
2021-09-14 12:30:22 +02:00
2019-04-15 14:59:53 +01:00
###############################
2022-11-22 19:27:25 +01:00
# SCB Monitoring (LND)
2019-04-15 14:59:53 +01:00
###############################
2021-12-14 23:34:35 +01:00
# check every 1min (only when lnd active)
recheckSCB = 0
if [ " ${ lightning } " = = "lnd" ] || [ " ${ lnd } " = = "on" ] ; then
recheckSCB = $(( $counter % 60 ))
fi
2019-04-15 14:59:53 +01:00
if [ ${ recheckSCB } -eq 1 ] ; then
2019-04-17 04:44:05 +01:00
#echo "SCB Monitoring ..."
2019-04-15 18:45:16 +01:00
source ${ configFile }
2019-04-15 14:59:53 +01:00
# check if channel.backup exists
2020-06-30 16:35:33 +02:00
scbPath = /mnt/hdd/lnd/data/chain/${ network } /${ chain } net/channel.backup
2021-12-14 23:34:35 +01:00
scbExists = $( ls $scbPath 2>/dev/null | grep -c 'channel.backup' )
2019-04-15 14:59:53 +01:00
if [ ${ scbExists } -eq 1 ] ; then
2021-12-14 23:34:35 +01:00
2020-06-30 16:35:33 +02:00
# timestamp backup filename
timestampedFileName = channel-$( date "+%Y%m%d-%H%M%S" ) .backup
2020-08-14 14:49:00 +02:00
localBackupDir = /home/admin/backups/scb
2020-06-30 16:35:33 +02:00
localBackupPath = ${ localBackupDir } /channel.backup
localTimestampedPath = ${ localBackupDir } /${ timestampedFileName }
2019-04-17 04:44:05 +01:00
#echo "Found Channel Backup File .. check if changed .."
2021-12-14 23:34:35 +01:00
md5checksumORG = $( md5sum $scbPath 2>/dev/null | head -n1 | cut -d " " -f1)
md5checksumCPY = $( md5sum $localBackupPath 2>/dev/null | head -n1 | cut -d " " -f1)
2019-04-15 14:59:53 +01:00
if [ " ${ md5checksumORG } " != " ${ md5checksumCPY } " ] ; then
echo "--> Channel Backup File changed"
2019-04-15 18:45:16 +01:00
# make copy to sd card (as local basic backup)
2024-04-08 11:48:12 +02:00
mkdir -p ${ localBackupDir } 2>/dev/null
2021-12-14 23:34:35 +01:00
cp $scbPath $localBackupPath
2024-04-08 11:48:12 +02:00
if [ $? -eq 0 ] ; then
echo " OK channel.backup copied to ' ${ localBackupPath } ' "
else
logger -p daemon.err " _background.sh FAIL channel.backup copy to ' ${ localBackupPath } ' "
echo " FAIL channel.backup copy to ' ${ localBackupPath } ' "
fi
2021-12-14 23:34:35 +01:00
cp $scbPath $localTimestampedPath
2024-04-08 11:48:12 +02:00
if [ $? -eq 0 ] ; then
echo " OK channel.backup copied to ' ${ localTimestampedPath } ' "
else
logger -p daemon.err " _background.sh FAIL channel.backup copy to ' ${ localTimestampedPath } ' "
echo " FAIL channel.backup copy to ' ${ localTimestampedPath } ' "
fi
# copy to boot drive (for easy recovery)
if [ " ${ raspi_bootdir } " != "" ] ; then
cp $scbPath ${ raspi_bootdir } /channel.backup
if [ $? -eq 0 ] ; then
echo " OK channel.backup copied to ' ${ raspi_bootdir } /channel.backup' "
else
logger -p daemon.err " _background.sh FAIL channel.backup copy to ' ${ raspi_bootdir } /channel.backup' "
echo " FAIL channel.backup copy to ' ${ raspi_bootdir } /channel.backup' "
fi
else
echo "No boot drive found - skip copy to boot"
fi
2020-06-30 16:35:33 +02:00
2020-07-04 01:00:15 +02:00
# check if a additional local backup target is set
2020-07-04 14:03:41 +02:00
# see ./config.scripts/blitz.backupdevice.sh
if [ " ${ localBackupDeviceUUID } " != "" ] && [ " ${ localBackupDeviceUUID } " != "off" ] ; then
2020-07-04 01:00:15 +02:00
# check if device got mounted on "/mnt/backup" (gets mounted by _bootstrap.sh)
backupDeviceExists = $( df | grep -c "/mnt/backup" )
if [ ${ backupDeviceExists } -gt 0 ] ; then
echo "--> Additional Local Backup Device"
2021-12-14 23:34:35 +01:00
cp ${ localBackupPath } /mnt/backup/
cp ${ localTimestampedPath } /mnt/backup/
2020-07-04 01:00:15 +02:00
2021-08-27 03:59:21 -04:00
# check results
2020-07-04 01:00:15 +02:00
result = $?
if [ ${ result } -eq 0 ] ; then
2021-08-27 03:59:21 -04:00
echo "OK - Successful Copy to additional Backup Device"
2020-07-04 01:00:15 +02:00
else
echo " FAIL - Copy to additional Backup Device exited with ${ result } "
fi
2020-06-30 17:00:54 +02:00
else
2020-07-04 01:00:15 +02:00
echo "FAIL - BackupDrive mount - check if device is connected & UUID is correct" >> $logFile
2020-06-30 17:00:54 +02:00
fi
fi
2024-02-16 12:03:42 -05:00
# check if a SCP backup target is set
2020-05-01 20:51:15 +02:00
# parameter in raspiblitz.conf:
2024-02-16 12:03:42 -05:00
# scpBackupTarget='[USER]@[SERVER]:[DIRPATH-WITHOUT-ENDING-/]'
# optionally a custom option string for the scp command can be set with
# scpBackupOptions='[YOUR-CUSTOM-OPTIONS]'
2019-04-15 18:45:16 +01:00
# On target server add the public key of your RaspiBlitz to the authorized_keys for the user
# https://www.linode.com/docs/security/authentication/use-public-key-authentication-with-ssh/
2024-02-16 12:03:42 -05:00
if [ ${# scpBackupTarget } -gt 0 ] ; then
2022-09-29 13:56:23 -05:00
echo "--> Offsite-Backup SFTP Server"
2024-02-16 12:03:42 -05:00
if [ " ${ scpBackupOptions } " = = "" ] ; then
scpBackupOptions = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
2021-12-14 23:34:35 +01:00
fi
2019-04-16 02:10:58 +01:00
# its ok to ignore known host, because data is encrypted (worst case of MiM would be: no offsite channel backup)
2020-05-01 20:51:15 +02:00
# but its more likely that without ignoring known host, script might not run thru and that way: no offsite channel backup
2024-02-16 12:03:42 -05:00
scp ${ scpBackupOptions } ${ localBackupPath } ${ scpBackupTarget } /
scp ${ scpBackupOptions } ${ localTimestampedPath } ${ scpBackupTarget } /
2019-04-15 19:37:41 +01:00
result = $?
if [ ${ result } -eq 0 ] ; then
2024-02-16 12:03:42 -05:00
echo "OK - SCP Backup exited with 0"
2019-04-15 19:37:41 +01:00
else
2024-02-16 12:03:42 -05:00
echo " FAIL - SCP Backup exited with ${ result } "
2019-04-15 19:37:41 +01:00
fi
2019-04-15 18:45:16 +01:00
fi
2021-09-14 22:36:54 +07:00
# check if Nextcloud backups are enabled
if [ $nextcloudBackupServer ] && [ $nextcloudBackupUser ] && [ $nextcloudBackupPassword ] ; then
echo "--> Offsite-Backup Nextcloud"
2021-12-14 23:34:35 +01:00
source <( /home/admin/config.scripts/nextcloud.upload.sh upload ${ localBackupPath } )
source <( /home/admin/config.scripts/nextcloud.upload.sh upload ${ localTimestampedPath } )
2021-09-14 22:36:54 +07:00
if [ ${# err } -gt 0 ] ; then
echo " FAIL - ${ err } "
else
echo " OK - ${ upload } "
fi
fi
2019-04-17 04:44:05 +01:00
#else
# echo "Channel Backup File not changed."
2019-04-15 14:59:53 +01:00
fi
2019-04-17 04:44:05 +01:00
#else
# echo "No Channel Backup File .."
2019-04-15 14:59:53 +01:00
fi
fi
2022-11-22 19:27:25 +01:00
###################################
# Emergency Backup Monitoring (CLN)
###################################
2022-11-17 11:11:56 -06:00
# check every 1min for cln
recheckER = 0
if [ " ${ lightning } " = = "cl" ] || [ " ${ cl } " = = "on" ] ; then
recheckER = $(( $counter % 60 ))
fi
if [ ${ recheckER } -eq 1 ] ; then
#echo "ER Monitoring ..."
source ${ configFile }
source <( /home/admin/config.scripts/network.aliases.sh getvars cl ${ chain } net)
# check if emergency.recover exists
erPath = /home/bitcoin/.lightning/${ CLNETWORK } /emergency.recover
erExists = $( ls $erPath 2>/dev/null | grep -c 'emergency.recover' )
if [ ${ erExists } -eq 1 ] ; then
# timestamp backup filename
timestampedFileName = ${ netprefix } emergency-$( date "+%Y%m%d-%H%M%S" ) .recover
localBackupDir = /home/admin/backups/er
localBackupPath = ${ localBackupDir } /emergency.recover
localTimestampedPath = ${ localBackupDir } /${ timestampedFileName }
#echo "Found Channel Backup File .. check if changed .."
md5checksumORG = $( md5sum $erPath 2>/dev/null | head -n1 | cut -d " " -f1)
md5checksumCPY = $( md5sum $localBackupPath 2>/dev/null | head -n1 | cut -d " " -f1)
if [ " ${ md5checksumORG } " != " ${ md5checksumCPY } " ] ; then
echo "--> Channel Backup File changed"
# make copy to sd card (as local basic backup)
mkdir -p /home/admin/backups/er/ 2>/dev/null
cp $erPath $localBackupPath
cp $erPath $localTimestampedPath
2024-04-08 11:48:12 +02:00
cp $erPath ${ raspi_bootdir } /${ netprefix } emergency.recover
echo " OK emergency.recover copied to ' ${ localBackupPath } ' and ' ${ localTimestampedPath } ' and ' ${ raspi_bootdir } / ${ netprefix } emergency.recover' "
2022-11-17 11:11:56 -06:00
# check if a additional local backup target is set
# see ./config.scripts/blitz.backupdevice.sh
if [ " ${ localBackupDeviceUUID } " != "" ] && [ " ${ localBackupDeviceUUID } " != "off" ] ; then
# check if device got mounted on "/mnt/backup" (gets mounted by _bootstrap.sh)
backupDeviceExists = $( df | grep -c "/mnt/backup" )
if [ ${ backupDeviceExists } -gt 0 ] ; then
echo "--> Additional Local Backup Device"
cp ${ localBackupPath } /mnt/backup/
cp ${ localTimestampedPath } /mnt/backup/
# check results
result = $?
if [ ${ result } -eq 0 ] ; then
echo "OK - Successful Copy to additional Backup Device"
else
echo " FAIL - Copy to additional Backup Device exited with ${ result } "
fi
else
echo "FAIL - BackupDrive mount - check if device is connected & UUID is correct" >> $logFile
fi
fi
2024-02-16 12:03:42 -05:00
# check if a SCP backup target is set
2022-11-17 11:11:56 -06:00
# parameter in raspiblitz.conf:
2024-02-16 12:03:42 -05:00
# scpBackupTarget='[USER]@[SERVER]:[DIRPATH-WITHOUT-ENDING-/]'
# optionally a custom option string for the scp command can be set with
# scpBackupOptions='[YOUR-CUSTOM-OPTIONS]'
2022-11-17 11:11:56 -06:00
# On target server add the public key of your RaspiBlitz to the authorized_keys for the user
# https://www.linode.com/docs/security/authentication/use-public-key-authentication-with-ssh/
2024-02-16 12:03:42 -05:00
if [ ${# scpBackupTarget } -gt 0 ] ; then
2022-11-17 11:11:56 -06:00
echo "--> Offsite-Backup SFTP Server"
2024-02-16 12:03:42 -05:00
if [ " ${ scpBackupOptions } " = = "" ] ; then
scpBackupOptions = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
2022-11-17 11:11:56 -06:00
fi
# its ok to ignore known host, because data is encrypted (worst case of MiM would be: no offsite channel backup)
# but its more likely that without ignoring known host, script might not run thru and that way: no offsite channel backup
2024-02-16 12:03:42 -05:00
scp ${ scpBackupOptions } ${ localBackupPath } ${ scpBackupTarget } /
scp ${ scpBackupOptions } ${ localTimestampedPath } ${ scpBackupTarget } /
2022-11-17 11:11:56 -06:00
result = $?
if [ ${ result } -eq 0 ] ; then
2024-02-16 12:03:42 -05:00
echo "OK - SCP Backup exited with 0"
2022-11-17 11:11:56 -06:00
else
2024-02-16 12:03:42 -05:00
echo " FAIL - SCP Backup exited with ${ result } "
2022-11-17 11:11:56 -06:00
fi
fi
# check if Nextcloud backups are enabled
if [ $nextcloudBackupServer ] && [ $nextcloudBackupUser ] && [ $nextcloudBackupPassword ] ; then
echo "--> Offsite-Backup Nextcloud"
source <( /home/admin/config.scripts/nextcloud.upload.sh upload ${ localBackupPath } )
source <( /home/admin/config.scripts/nextcloud.upload.sh upload ${ localTimestampedPath } )
if [ ${# err } -gt 0 ] ; then
echo " FAIL - ${ err } "
else
echo " OK - ${ upload } "
fi
fi
#else
# echo "Channel Backup File not changed."
fi
#else
# echo "No Channel Backup File .."
fi
fi
2020-05-27 00:15:02 +02:00
###############################
2022-12-07 19:39:28 +00:00
# SUBSCRIPTION RENEWS
2020-05-27 00:15:02 +02:00
###############################
2020-07-15 14:49:42 +02:00
# check every 20min
recheckSubscription = $(( ( $counter % 1200 ) + 1 ))
2020-05-27 00:15:02 +02:00
if [ ${ recheckSubscription } -eq 1 ] ; then
# IP2TOR subscriptions (that will need renew in next 20min = 1200 secs)
2020-07-15 14:49:42 +02:00
sudo -u admin /home/admin/config.scripts/blitz.subscriptions.ip2tor.py subscriptions-renew 1800
2020-05-27 00:15:02 +02:00
fi
2019-12-12 10:26:21 +01:00
###############################
# RAID data check (BRTFS)
###############################
# see https://github.com/rootzoll/raspiblitz/issues/360#issuecomment-467698260
# check every hour
2020-05-27 00:15:02 +02:00
recheckRAID = $(( ( $counter % 3600 ) + 1 ))
2019-12-12 10:26:21 +01:00
if [ ${ recheckRAID } -eq 1 ] ; then
2021-09-14 12:30:22 +02:00
2021-12-14 23:34:35 +01:00
# check if BTRTFS raid is active & scrub
2024-06-19 23:02:41 +02:00
logger -p info "background.sh - RAID data check"
2021-12-14 23:34:35 +01:00
source <( /home/admin/config.scripts/blitz.datadrive.sh status)
if [ " ${ isBTRFS } " = = "1" ] && [ " ${ isRaid } " = = "1" ] ; then
2019-12-12 10:26:21 +01:00
echo "STARTING BTRFS RAID DATA CHECK ..."
2021-12-14 23:34:35 +01:00
btrfs scrub start /mnt/hdd/
2019-12-12 10:26:21 +01:00
fi
fi
2018-12-13 21:59:58 +01:00
###############################
2018-12-22 16:44:15 +01:00
# LND AUTO-UNLOCK
###############################
2021-12-14 23:34:35 +01:00
# check every 10secs (only if LND is active)
recheckAutoUnlock = 0
if [ " ${ lightning } " = = "lnd" ] || [ " ${ lnd } " = = "on" ] ; then
recheckAutoUnlock = $(( ( $counter % 10 ) + 1 ))
fi
2018-12-22 16:44:15 +01:00
if [ ${ recheckAutoUnlock } -eq 1 ] ; then
# check if auto-unlock feature if activated
if [ " ${ autoUnlock } " = "on" ] ; then
# check if lnd is locked
2021-08-04 00:18:30 +02:00
source <( /home/admin/config.scripts/lnd.unlock.sh status)
if [ " ${ locked } " != "0" ] ; then
2018-12-22 16:44:15 +01:00
2018-12-23 15:47:01 +01:00
echo "STARTING AUTO-UNLOCK ..."
2021-12-14 23:34:35 +01:00
/home/admin/config.scripts/lnd.unlock.sh
2021-09-14 12:30:22 +02:00
2018-12-22 16:44:15 +01:00
fi
fi
fi
2018-12-11 13:15:15 +01:00
###############################
# Prepare next loop
###############################
# sleep 1 sec
sleep 1
2022-12-07 19:39:28 +00:00
done
