raspiblitz/home.admin/_background.sh

#!/bin/bash

# This script runs on after start in background
# as a service and gets restarted on failure
# it runs ALMOST every seconds

# INFOFILE - state data from bootstrap
infoFile="/home/admin/raspiblitz.info"

# CONFIGFILE - configuration of RaspiBlitz
configFile="/mnt/hdd/raspiblitz.conf"

# LOGS see: sudo journalctl -f -u background

# Check if HDD contains configuration
configExists=$(ls ${configFile} | grep -c '.conf')
if [ ${configExists} -eq 1 ]; then
    source ${configFile}
else
    source ${infoFile}
fi

echo "_background.sh STARTED"

# monitor lost LND sync
syncedSince=0
lastSyncState=0

counter=0
while [ 1 ]
do

  ###############################
  # Prepare this loop
  ###############################

  # count up
  counter=$(($counter+1))

  # gather the uptime seconds
  upSeconds=$(cat /proc/uptime | grep -o '^[0-9]\+')

  ####################################################
  # RECHECK DHCP-SERVER 
  # https://github.com/rootzoll/raspiblitz/issues/160
  ####################################################

  # every 5 minutes
  recheckDHCP=$((($counter % 300)+1))
  if [ ${recheckDHCP} -eq 1 ]; then
    echo "*** RECHECK DHCP-SERVER  ***"

    # get the local network IP
    localip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/')
    echo "localip(${localip})"

    # detect a missing DHCP config 
    if [ "${localip:0:4}" = "169." ]; then
      echo "Missing DHCP detected ... trying emergency reboot"
      sudo shutdown -r now
    else
      echo "DHCP OK"
    fi

  fi

  ####################################################
  # CHECK FOR UNDERVOLTAGE REPORTS
  # every 1 hour scan for undervoltage reports
  ####################################################
  recheckUndervoltage=$(($counter % 3600))
  if [ ${recheckUndervoltage} -eq 1 ]; then
    echo "*** RECHECK UNDERVOLTAGE ***"
    countReports=$(sudo cat /var/log/syslog | grep -c "Under-voltage detected!")
    echo "${countReports} undervoltage reports found in syslog"
    if [ ${#undervoltageReports} -eq 0 ]; then
      # write new value to info file
      undervoltageReports="${countReports}"
      echo "undervoltageReports=${undervoltageReports}" >> ${infoFile}
    else
      # update value in info file
      sed -i "s/^undervoltageReports=.*/undervoltageReports=${countReports}/g" ${infoFile}
    fi
  fi

  ####################################################
  # RECHECK PUBLIC IP
  # when public IP changes, restart LND with new IP
  ####################################################

  # every 15min - not too often
  # because its a ping to external service
  recheckPublicIP=$((($counter % 900)+1))
  # prevent when lndAddress is set
  if [ ${#lndAddress} -gt 3 ]; then
    recheckPublicIP=0
  fi
  updateDynDomain=0
  if [ ${recheckPublicIP} -eq 1 ]; then
    echo "*** RECHECK PUBLIC IP ***"

    # execute only after setup when config exists
    if [ ${configExists} -eq 1 ]; then

      # get actual public IP
      freshPublicIP=$(curl -s http://v4.ipv6-test.com/api/myip.php 2>/dev/null)

      # sanity check on IP data
      # see https://github.com/rootzoll/raspiblitz/issues/371#issuecomment-472416349
      echo "-> sanity check of new IP data"
      if [[ $freshPublicIP =~ ^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$ ]]; then
        echo "OK IPv6"
      elif [[ $freshPublicIP =~ ^([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$ ]]; then
        echo "OK IPv4"
      else
        echo "FAIL - not an IPv4 or IPv6 address"
        freshPublicIP=""
      fi

      if [ ${#freshPublicIP} -eq 0 ]; then 

        echo "freshPublicIP is ZERO - ignoring"

      # check if changed
      elif [ "${freshPublicIP}" != "${publicIP}" ]; then

        # 1) update config file
        echo "update config value"
        sed -i "s/^publicIP=.*/publicIP='${freshPublicIP}'/g" ${configFile}
        publicIP='${freshPublicIP}'

        # 2) only restart LND if dynDNS is activated
        # because this signals that user wants "public node"
        if [ ${#dynDomain} -gt 0 ]; then
          echo "restart LND with new environment config"
          # restart and let to auto-unlock (if activated) do the rest
          sudo systemctl restart lnd.service
        fi

        # 2) trigger update if dnyamic domain (if set)
        updateDynDomain=1

      else
        echo "public IP has not changed"
      fi

    else
      echo "skip - because setup is still running"
    fi

  fi


  ###############################
  # LND MONITOR LOST SYNC
  ###############################

  # check every 5min
  recheckSync=$(($counter % 300))
  if [ ${recheckSync} -eq 1 ]; then
    source ${configFile}
    echo "LND MONITOR LOST SYNC ..."
    lndSynced=$(sudo -u bitcoin /usr/local/bin/lncli --chain=${network} --network=${chain}net getinfo 2>/dev/null | jq -r '.synced_to_chain' | grep -c true)
    echo "lndSynced(${lndSynced})"
    echo "syncedSince(${syncedSince})"
    echo "lastSyncState(${lastSyncState})"
    if [ ${lndSynced} -eq ${lastSyncState} ]; then

      echo "no sync change"
      if [ ${lndSynced} -eq 1 ]; then
        echo "all is good - LND still in sync now for:"
        actualSecondsTimestamp=$(date +%s)
        secondsInSync=$(echo "${actualSecondsTimestamp}-"${syncedSince} | bc)
        echo "${secondsInSync} seconds"
        if [ "${backupTorrentSeeding}" == "on" ]; then
          echo "Backup Torrent Seeding is ON - check if already running"
          source <(sudo -u admin /home/admin/50torrentHDD.sh status)
          if [ "${baseSeeding}" == "0" ] || [ "${updateSeeding}" == "0" ]; then
            echo "---> STARTING Backup Torrent Seeding"
            sudo -u admin /home/admin/50torrentHDD.sh backup-torrent-hosting
          else
            echo "Backup Torrent Seeding - already running"
          fi
        else
          echo "Backup Torrent Seeding is OFF"
        fi
      else
        echo "still not in sync"
        if [ ${syncedSince} -gt 0 ]; then
        
          echo "was in sync at least once since rinning but lost now for:"
          actualSecondsTimestamp=$(date +%s)
          secondsOutOfSync=$(echo "${actualSecondsTimestamp}-"${syncedSince} | bc)
          echo "${secondsOutOfSync} seconds"

          # when >10min out of sync
          if [ ${secondsOutOfSync} -gt 600 ]; then
            echo "! LND fell out of sync for longer then 10 minutes !"
            if [ "${backupTorrentSeeding}" == "on" ]; then
              echo "Backup Torrent Seeding is ON - check if still running"
              source <(sudo -u admin /home/admin/50torrentHDD.sh status)
              if [ "${baseSeeding}" == "1" ] || [ "${updateSeeding}" == "1" ]; then
                echo "---> STOPPING Backup Torrent Seeding"
                sudo -u admin /home/admin/50torrentHDD.sh stop
              else
                echo "No Backup Torrent Seeding - already stopped"
              fi
            else
              echo "Backup Torrent Seeding is OFF"
            fi
          fi

          # when >1h out of sync
          if [ ${secondsOutOfSync} -gt 3600 ]; then
            echo "!!!! LND fell out of sync for longer then 1 hour !!!"
            # TODO: When auto-unlock is ON --> consider implementing restart (this sometimes help)
          fi

        else
          echo "LND was never in sync since since started (could be multiple reasons - dont act)"
        fi
      fi

    else
      echo "sync change detected"
      if [ ${lastSyncState} -eq 1 ] && [ ${lndSynced} -eq 0 ]; then
        echo "--> LND SNC LOST"
      else
        if [ ${syncedSince} -eq 0 ]; then
          echo "--> LND SYNC GAINED"
        else
          echo "--> LND SYNC RECOVERED"
        fi
        syncedSince=$(date +%s)
      fi
    fi
  
    lastSyncState=${lastSyncState}
  fi

  ###############################
  # LND AUTO-UNLOCK
  ###############################

  # check every 10secs
  recheckAutoUnlock=$((($counter % 10)+1))
  if [ ${recheckAutoUnlock} -eq 1 ]; then

    # check if auto-unlock feature if activated
    if [ "${autoUnlock}" = "on" ]; then

      # check if lnd is locked
      locked=$(sudo -u bitcoin /usr/local/bin/lncli --chain=${network} --network=${chain}net getinfo 2>&1 | grep -c unlock)
      if [ ${locked} -gt 0 ]; then

        echo "STARTING AUTO-UNLOCK ..."

        # building REST command
        passwordC=$(sudo cat /root/lnd.autounlock.pwd)
        command="sudo python /home/admin/config.scripts/lnd.unlock.py '${passwordC}'"
        bash -c "${command}"
        
      fi
    fi
  fi

  ###############################
  # UPDATE DYNAMIC DOMAIN
  # like afraid.org
  # ! experimental
  ###############################

  # if not activated above, update every 6 hours
  if [ ${updateDynDomain} -eq 0 ]; then
    # dont +1 so that it gets executed on first loop
    updateDynDomain=$(($counter % 21600))
  fi
  if [ ${updateDynDomain} -eq 1 ]; then
    echo "*** UPDATE DYNAMIC DOMAIN ***"
    # check if update URL for dyn Domain is set
    if [ ${#dynUpdateUrl} -gt 6 ]; then
      # calling the update url
      echo "calling: ${dynUpdateUrl}"
      echo "to update domain: ${dynDomain}"
      curl --connect-timeout 6 ${dynUpdateUrl}
    else
      echo "'dynUpdateUrl' not set in ${configFile}"
    fi
  fi

  ###############################
  # Prepare next loop
  ###############################

  # sleep 1 sec
  sleep 1

  # limit counter to max seconds per week:
  # 604800 = 60sec * 60min * 24hours * 7days
  if [ ${counter} -gt 604800 ]; then
    counter=0
    echo "counter zero reset"
  fi

done