ci: update images, add arm64-rpi-debian12 build (#3985)

* ci: update images, add arm64-rpi-debian12 build

* ci: add apt update and concurrency

* fix: use netcat-openbsd in build

* ci: add concurrency to all jobs

* fix: simplify image check and add python 3.11

* ci: update amd64 base image to Debian 12.0.0

* build: separate apt_installs

* add btrfs-progs only if not bookworm on aarch64

* build: remove armbian, add aarch64 debian

---------

Co-authored-by: /rootzoll <christian@geektank.de>

.github/workflows/amd64-fatpack-image.yml

@@ -1,5 +1,9 @@
 name: amd64-fatpack-image-build
 
+concurrency:
+  group: amd64-fatpack-image-build-${{ github.head_ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   #push:

.github/workflows/amd64-lean-image.yml

@@ -1,27 +1,31 @@
 name: amd64-lean-image-build
 
+concurrency:
+  group: amd64-lean-image-build-${{ github.head_ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
-    branches: [ "dev", "v1.8", "v1.9" ]
+    branches: ["dev", "v1.8", "v1.9"]
     paths:
-      - 'build_sdcard.sh'
-      - 'home.admin/bitcoin.install.sh'
-      - 'home.admin/tor.install.sh'
-      - 'home.admin/blitz.i2pd.sh'
-      - 'home.admin/blitz.web.sh'
-      - 'home.admin/blitz.display.sh'
-      - 'ci/amd64/**'
+      - "build_sdcard.sh"
+      - "home.admin/bitcoin.install.sh"
+      - "home.admin/tor.install.sh"
+      - "home.admin/blitz.i2pd.sh"
+      - "home.admin/blitz.web.sh"
+      - "home.admin/blitz.display.sh"
+      - "ci/amd64/**"
   pull_request:
-    branches: [ "dev", "v1.8", "v1.9" ]
+    branches: ["dev", "v1.8", "v1.9"]
     paths:
-      - 'build_sdcard.sh'
-      - 'home.admin/bitcoin.install.sh'
-      - 'home.admin/tor.install.sh'
-      - 'home.admin/blitz.i2pd.sh'
-      - 'home.admin/blitz.web.sh'
-      - 'home.admin/blitz.display.sh'
-      - 'ci/amd64/**'
+      - "build_sdcard.sh"
+      - "home.admin/bitcoin.install.sh"
+      - "home.admin/tor.install.sh"
+      - "home.admin/blitz.i2pd.sh"
+      - "home.admin/blitz.web.sh"
+      - "home.admin/blitz.display.sh"
+      - "ci/amd64/**"
 
 jobs:
   amd64-image-build:

.github/workflows/arm64-rpi-debian12-lean-image.yml

@@ -0,0 +1,71 @@
+name: arm64-rpi-debian12-lean-image-build
+
+concurrency:
+  group: arm64-rpi-debian12-lean-image-build-${{ github.head_ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["dev"]
+    paths:
+      - "build_sdcard.sh"
+      - "home.admin/bitcoin.install.sh"
+      - "home.admin/tor.install.sh"
+      - "home.admin/blitz.i2pd.sh"
+      - "home.admin/blitz.web.sh"
+      - "home.admin/blitz.display.sh"
+      - "ci/arm64-rpi/**"
+  pull_request:
+    branches: ["dev"]
+    paths:
+      - "build_sdcard.sh"
+      - "home.admin/bitcoin.install.sh"
+      - "home.admin/tor.install.sh"
+      - "home.admin/blitz.i2pd.sh"
+      - "home.admin/blitz.web.sh"
+      - "home.admin/blitz.display.sh"
+      - "ci/arm64-rpi/**"
+
+jobs:
+  arm64-rpi-image-build:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set values
+        run: |
+          echo "BUILD_DATE=$(date +"%Y-%m-%d")" >> $GITHUB_ENV
+          echo "BUILD_VERSION=$(git describe --always --tags)" >> $GITHUB_ENV
+
+      - name: Hello Raspiblitz
+        run: echo "Building the raspiblitz-arm64-rpi-lean-image-${{ env.BUILD_DATE }}-${{ env.BUILD_VERSION }}"
+
+      - name: Run the build script
+        run: |
+          cd ci/arm64-rpi
+          bash packer.build.arm64-rpi.sh lean $GITHUB_ACTOR $GITHUB_HEAD_REF "https://raspi.debian.net/tested/20230612_raspi_4_bookworm.img.xz" "a68cd2bfe7831c438d8a5d832803ae0db17afec9f3cd370d9e8748c7b5456283"
+
+      - name: Compute checksum of the raw image
+        run: |
+          cd ci/arm64-rpi
+          sha256sum raspiblitz-arm64-rpi-lean.img > raspiblitz-arm64-rpi-lean.img.sha256
+
+      - name: Compress image
+        run: |
+          cd ci/arm64-rpi
+          gzip -v9 raspiblitz-arm64-rpi-lean.img
+
+      - name: Compute checksum of the compressed image
+        run: |
+          cd ci/arm64-rpi
+          sha256sum raspiblitz-arm64-rpi-lean.img.gz > raspiblitz-arm64-rpi-lean.img.gz.sha256
+
+      - name: Upload the image and checksums
+        uses: actions/upload-artifact@v3
+        with:
+          name: raspiblitz-arm64-rpi--debian12-image-${{ env.BUILD_DATE }}-${{ env.BUILD_VERSION }}
+          path: |
+            ${{ github.workspace }}/ci/arm64-rpi/raspiblitz-arm64-rpi-lean.img.sha256
+            ${{ github.workspace }}/ci/arm64-rpi/raspiblitz-arm64-rpi-lean.img.gz
+            ${{ github.workspace }}/ci/arm64-rpi/raspiblitz-arm64-rpi-lean.img.gz.sha256

.github/workflows/arm64-rpi-fatpack-image.yml

@@ -1,5 +1,9 @@
 name: arm64-rpi-fatpack-image-build
 
+concurrency:
+  group: arm64-rpi-fatpack-image-build-${{ github.head_ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   #push:

.github/workflows/arm64-rpi-lean-image.yml

@@ -1,27 +1,31 @@
 name: arm64-rpi-lean-image-build
 
+concurrency:
+  group: arm64-rpi-lean-image-build-${{ github.head_ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
-    branches: [ "dev", "v1.8", "v1.9" ]
+    branches: ["dev", "v1.8", "v1.9"]
     paths:
-      - 'build_sdcard.sh'
-      - 'home.admin/bitcoin.install.sh'
-      - 'home.admin/tor.install.sh'
-      - 'home.admin/blitz.i2pd.sh'
-      - 'home.admin/blitz.web.sh'
-      - 'home.admin/blitz.display.sh'
-      - 'ci/arm64-rpi/**'
+      - "build_sdcard.sh"
+      - "home.admin/bitcoin.install.sh"
+      - "home.admin/tor.install.sh"
+      - "home.admin/blitz.i2pd.sh"
+      - "home.admin/blitz.web.sh"
+      - "home.admin/blitz.display.sh"
+      - "ci/arm64-rpi/**"
   pull_request:
-    branches: [ "dev", "v1.8", "v1.9" ]
+    branches: ["dev", "v1.8", "v1.9"]
     paths:
-      - 'build_sdcard.sh'
-      - 'home.admin/bitcoin.install.sh'
-      - 'home.admin/tor.install.sh'
-      - 'home.admin/blitz.i2pd.sh'
-      - 'home.admin/blitz.web.sh'
-      - 'home.admin/blitz.display.sh'
-      - 'ci/arm64-rpi/**'
+      - "build_sdcard.sh"
+      - "home.admin/bitcoin.install.sh"
+      - "home.admin/tor.install.sh"
+      - "home.admin/blitz.i2pd.sh"
+      - "home.admin/blitz.web.sh"
+      - "home.admin/blitz.display.sh"
+      - "ci/arm64-rpi/**"
 
 jobs:
   arm64-rpi-image-build:

alternative.platforms/amd64/packer/raspiblitz.json

@@ -64,9 +64,9 @@
   "variables": {
     "branch": "dev",
     "github_user": "rootzoll",
-    "iso_checksum": "2af8f43d4a7ab852151a7f630ba596572213e17d3579400b5648eba4cc974ed0",
+    "iso_checksum": "3b0e9718e3653435f20d8c2124de6d363a51a1fd7f911b9ca0c6db6b3d30d53e",
     "iso_checksum_type": "sha256",
-    "iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-10.6.0-amd64-netinst.iso"
+    "iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-12.0.0-amd64-netinst.iso"
   }
 }
 

build_sdcard.sh

@@ -148,13 +148,14 @@ range_argument(){
   fi
 }
 
-apt_install(){
-    apt install -y ${@}
+apt_install() {
+  for package in "$@"; do
+    apt install -y -q "$package"
     if [ $? -eq 100 ]; then
-        echo "FAIL! apt failed to install needed packages!"
-        echo ${@}
-        exit 1
+      echo "FAIL! apt failed to install package: $package"
+      exit 1
     fi
+  done
 }
 
 general_utils="curl"
@@ -248,14 +249,8 @@ if [ $(cat /etc/os-release 2>/dev/null | grep -c 'Debian') -gt 0 ]; then
   if [ -f /etc/apt/sources.list.d/raspi.list ] && [ "${cpu}" = aarch64 ]; then
     # default image for RaspberryPi
     baseimage="raspios_arm64"
-  elif [ $(uname -n | grep -c 'rpi') -gt 0 ] && [ "${cpu}" = aarch64 ]; then
-    # experimental: a clean alternative image of debian for RaspberryPi
-    baseimage="debian_rpi64"
-  elif [ "${cpu}" = "arm" ] || [ "${cpu}" = "aarch64" ]; then
-    # experimental: fallback for all debian on arm
-    baseimage="armbian"
   else
-    # experimental: fallback for all debian on other CPUs
+    # experimental: fallback for all to debian
     baseimage="debian"
   fi
 elif [ $(cat /etc/os-release 2>/dev/null | grep -c 'Ubuntu') -gt 0 ]; then
@@ -296,7 +291,7 @@ HandleLidSwitchDocked=ignore" | tee /etc/systemd/logind.conf.d/nosuspend.conf
 # https://github.com/rootzoll/raspiblitz/issues/138
 # https://daker.me/2014/10/how-to-fix-perl-warning-setting-locale-failed-in-raspbian.html
 # https://stackoverflow.com/questions/38188762/generate-all-locales-in-a-docker-image
-if [ "${baseimage}" = "raspios_arm64" ]||[ "${baseimage}" = "debian_rpi64" ]||[ "${baseimage}" = "armbian" ]; then
+if [ "${baseimage}" = "raspios_arm64" ] || [ "${baseimage}" = "debian" ]; then
   echo -e "\n*** FIXING LOCALES FOR BUILD ***"
 
   sed -i "s/^# en_US.UTF-8 UTF-8.*/en_US.UTF-8 UTF-8/g" /etc/locale.gen
@@ -323,7 +318,7 @@ echo -e "\n*** SOFTWARE UPDATE ***"
 # based on https://raspibolt.org/system-configuration.html#system-update
 # htop git curl bash-completion vim jq dphys-swapfile bsdmainutils -> helpers
 # autossh telnet vnstat -> network tools bandwidth monitoring for future statistics
-# parted dosfstolls -> prepare for format data drive
+# parted dosfstools -> prepare for format data drive
 # btrfs-progs -> prepare for BTRFS data drive raid
 # fbi -> prepare for display graphics mode. https://github.com/rootzoll/raspiblitz/pull/334
 # sysbench -> prepare for powertest
@@ -339,15 +334,16 @@ echo -e "\n*** SOFTWARE UPDATE ***"
 # sqlite3 -> database
 # fdisk -> create partitions
 # lsb-release -> needed to know which distro version we're running to add APT sources
-general_utils="policykit-1 htop git curl bash-completion vim jq dphys-swapfile bsdmainutils autossh telnet vnstat parted dosfstools btrfs-progs fbi sysbench build-essential dialog bc python3-dialog unzip whois fdisk lsb-release smartmontools"
-
+general_utils="policykit-1 htop git curl bash-completion vim jq dphys-swapfile bsdmainutils autossh telnet vnstat parted dosfstools fbi sysbench build-essential dialog bc python3-dialog unzip whois fdisk lsb-release smartmontools"
+# add btrfs-progs if not bookworm on aarch64
+[ "${architecture}" = "aarch64" ] && ! grep "12 (bookworm)" < /etc/os-release && general_utils="${general_utils} btrfs-progs"
 # python3-mako --> https://github.com/rootzoll/raspiblitz/issues/3441
 python_dependencies="python3-venv python3-dev python3-wheel python3-jinja2 python3-pip python3-mako"
 server_utils="rsync net-tools xxd netcat-openbsd openssh-client openssh-sftp-server sshpass psmisc ufw sqlite3"
 [ "${baseimage}" = "armbian" ] && armbian_dependencies="armbian-config" # add armbian-config
 [ "${architecture}" = "amd64" ] && amd64_dependencies="network-manager" # add amd64 dependency
 
-apt_install ${general_utils} ${python_dependencies} ${server_utils} ${armbian_dependencies} ${amd64_dependencies}
+apt_install ${general_utils} ${python_dependencies} ${server_utils} ${amd64_dependencies}
 apt clean -y
 apt autoremove -y
 
@@ -356,14 +352,14 @@ echo -e "\n*** Python DEFAULT libs & dependencies ***"
 if [ -f "/usr/bin/python3.11" ]; then
   # use python 3.11 if available
   update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1
-  # keep pyhton backwards compatible
+  # keep python backwards compatible
   ln -s /usr/bin/python3.11 /usr/bin/python3.9
   ln -s /usr/bin/python3.11 /usr/bin/python3.10
-  echo "python calls python3.10"
+  echo "python calls python3.11"
 elif [ -f "/usr/bin/python3.10" ]; then
   # use python 3.10 if available
   update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
-  # keep pyhton backwards compatible
+  # keep python backwards compatible
   ln -s /usr/bin/python3.10 /usr/bin/python3.9
   echo "python calls python3.10"
 elif [ -f "/usr/bin/python3.9" ]; then
@@ -402,7 +398,7 @@ if [ "$(compgen -u | grep -c pi)" -eq 0 ];then
 fi
 
 # special prepare when Raspbian
-if [ "${baseimage}" = "raspios_arm64" ] || [ "${baseimage}" = "debian_rpi64" ]; then
+if [ "${baseimage}" = "raspios_arm64" ] || [ "${architecture}" = "aarch64" ] && [ "${baseimage}" = "debian" ]; then
 
   echo -e "\n*** PREPARE RASPBERRY OS VARIANTS ***"
   apt_install raspi-config
@@ -473,8 +469,7 @@ echo "root:raspiblitz" | chpasswd
 echo "pi:raspiblitz" | chpasswd
 
 # prepare auto-start of 00infoLCD.sh script on pi user login (just kicks in if auto-login of pi is activated in HDMI or LCD mode)
-if [ "${baseimage}" = "raspios_arm64" ] || [ "${baseimage}" = "debian_rpi64" ] || \
-   [ "${baseimage}" = "armbian" ] || [ "${baseimage}" = "ubuntu" ]; then
+if [ "${baseimage}" = "raspios_arm64" ] || [ "${baseimage}" = "debian" ] || [ "${baseimage}" = "ubuntu" ]; then
   homeFile=/home/pi/.bashrc
   autostartDone=$(grep -c "automatic start the LCD" $homeFile)
   if [ ${autostartDone} -eq 0 ]; then
@@ -727,7 +722,7 @@ echo "Activating CACHE RAM DISK ... "
 /home/admin/_cache.sh keyvalue on
 
 # *** Wifi, Bluetooth & other RaspberryPi configs ***
-if [ "${baseimage}" = "raspios_arm64"  ] || [ "${baseimage}" = "debian_rpi64" ]; then
+if [ "${baseimage}" = "raspios_arm64"  ] || [ "${architecture}" = "aarch64" ] && [ "${baseimage}" = "debian" ]; then
 
   if [ "${wifi_region}" == "off" ]; then
     echo -e "\n*** DISABLE WIFI ***"

ci/arm64-rpi/arm64-rpi.pkr.hcl

@@ -1,13 +1,15 @@
-variable "pack" {}
-variable "github_user" {}
-variable "branch" {}
+variable "pack" { default = "lean" }
+variable "github_user" { default = "raspiblitz" }
+variable "branch" { default = "dev" }
+variable "image_link" { default = "https://downloads.raspberrypi.org/raspios_arm64/images/raspios_arm64-2023-05-03/2023-05-03-raspios-bullseye-arm64.img.xz" }
+variable "image_checksum" { default = "e7c0c89db32d457298fbe93195e9d11e3e6b4eb9e0683a7beb1598ea39a0a7aa" }
 
 source "arm" "raspiblitz-arm64-rpi" {
   file_checksum_type    = "sha256"
-  file_checksum         = "c42856ffca096480180b5aff66e1dad2f727fdc33359b24e0d2d49cc7676b576"
+  file_checksum         = var.image_checksum
   file_target_extension = "xz"
   file_unarchive_cmd    = ["xz", "--decompress", "$ARCHIVE_PATH"]
-  file_urls             = ["https://downloads.raspberrypi.org/raspios_arm64/images/raspios_arm64-2022-09-26/2022-09-22-raspios-bullseye-arm64.img.xz"]
+  file_urls             = [var.image_link]
   image_build_method    = "resize"
   image_chroot_env      = ["PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin"]
   image_partitions {
@@ -41,6 +43,7 @@ build {
       "echo 'nameserver 1.1.1.1' > /etc/resolv.conf",
       "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf",
       "echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections",
+      "apt-get update",
       "apt-get install -y sudo wget",
       "apt-get -y autoremove",
       "apt-get -y clean",
@@ -48,7 +51,7 @@ build {
   }
 
   provisioner "shell" {
-    environment_vars =  [
+    environment_vars = [
       "github_user=${var.github_user}",
       "branch=${var.branch}",
       "pack=${var.pack}"

ci/arm64-rpi/packer.build.arm64-rpi.sh

@@ -2,25 +2,52 @@
 
 if [ $# -gt 0 ]; then
   pack=$1
-else
-  pack=lean
 fi
 
 if [ $# -gt 1 ]; then
   github_user=$2
-else
-  github_user=rootzoll
 fi
 
 if [ $# -gt 2 ]; then
   branch=$3
-else
-  branch=dev
+fi
+
+if [ $# -gt 4 ]; then
+  image_link="$4"
+  image_checksum="$5"
+fi
+
+# Initialize the variables string
+vars=""
+
+# Add the pack variable if it is defined
+if [ -n "${pack}" ]; then
+  vars="$vars -var pack=${pack}"
+fi
+
+# Add the github_user variable if it is defined
+if [ -n "${github_user}" ]; then
+  vars="$vars -var github_user=${github_user}"
+fi
+
+# Add the branch variable if it is defined
+if [ -n "${branch}" ]; then
+  vars="$vars -var branch=${branch}"
+fi
+
+# Add the image_link variable if it is defined
+if [ -n "${image_link}" ]; then
+  vars="$vars -var image_link=${image_link}"
+fi
+
+# Add the image_checksum variable if it is defined
+if [ -n "${image_checksum}" ]; then
+  vars="$vars -var image_checksum=${image_checksum}"
 fi
 
 # Build the image in docker
 echo -e "\nBuild Packer image..."
 # from https://hub.docker.com/r/mkaczanowski/packer-builder-arm/tags
 docker run --rm --privileged -v /dev:/dev -v ${PWD}:/build \
- mkaczanowski/packer-builder-arm:1.0.5@sha256:a6371a3230b94c308e9a8cd55da1ae0d23bd670b7e558b39683c24f489d03e53 \
- build -var github_user=${github_user} -var branch=${branch} -var pack=${pack} arm64-rpi.pkr.hcl
+  mkaczanowski/packer-builder-arm:1.0.6@sha256:ca758415f498e6d4a1d7e82380d6f166d97262cb44af624ff586e6a6c96ad795 \
+  build ${vars} arm64-rpi.pkr.hcl