From 0d0e5581a5195178e09a16d96f65224a3c5d1c2e Mon Sep 17 00:00:00 2001 From: openoms <43343391+openoms@users.noreply.github.com> Date: Sun, 30 Jul 2023 10:51:33 +0200 Subject: [PATCH] ci: UEFI image generation and build updates (#4033) * ci: use uefi for amd64 image, debian 12.1.0 * build: add amd64 legacyboot image action * ci: store var values in config, update readme * fix: add raspi repo on aarch64 only fixes: https://github.com/raspiblitz/raspiblitz/issues/4029 * docs: guide to extend the root partition --- .github/workflows/amd64-fatpack-image.yml | 4 +- .github/workflows/amd64-lean-image.yml | 2 +- .../workflows/amd64-lean-legacyboot-image.yml | 71 ++++++++ Makefile | 56 ++++++- build_sdcard.sh | 3 +- ci/README.md | 158 +++++++++++------- ci/amd64/_common/env.sh | 4 + ci/amd64/debian/amd64-debian.json | 81 --------- ci/amd64/debian/build.amd64-debian.pkr.hcl | 115 +++++++++++++ ci/amd64/debian/http/debian-9/preseed.cfg | 47 ------ ci/amd64/debian/http/preseed.cfg | 72 ++++++++ .../{raspiblitz.sh => build.raspiblitz.sh} | 2 +- ci/amd64/packer.build.amd64-debian.sh | 26 ++- ...64-rpi.pkr.hcl => build.arm64-rpi.pkr.hcl} | 2 +- .../{raspiblitz.sh => build.raspiblitz.sh} | 0 ci/arm64-rpi/packer.build.arm64-rpi.local.sh | 11 +- ci/arm64-rpi/packer.build.arm64-rpi.sh | 13 +- ci/set_variables.sh | 103 +++++++----- 18 files changed, 509 insertions(+), 261 deletions(-) create mode 100644 .github/workflows/amd64-lean-legacyboot-image.yml create mode 100644 ci/amd64/_common/env.sh delete mode 100644 ci/amd64/debian/amd64-debian.json create mode 100644 ci/amd64/debian/build.amd64-debian.pkr.hcl delete mode 100644 ci/amd64/debian/http/debian-9/preseed.cfg create mode 100644 ci/amd64/debian/http/preseed.cfg rename ci/amd64/debian/scripts/{raspiblitz.sh => build.raspiblitz.sh} (92%) rename ci/arm64-rpi/{arm64-rpi.pkr.hcl => build.arm64-rpi.pkr.hcl} (98%) rename ci/arm64-rpi/{raspiblitz.sh => build.raspiblitz.sh} (100%) mode change 100644 => 100755 ci/set_variables.sh diff --git a/.github/workflows/amd64-fatpack-image.yml b/.github/workflows/amd64-fatpack-image.yml index 1e5a6eb30..8b4207539 100644 --- a/.github/workflows/amd64-fatpack-image.yml +++ b/.github/workflows/amd64-fatpack-image.yml @@ -61,12 +61,12 @@ jobs: echo "BUILD_VERSION=$(git describe --always --tags)" >> $GITHUB_ENV - name: Hello RaspiBlitz - run: echo "Building the raspiblitz-amd64-debian-image-${{env.BUILD_DATE}}-${{env.BUILD_VERSION}} at " + run: echo "Building the raspiblitz-amd64-debian-image-${{env.BUILD_DATE}}-${{env.BUILD_VERSION}}" - name: Run the build script run: | cd ci/amd64 - bash packer.build.amd64-debian.sh fatpack $GITHUB_ACTOR $GITHUB_HEAD_REF + bash packer.build.amd64-debian.sh --pack fatpack --github_user $GITHUB_ACTOR --branch $GITHUB_HEAD_REF --preseed_file preseed.cfg --boot uefi--desktop gnome - name: Compute checksum of the raw image run: | diff --git a/.github/workflows/amd64-lean-image.yml b/.github/workflows/amd64-lean-image.yml index 750ca9d33..f36059778 100644 --- a/.github/workflows/amd64-lean-image.yml +++ b/.github/workflows/amd64-lean-image.yml @@ -44,7 +44,7 @@ jobs: - name: Run the build script run: | cd ci/amd64 - bash packer.build.amd64-debian.sh lean $GITHUB_ACTOR $GITHUB_HEAD_REF + bash packer.build.amd64-debian.sh --pack lean --github_user $GITHUB_ACTOR --branch $GITHUB_HEAD_REF --preseed_file preseed.cfg --boot uefi --desktop gnome - name: Compute checksum of the raw image run: | diff --git a/.github/workflows/amd64-lean-legacyboot-image.yml b/.github/workflows/amd64-lean-legacyboot-image.yml new file mode 100644 index 000000000..7fa20f049 --- /dev/null +++ b/.github/workflows/amd64-lean-legacyboot-image.yml @@ -0,0 +1,71 @@ +name: amd64-lean-legacyboot-image-build + +concurrency: + group: amd64-lean-legacyboot-image-build-${{ github.head_ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: ["dev", "v1.8", "v1.9"] + paths: + - "build_sdcard.sh" + - "home.admin/bitcoin.install.sh" + - "home.admin/tor.install.sh" + - "home.admin/blitz.i2pd.sh" + - "home.admin/blitz.web.sh" + - "home.admin/blitz.display.sh" + - "ci/amd64/**" + pull_request: + branches: ["dev", "v1.8", "v1.9"] + paths: + - "build_sdcard.sh" + - "home.admin/bitcoin.install.sh" + - "home.admin/tor.install.sh" + - "home.admin/blitz.i2pd.sh" + - "home.admin/blitz.web.sh" + - "home.admin/blitz.display.sh" + - "ci/amd64/**" + +jobs: + amd64-image-build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set values + run: | + echo "BUILD_DATE=$(date +"%Y-%m-%d")" >> $GITHUB_ENV + echo "BUILD_VERSION=$(git describe --always --tags)" >> $GITHUB_ENV + + - name: Hello RaspiBlitz + run: echo "Building the raspiblitz-amd64-debian-image-${{env.BUILD_DATE}}-${{env.BUILD_VERSION}}" + + - name: Run the build script + run: | + cd ci/amd64 + bash packer.build.amd64-debian.sh --pack lean --github_user $GITHUB_ACTOR --branch $GITHUB_HEAD_REF --preseed_file preseed.cfg --boot bios --desktop none + + - name: Compute checksum of the raw image + run: | + cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu + sha256sum raspiblitz-amd64-debian-lean.qcow2 > raspiblitz-amd64-debian-lean.qcow2.sha256 + + - name: Compress image + run: | + cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu + gzip -v9 raspiblitz-amd64-debian-lean.qcow2 + + - name: Compute checksum of the compressed image + run: | + cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu + sha256sum raspiblitz-amd64-debian-lean.qcow2.gz > raspiblitz-amd64-debian-lean.qcow2.gz.sha256 + + - name: Upload the image and checksums + uses: actions/upload-artifact@v3 + with: + name: raspiblitz-amd64-image-${{env.BUILD_DATE}}-${{env.BUILD_VERSION}} + path: | + ${{github.workspace}}/ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu/raspiblitz-amd64-debian-lean.qcow2.sha256 + ${{github.workspace}}/ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu/raspiblitz-amd64-debian-lean.qcow2.gz + ${{github.workspace}}/ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu/raspiblitz-amd64-debian-lean.qcow2.gz.sha256 diff --git a/Makefile b/Makefile index e50f48d11..13882020f 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,16 @@ SHELL = /bin/bash GITHUB_ACTOR = $(shell git remote -v | grep origin | head -1 | cut -d/ -f4) GITHUB_HEAD_REF = $(shell git rev-parse --abbrev-ref HEAD) -amd64-lean-image: +amd64-lean-desktop-uefi-image: # Run the build script cd ci/amd64 && \ - bash packer.build.amd64-debian.sh lean $(GITHUB_ACTOR) $(GITHUB_HEAD_REF) 0 + bash packer.build.amd64-debian.sh \ + --pack lean \ + --github_user $(GITHUB_ACTOR) \ + --branch $(GITHUB_HEAD_REF) \ + --preseed_file preseed.cfg \ + --boot uefi\ + --desktop gnome # Compute the checksum of the qemu image cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu && \ @@ -22,10 +28,42 @@ amd64-lean-image: # List the generated files ls -lah ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu/raspiblitz-amd64-debian-lean.qcow2.* -amd64-fatpack-image: +amd64-lean-server-legacyboot-image: # Run the build script cd ci/amd64 && \ - bash packer.build.amd64-debian.sh fatpack $(GITHUB_ACTOR) $(GITHUB_HEAD_REF) + bash packer.build.amd64-debian.sh \ + --pack lean \ + --github_user $(GITHUB_ACTOR) \ + --branch $(GITHUB_HEAD_REF) \ + --preseed_file preseed.cfg \ + --boot bios-256k.bin \ + --desktop none + + # Compute the checksum of the qemu image + cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu && \ + sha256sum raspiblitz-amd64-debian-lean.qcow2 > raspiblitz-amd64-debian-lean.qcow2.sha256 + + # Compress the image + cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu && \ + gzip -v9 raspiblitz-amd64-debian-lean.qcow2 + + # Compute the checksum of the compressed image + cd ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu && \ + sha256sum raspiblitz-amd64-debian-lean.qcow2.gz > raspiblitz-amd64-debian-lean.qcow2.gz.sha256 + + # List the generated files + ls -lah ci/amd64/builds/raspiblitz-amd64-debian-lean-qemu/raspiblitz-amd64-debian-lean.qcow2.* + +amd64-fatpack-desktop-uefi-image: + # Run the build script + cd ci/amd64 && \ + bash packer.build.amd64-debian.sh \ + --pack fatpack \ + --github_user $(GITHUB_ACTOR) \ + --branch $(GITHUB_HEAD_REF) \ + --preseed_file preseed.cfg \ + --boot uefi\ + --desktop gnome # Compute the checksum of the qemu image cd ci/amd64/builds/raspiblitz-amd64-debian-fatpack-qemu && \ @@ -45,7 +83,10 @@ amd64-fatpack-image: arm64-rpi-lean-image: # Run the build script cd ci/arm64-rpi && \ - bash packer.build.arm64-rpi.local.sh lean $(GITHUB_ACTOR) $(GITHUB_HEAD_REF) + bash packer.build.arm64-rpi.local.sh \ + --pack lean \ + --github_user $(GITHUB_ACTOR) \ + --branch $(GITHUB_HEAD_REF) # Compute the checksum of the raw image cd ci/arm64-rpi/packer-builder-arm && \ @@ -65,7 +106,10 @@ arm64-rpi-lean-image: arm64-rpi-fatpack-image: # Run the build script cd ci/arm64-rpi && \ - bash packer.build.arm64-rpi.local.sh fatpack $(GITHUB_ACTOR) $(GITHUB_HEAD_REF) + bash packer.build.arm64-rpi.local.sh \ + --pack fatpack \ + --github_user $(GITHUB_ACTOR) \ + --branch $(GITHUB_HEAD_REF) # Compute the checksum of the raw image cd ci/arm64-rpi/packer-builder-arm && \ diff --git a/build_sdcard.sh b/build_sdcard.sh index fa02aadde..bcaa06b2f 100644 --- a/build_sdcard.sh +++ b/build_sdcard.sh @@ -291,9 +291,8 @@ HandleLidSwitchDocked=ignore" | tee /etc/systemd/logind.conf.d/nosuspend.conf # https://github.com/rootzoll/raspiblitz/issues/138 # https://daker.me/2014/10/how-to-fix-perl-warning-setting-locale-failed-in-raspbian.html # https://stackoverflow.com/questions/38188762/generate-all-locales-in-a-docker-image -if [ "${baseimage}" = "raspios_arm64" ] || [ "${baseimage}" = "debian" ]; then +if [ "${baseimage}" = "raspios_arm64" ] || [ "${cpu}" = aarch64 ] && [ "${baseimage}" = "debian" ]; then echo -e "\n*** FIXING LOCALES FOR BUILD ***" - sed -i "s/^# en_US.UTF-8 UTF-8.*/en_US.UTF-8 UTF-8/g" /etc/locale.gen sed -i "s/^# en_US ISO-8859-1.*/en_US ISO-8859-1/g" /etc/locale.gen locale-gen diff --git a/ci/README.md b/ci/README.md index f978fc2c1..fc50e5c2b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,29 +1,33 @@ # Automated builds -* The images are built using the dev branch. -* The lean image has no Gnome desktop or WebUI installed. -* Issue: https://github.com/rootzoll/raspiblitz/issues/3053 -* The templates are made using: https://github.com/chef/bento + +
+Table of Contents - [Local build](#local-build) - [Generate an arm64-rpi image](#generate-an-arm64-rpi-image) - [Generate an amd64 image](#generate-an-amd64-image) + - [amd64-lean-desktop-uefi-image](#amd64-lean-desktop-uefi-image) + - [amd64-lean-server-legacyboot-image](#amd64-lean-server-legacyboot-image) - [Images generated in github actions](#images-generated-in-github-actions) - [Write the image to a disk connected with USB](#write-the-image-to-a-disk-connected-with-usb) - [Convert the qcow2 volume to a raw disk image](#convert-the-qcow2-volume-to-a-raw-disk-image) - [Write to a disk connected with USB with Balena Etcher or `dd`](#write-to-a-disk-connected-with-usb-with-balena-etcher-or-dd) - - [Extend the partition on the new disk (optional)](#extend-the-partition-on-the-new-disk-optional) - [The first boot](#the-first-boot) - - [fatpack image](#fatpack-image) - - [lean image](#lean-image) - - [Add Gnome desktop (optional)](#add-gnome-desktop-optional) + - [Lean image with Gnome desktop (default image)](#lean-image-with-gnome-desktop-default-image) + - [Lean server image without Gnome desktop](#lean-server-image-without-gnome-desktop) + - [Add Gnome desktop to the server image (optional)](#add-gnome-desktop-to-the-server-image-optional) + - [Fatpack images](#fatpack-images) + - [Extend the root partition (optional - recommended)](#extend-the-root-partition-optional---recommended) - [Add wifi (optional)](#add-wifi-optional) - [Add wifi driver (optional)](#add-wifi-driver-optional) - [Workflow notes](#workflow-notes) - - [Packer .json settings:](#packer-json-settings) - [VNC](#vnc) + - [Packer settings](#packer-settings) - [Flashing](#flashing) +
+ ## Local build with the [Makefile](https://github.com/rootzoll/raspiblitz/blob/dev/Makefile) * needs ~20 GB free space @@ -33,7 +37,7 @@ with the [Makefile](https://github.com/rootzoll/raspiblitz/blob/dev/Makefile) * Preparation: ``` # change to a mountpoint with sufficient space (check with 'df -h') - cd /var/log + cd $HOME/ # switch to root sudo su # install git and make @@ -50,94 +54,121 @@ with the [Makefile](https://github.com/rootzoll/raspiblitz/blob/dev/Makefile) ``` make arm-rpi-lean-image ``` -* find the image and sh256 hashes in the `ci/arm64-rpi/packer-builder-arm` directory +* find the image and sha256 hashes in the `ci/arm64-rpi/packer-builder-arm` directory * the .img.gz file can be written to an SDcard directly with Balena Etcher ### Generate an amd64 image -The workflow locally and in github actions generates a .qcow2 format amd64 image. +* The workflow locally and in github actions generates a .qcow2 format amd64 image. +* When finished find the compressed .qcow2 image and sha256 hashes in the `ci/amd64/builds` directory + +#### amd64-lean-desktop-uefi-image +* lean image, Gnome desktop, UEFI boot +* Tested with + * written to disk and booted with UEFI + ``` + make amd64-lean-desktop-uefi-image + ``` + +#### amd64-lean-server-legacyboot-image +* lean image, no desktop (cli only), legacy boot for old computers * Tested with * libvirt / virsh / virt-manager (https://virt-manager.org/) - * written to disk and booted with legacy boot (non-UEFI) + * written to disk and booted with legacy boot (non-UEFI / CSM mode) ``` - make amd64-lean-image + amd64-lean-server-legacyboot-image ``` -* find the compressed .qcow2 image and sh256 hashes in the `ci/amd64/builds` directory ## Images generated in github actions * To see the downloadable artifacts will need to log in to GitHub -* Find the latest successful builds for amd64 using the dev branch at: +* Find the latest successful build of the default amd64 image: https://github.com/rootzoll/raspiblitz/actions/workflows/amd64-lean-image.yml?query=workflow%3Aamd64-lean-image-build+branch%3Adev+is%3Asuccess++ ``` # unzip to the same directory - unzip raspiblitz-amd64-image-YEAR-MM-DD-COMMITHASH.zip + unzip ./raspiblitz-amd64-image-YEAR-MM-DD-COMMITHASH.zip ``` ## Write the image to a disk connected with USB -* identify the connected disk with `lsblk` eg `/dev/sdd` - ### Convert the qcow2 volume to a raw disk image -* the raw image is 33.5 GB +* the raw image is 30GB ``` # unzip - gzip -dkv raspiblitz-amd64-debian-11.5-lean.qcow2.gz + gzip -dkv raspiblitz-amd64-debian-lean.qcow2.gz # convert - qemu-img convert raspiblitz-amd64-debian-11.5-lean.qcow2 raspiblitz-amd64-debian-11.5-lean.img + qemu-img convert ./raspiblitz-amd64-debian-lean.qcow2 ./raspiblitz-amd64-debian-lean.img ``` ### Write to a disk connected with USB with Balena Etcher or `dd` +* identify the connected disk with `lsblk` eg,: `/dev/sdk` * [Balena Etcher](https://www.balena.io/etcher/) to write the .img to disk * dd to write the .img to disk ``` - # identify partitions - lsblk - # write to disk - sudo dd if=./raspiblitz-amd64-debian-11.5-lean.img of=/dev/sde bs=4M status=progress + disk="/dev/sdk" + sudo dd if=./raspiblitz-amd64-debian-lean.img of=${disk} bs=4M status=progress ``` - * qemu-image dd to write the .qcow2 directly to disk ``` sudo apt install -y qemu-utils - sudo qemu-img dd if=./raspiblitz-amd64-debian-11.5-lean.qcow2 of=/dev/sde bs=4M - ``` -### Extend the partition on the new disk (optional) -* Use Disks to resize the Extended Partition to the full size of the disk -* To extend the LVM: - ``` - # identify the USB connected disk - lsblk - df -h - # extend the lvm to the full free space and resize the filesystem - sudo lvextend -r -l +100%FREE /dev/mapper/raspiblitz--amd74--debian--11--vg-root - - # alternatively download the script - git clone https://git.scs.carleton.ca/git/extend-lvm.git - # run with the disk as the parameter (sde for example) - sudo bash extend-lvm/extend-lvm.sh /dev/sde + disk="/dev/sdk" + sudo qemu-img dd if=./raspiblitz-amd64-debian-lean.qcow2 of=${disk} bs=4M ``` ## The first boot -### fatpack image +### Lean image with Gnome desktop (default image) * log in on screen: * username: `admin` * password: `raspiblitz` - * start a terminal for guidance +* connect with ssh over the LAN + * username: `admin` + * password: `raspiblitz` -* alternatively open a browser and go to: +### Lean server image without Gnome desktop +* press any key to get to a login prompt after the splash screen + * username: `admin` + * password: `raspiblitz` + +#### Add Gnome desktop to the server image (optional) +* Connect to the internet (easiest to plug in a LAN cable - use a USB - LAN adapter if have no port) + ``` + apt install gnome + systemctl start gdm + ``` + +### Fatpack images +* can open a browser and go to: * http://localhost * can also open the WebUI on another computer * Find the the RaspiBlitz_IP in your router dashboard, in the terminal prompt or with `hostname -I` * open: http://RaspiBlitz_IP -### lean image -* press any key to get to a login prompt after the splash screen - * username: `admin` - * password: `raspiblitz` - -#### Add Gnome desktop (optional) -* Connect to the internet (easiest to plug in a LAN cable - use a USB - LAN adapter if have no port) +### Extend the root partition (optional - recommended) +* The default image is 30GB. The partition can be extended to the full size of the disk. +* The lvm partition can be extended while mounted so this step can be done later as well while the system is running. +* CLI (recommended) ``` - apt install gnome - systemctl start gdm + # identify the USB connected disk + lsblk + df -h + # select the disk carefully + disk="/dev/sde" + # resize the extended partition to the full size of the disk + sudo parted ${disk} -- resizepart 2 100% + # resize the lvm partition to the full size of the disk + sudo parted ${disk} -- resizepart 5 100% + # extend the physical volume to size of the lvm partition + sudo pvresize ${disk}5 + # extend the root lvm to the full free space and resize the filesystem + sudo lvextend -r -l +100%FREE /dev/mapper/raspiblitz--amd64--vg-root + ``` +* GUI with GParted + ``` + # install + sudo apt install gparted + # start the gparted GUI + sudo gparted + # resize the extended partition to the full size of the disk + # extend the lvm to the full free space and resize the filesystem (extends the swap space by default) + # in CLI: extend the root lvm + sudo lvextend -r -l +100%FREE /dev/mapper/raspiblitz--amd64--vg-root ``` ### Add wifi (optional) @@ -153,9 +184,13 @@ https://github.com/rootzoll/raspiblitz/actions/workflows/amd64-lean-image.yml?qu ``` sudo apt update && sudo apt install firmware-iwlwifi ``` +* alternatively download the deb package from: http://ftp.debian.org/debian/pool/non-free-firmware/f/firmware-nonfree/firmware-iwlwifi_20230210-5_all.deb +* install with: + ``` + sudo dpkg -i firmware-iwlwifi_20230210-5_all.deb + ``` ## Workflow notes - The github workflow files are the equivalent of the Makefile commands run locally. The local repo owner (`GITHUB_ACTOR`) and branch (`GITHUB_HEAD_REF`) is picked up. The build_sdcard.sh is downloaded from the source branch and built with the options pack=[lean|fatpack] to set fatpack=[0|1]. @@ -176,8 +211,11 @@ After the image is built (and there is no exit with errors) the next steps are: * compute checksum of the compressed image * (in github actions: upload the artifacts in one .zip file) -### Packer .json settings: -* `disk_size` - the size op the raw image. The .qcow2 file is compressed. +### VNC +* can follow the setup locally in VNC with the port stated in the first part of the logs eg: `Found available VNC port: 5900 on IP: 127.0.0.1` + +### Packer settings +* `disk_size` / `image_size` - the size op the raw image. The .qcow2 file is compressed. * `template` - image filename * `output_directory` - directory under builds where the image will be placed * the `pi` user is given passwordless sudo access and used for the image setup @@ -197,7 +235,5 @@ After the image is built (and there is no exit with errors) the next steps are: cat 2022-09-22-raspios-bullseye-arm64.img.xz.sha256 ``` -### VNC -* can follow the setup locally in VNC with the port stated in the first part of the logs eg: `Found available VNC port: 5952 on IP: 127.0.0.1` ### Flashing -* using `qemu-img dd bs=4M if=raspiblitz-amd64-debian-11.5-lean.qcow2 of=/dev/sdd` changed the UUID so it won't boot without editing GRUB +* using `qemu-img dd bs=4M if=raspiblitz-amd64-debian-lean.qcow2 of=/dev/sdd` changed the UUID so it won't boot without editing GRUB diff --git a/ci/amd64/_common/env.sh b/ci/amd64/_common/env.sh new file mode 100644 index 000000000..4b9fe8626 --- /dev/null +++ b/ci/amd64/_common/env.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "# Display the environment variables" +env diff --git a/ci/amd64/debian/amd64-debian.json b/ci/amd64/debian/amd64-debian.json deleted file mode 100644 index 1e5a7a9af..000000000 --- a/ci/amd64/debian/amd64-debian.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "variables": { - "box_basename": "debian", - "build_directory": "../builds", - "build_timestamp": "{{isotime \"20060102150405\"}}", - "pack": "{{user `pack`}}", - "cpus": "4", - "disk_size": "30000", - "git_revision": "__unknown_git_revision__", - "guest_additions_url": "", - "headless": "false", - "http_directory": "{{template_dir}}/http", - "http_proxy": "{{env `http_proxy`}}", - "https_proxy": "{{env `https_proxy`}}", - "iso_checksum": "3b0e9718e3653435f20d8c2124de6d363a51a1fd7f911b9ca0c6db6b3d30d53e", - "iso_name": "debian-12.0.0-amd64-netinst.iso", - "memory": "4096", - "mirror": "http://cdimage.debian.org/cdimage/release", - "mirror_directory": "current/amd64/iso-cd", - "name": "debian", - "no_proxy": "{{env `no_proxy`}}", - "preseed_path": "debian-9/preseed.cfg", - "qemu_display": "none", - "qemu_bios": "bios-256k.bin", - "template": "raspiblitz-amd64-debian-{{user `pack`}}", - "boot_command": "install preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `preseed_path`}} debian-installer=en_US.UTF-8 auto locale=en_US.UTF-8 kbd-chooser/method=us keyboard-configuration/xkb-keymap=us netcfg/get_hostname={{ .Name }} netcfg/get_domain=vagrantup.com fb=false debconf/frontend=noninteractive console-setup/ask_detect=false console-keymaps-at/keymap=us grub-installer/bootdev=default ", - "version": "TIMESTAMP" -}, - "builders": [ - { - "boot_command": "{{user `boot_command`}}", - "boot_wait": "5s", - "cpus": "{{ user `cpus` }}", - "disk_size": "{{user `disk_size`}}", - "headless": "{{ user `headless` }}", - "http_directory": "{{user `http_directory`}}", - "iso_checksum": "{{user `iso_checksum`}}", - "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}", - "memory": "{{ user `memory` }}", - "output_directory": "{{ user `build_directory` }}/{{user `template`}}-qemu", - "shutdown_command": "echo 'raspiblitz' | sudo /sbin/shutdown -hP now", - "ssh_password": "raspiblitz", - "ssh_port": 22, - "ssh_timeout": "10000s", - "ssh_username": "pi", - "type": "qemu", - "format": "qcow2", - "vm_name": "{{ user `template` }}.qcow2", - "qemuargs": [ - [ "-m", "{{ user `memory` }}" ], - [ "-bios", "{{ user `qemu_bios` }}" ], - [ "-display", "{{ user `qemu_display` }}" ] - ] - } - ], - "provisioners": [ - { - "type": "shell", - "environment_vars": [ - "HOME_DIR=/home/pi", - "http_proxy={{user `http_proxy`}}", - "https_proxy={{user `https_proxy`}}", - "no_proxy={{user `no_proxy`}}", - "github_user={{user `github_user`}}", - "branch={{user `branch`}}", - "pack={{user `pack`}}" - ], - "execute_command": "echo 'raspiblitz' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'", - "expect_disconnect": true, - "scripts": [ - "{{template_dir}}/scripts/update.sh", - "{{template_dir}}/../_common/sshd.sh", - "{{template_dir}}/scripts/networking.sh", - "{{template_dir}}/scripts/sudoers.sh", - "{{template_dir}}/scripts/systemd.sh", - "{{template_dir}}/scripts/raspiblitz.sh", - "{{template_dir}}/scripts/cleanup.sh" - ] - } - ] -} diff --git a/ci/amd64/debian/build.amd64-debian.pkr.hcl b/ci/amd64/debian/build.amd64-debian.pkr.hcl new file mode 100644 index 000000000..0b051b583 --- /dev/null +++ b/ci/amd64/debian/build.amd64-debian.pkr.hcl @@ -0,0 +1,115 @@ +packer { + required_version = ">= 1.7.0, < 2.0.0" + + required_plugins { + qemu = { + source = "github.com/hashicorp/qemu" + version = ">= 1.0.0, < 2.0.0" + } + } +} + +variable "pack" { default = "lean" } +variable "github_user" { default = "raspiblitz" } +variable "branch" { default = "dev" } +variable "desktop" { default = "none" } + +variable "boot" { default = "uefi" } +variable "preseed_file" { default = "preseed.cfg" } +variable "hostname" { default = "raspiblitz-amd64" } + +variable "iso_name" { default = "debian-12.1.0-amd64-netinst.iso" } +variable "iso_checksum" { default = "9f181ae12b25840a508786b1756c6352a0e58484998669288c4eec2ab16b8559" } + +variable "disk_size" { default = "30000" } +variable "memory" { default = "4096" } +variable "cpus" { default = "4" } + +locals { + name_template = "${var.hostname}-debian-${var.pack}" + bios_file = var.boot == "uefi" ? "OVMF.fd" : "bios-256k.bin" + boot_command = var.boot == "uefi" ? [ + "c", + "linux /install.amd/vmlinuz ", + "auto=true ", + "url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/${var.preseed_file} ", + "hostname=${var.hostname} ", + "domain=${var.hostname}.local ", + "interface=auto ", + "vga=788 noprompt quiet --", + "initrd /install.amd/initrd.gz", + "boot" + ] : [ + "install ", + " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/${var.preseed_file} ", + "debian-installer=en_US.UTF-8 ", + "auto ", + "locale=en_US.UTF-8 ", + "kbd-chooser/method=us ", + "keyboard-configuration/xkb-keymap=us ", + "netcfg/get_hostname=${var.hostname} ", + "netcfg/get_domain=${var.hostname}.local ", + "fb=false ", + "debconf/frontend=noninteractive ", + "console-setup/ask_detect=false ", + "console-keymaps-at/keymap=us ", + "grub-installer/bootdev=default ", + "" + ] +} + +source "qemu" "debian" { + boot_command = local.boot_command + boot_wait = "5s" + cpus = var.cpus + disk_size = var.disk_size + http_directory = "./http" + iso_checksum = var.iso_checksum + iso_url = "http://cdimage.debian.org/cdimage/release/current/amd64/iso-cd/${var.iso_name}" + memory = var.memory + output_directory = "../builds/${local.name_template}-qemu" + shutdown_command = "echo 'raspiblitz' | sudo /sbin/shutdown -hP now" + ssh_password = "raspiblitz" + ssh_port = 22 + ssh_timeout = "10000s" + ssh_username = "pi" + format = "qcow2" + vm_name = "${local.name_template}.qcow2" + headless = false + vnc_bind_address = "127.0.0.1" + vnc_port_max = 5900 + vnc_port_min = 5900 + qemuargs = [ + ["-m", var.memory], + ["-bios", local.bios_file], + ["-display", "none"] + ] +} + +build { + description = "Can't use variables here yet!" + sources = ["source.qemu.debian"] + + provisioner "shell" { + environment_vars = [ + "HOME_DIR=/home/pi", + "github_user=${var.github_user}", + "branch=${var.branch}", + "pack=${var.pack}", + "desktop=${var.desktop}" + ] + + execute_command = "echo 'raspiblitz' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'" + expect_disconnect = true + scripts = [ + "./../_common/env.sh", + "./scripts/update.sh", + "./../_common/sshd.sh", + "./scripts/networking.sh", + "./scripts/sudoers.sh", + "./scripts/systemd.sh", + "./scripts/build.raspiblitz.sh", + "./scripts/cleanup.sh" + ] + } +} diff --git a/ci/amd64/debian/http/debian-9/preseed.cfg b/ci/amd64/debian/http/debian-9/preseed.cfg deleted file mode 100644 index 7ab0c3414..000000000 --- a/ci/amd64/debian/http/debian-9/preseed.cfg +++ /dev/null @@ -1,47 +0,0 @@ -choose-mirror-bin mirror/http/proxy string -d-i apt-setup/use_mirror boolean true -d-i base-installer/kernel/override-image string linux-server -d-i clock-setup/utc boolean true -d-i clock-setup/utc-auto boolean true -d-i finish-install/reboot_in_progress note -d-i grub-installer/only_debian boolean true -d-i grub-installer/with_other_os boolean true -d-i keymap select us -d-i mirror/country string manual -d-i mirror/http/directory string /debian -d-i mirror/http/hostname string httpredir.debian.org -d-i mirror/http/proxy string -d-i partman-auto-lvm/guided_size string max -d-i partman-auto/choose_recipe select atomic -d-i partman-auto/method string lvm -d-i partman-lvm/confirm boolean true -d-i partman-lvm/confirm_nooverwrite boolean true -d-i partman-lvm/device_remove_lvm boolean true -d-i partman/choose_partition select finish -d-i partman/confirm boolean true -d-i partman/confirm_nooverwrite boolean true -d-i partman/confirm_write_new_label boolean true -d-i passwd/root-login boolean false -d-i passwd/root-password-again password raspiblitz -d-i passwd/root-password password raspiblitz -d-i passwd/user-fullname string pi -d-i passwd/user-uid string 1000 -d-i passwd/user-password password raspiblitz -d-i passwd/user-password-again password raspiblitz -d-i passwd/username string pi -d-i pkgsel/include string sudo bzip2 acpid cryptsetup zlib1g-dev wget curl dkms fuse make nfs-common net-tools cifs-utils rsync -d-i pkgsel/install-language-support boolean false -d-i pkgsel/update-policy select none -d-i pkgsel/upgrade select full-upgrade -# Prevent packaged version of VirtualBox Guest Additions being installed: -d-i preseed/early_command string sed -i \ - '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ - /usr/lib/pre-pkgsel.d/20install-hwpackages -d-i time/zone string UTC -d-i user-setup/allow-password-weak boolean true -d-i user-setup/encrypt-home boolean false -d-i preseed/late_command string sed -i '/^deb cdrom:/s/^/#/' /target/etc/apt/sources.list -apt-cdrom-setup apt-setup/cdrom/set-first boolean false -apt-mirror-setup apt-setup/use_mirror boolean true -popularity-contest popularity-contest/participate boolean false -tasksel tasksel/first multiselect standard, ssh-server diff --git a/ci/amd64/debian/http/preseed.cfg b/ci/amd64/debian/http/preseed.cfg new file mode 100644 index 000000000..843680b74 --- /dev/null +++ b/ci/amd64/debian/http/preseed.cfg @@ -0,0 +1,72 @@ +# https://github.com/chef/bento/blob/main/packer_templates/http/debian/preseed.cfg +# https://www.debian.org/releases/stable/example-preseed.txt +# https://github.com/tylert/packer-build/blob/master/source/debian/12_bookworm/base-uefi.preseed +# variables: https://github.com/tylert/packer-build/blob/master/source/debian/12_bookworm/base-uefi.pkr.hcl + +# Locale Setup +d-i debian-installer/language string en +d-i debian-installer/country string US +d-i debian-installer/locale string en_US.UTF-8 +# d-i localechooser/supported-locales multiselect en_CA.UTF-8 fr_CA.UTF-8 zh_CN.UTF-8 +# d-i pkgsel/install-language-support boolean true + +# Keyboard Setup +d-i keyboard-configuration/xkb-keymap select us + +# Clock Setup +# d-i time/zone string Canada/Eastern +d-i time/zone string UTC +d-i clock-setup/utc boolean true +# set above to false if making a bootable USB to run on same system as Windows + +# Network Setup +d-i netcfg/get_hostname string raspiblitz-amd64 +d-i netcfg/get_domain string +# https://bugs.launchpad.net/ubuntu/+source/netcfg/+bug/713385 +d-i netcfg/choose_interface select auto +# make sure you also add "interface=auto" to your boot command too +# https://bugs.launchpad.net/ubuntu/+source/netcfg/+bug/713385 + +# User Setup +d-i passwd/root-login boolean false +d-i passwd/root-password-again password raspiblitz +d-i passwd/root-password password raspiblitz +d-i passwd/user-fullname string pi +d-i passwd/user-uid string 1000 +d-i passwd/user-password password raspiblitz +d-i passwd/user-password-again password raspiblitz +d-i passwd/username string pi + +# Package Setup +d-i hw-detect/load_firmware boolean false +d-i hw-detect/load_media boolean false +apt-cdrom-setup apt-setup/cdrom/set-first boolean false +d-i mirror/country string manual +d-i mirror/http/hostname string httpredir.debian.org +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string +d-i apt-setup/contrib boolean true +d-i apt-setup/non-free boolean true + +tasksel tasksel/first multiselect ssh-server, standard +d-i pkgsel/include string sudo bzip2 acpid cryptsetup zlib1g-dev wget curl dkms fuse make nfs-common net-tools cifs-utils rsync +d-i pkgsel/install-language-support boolean false +d-i pkgsel/update-policy select none +d-i pkgsel/upgrade select full-upgrade + +popularity-contest popularity-contest/participate boolean false + +# Drive setup +d-i partman-auto-lvm/guided_size string max +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/method string lvm +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true +d-i partman-lvm/device_remove_lvm boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true +d-i partman/confirm_write_new_label boolean true + +# Final Setup +d-i finish-install/reboot_in_progress note diff --git a/ci/amd64/debian/scripts/raspiblitz.sh b/ci/amd64/debian/scripts/build.raspiblitz.sh similarity index 92% rename from ci/amd64/debian/scripts/raspiblitz.sh rename to ci/amd64/debian/scripts/build.raspiblitz.sh index 4d6cc31e6..62a9993d2 100644 --- a/ci/amd64/debian/scripts/raspiblitz.sh +++ b/ci/amd64/debian/scripts/build.raspiblitz.sh @@ -9,7 +9,7 @@ else fatpack="0" fi -if [ "${fatpack}" = "1" ]; then +if [ "${desktop}" = "gnome" ]; then echo 'Add Gnome desktop' export DEBIAN_FRONTEND=none sudo apt install gnome -y diff --git a/ci/amd64/packer.build.amd64-debian.sh b/ci/amd64/packer.build.amd64-debian.sh index 6118ed5e3..163f1bb89 100644 --- a/ci/amd64/packer.build.amd64-debian.sh +++ b/ci/amd64/packer.build.amd64-debian.sh @@ -1,13 +1,17 @@ #!/bin/bash -e -# Install packer -curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - -sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -sudo apt-get update -echo -e "\nInstalling packer..." -sudo apt-get install -y packer +# install packer +if ! packer version 2>/dev/null; then + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - + sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + sudo apt-get update + echo -e "\nInstalling packer..." + sudo apt-get install -y packer +else + echo "# Packer is installed" +fi -# Install qemu +# install qemu echo -e "\nInstalling qemu..." sudo apt-get install -y qemu-system @@ -16,6 +20,10 @@ source ../set_variables.sh set_variables "$@" # Build the image -echo -e "\nBuilding image..." +echo -e "\nBuilding the image..." cd debian -PACKER_LOG=1 packer build ${vars} -only=qemu amd64-debian.json +packer init -upgrade . +command="PACKER_LOG=1 packer build ${vars} -only=qemu packer.build.amd64-debian.hcl" +echo "# Running: $command" +if [ ${#vars} -eq 0 ];then exit 1;fi +PACKER_LOG=1 packer build ${vars} -only=qemu.debian build.amd64-debian.pkr.hcl || exit 1 diff --git a/ci/arm64-rpi/arm64-rpi.pkr.hcl b/ci/arm64-rpi/build.arm64-rpi.pkr.hcl similarity index 98% rename from ci/arm64-rpi/arm64-rpi.pkr.hcl rename to ci/arm64-rpi/build.arm64-rpi.pkr.hcl index 0b5c1e366..142647ebf 100644 --- a/ci/arm64-rpi/arm64-rpi.pkr.hcl +++ b/ci/arm64-rpi/build.arm64-rpi.pkr.hcl @@ -56,7 +56,7 @@ build { "branch=${var.branch}", "pack=${var.pack}" ] - script = "./raspiblitz.sh" + script = "./build.raspiblitz.sh" } provisioner "shell" { diff --git a/ci/arm64-rpi/raspiblitz.sh b/ci/arm64-rpi/build.raspiblitz.sh similarity index 100% rename from ci/arm64-rpi/raspiblitz.sh rename to ci/arm64-rpi/build.raspiblitz.sh diff --git a/ci/arm64-rpi/packer.build.arm64-rpi.local.sh b/ci/arm64-rpi/packer.build.arm64-rpi.local.sh index 20c2477a8..8cea7b821 100644 --- a/ci/arm64-rpi/packer.build.arm64-rpi.local.sh +++ b/ci/arm64-rpi/packer.build.arm64-rpi.local.sh @@ -58,11 +58,12 @@ go mod download go build || exit 1 # set vars -source ../set_variables.sh -set_variables "$@" +source <(../set_variables.sh "$@") -cp ../arm64-rpi.pkr.hcl ./ -cp ../raspiblitz.sh ./ +cp ../build.arm64-rpi.pkr.hcl ./ +cp ../build.raspiblitz.sh ./ echo -e "\n# Build the image" -packer build ${vars} arm64-rpi.pkr.hcl +command="packer build ${vars} build.arm64-rpi.pkr.hcl" +echo "# Running: $command" +$command || exit 1 diff --git a/ci/arm64-rpi/packer.build.arm64-rpi.sh b/ci/arm64-rpi/packer.build.arm64-rpi.sh index 8e13bc148..5d4a6b857 100644 --- a/ci/arm64-rpi/packer.build.arm64-rpi.sh +++ b/ci/arm64-rpi/packer.build.arm64-rpi.sh @@ -1,12 +1,13 @@ #!/bin/bash -e # set vars -source ../set_variables.sh -set_variables "$@" +source <(../set_variables.sh "$@") -# Build the image in docker -echo -e "\nBuild Packer image..." +# build the image in docker +echo -e "\nBuild the image..." # from https://hub.docker.com/r/mkaczanowski/packer-builder-arm/tags -docker run --rm --privileged -v /dev:/dev -v ${PWD}:/build \ +command="docker run --rm --privileged -v /dev:/dev -v ${PWD}:/build \ mkaczanowski/packer-builder-arm@sha256:0ff8ce0cf33e37be6c351c8bcb2643835c7f3525b7f591808b91c04238d45695 \ - build ${vars} arm64-rpi.pkr.hcl + build ${vars} build.arm64-rpi.pkr.hcl" +echo "# Running: $command" +$command || exit 1 diff --git a/ci/set_variables.sh b/ci/set_variables.sh old mode 100644 new mode 100755 index 3e9e42372..c02a49fa2 --- a/ci/set_variables.sh +++ b/ci/set_variables.sh @@ -1,46 +1,71 @@ +#!/bin/bash + function set_variables() { - if [ $# -gt 0 ]; then - pack=$1 - fi - if [ $# -gt 1 ]; then - github_user=$2 - fi + declare -A params + while (("$#")); do + case "$1" in + --pack) + params[pack]="$2" + shift 2 + ;; + --github_user) + params[github_user]="$2" + shift 2 + ;; + --branch) + params[branch]="$2" + shift 2 + ;; + # arm64-rpi + --image_link) + params[image_link]="$2" + shift 2 + ;; + # arm64-rpi + --image_checksum) + params[image_checksum]="$2" + shift 2 + ;; + # amd64 + # preseed.cfg + --preseed_file) + params[preseed_file]="$2" + shift 2 + ;; + # amd64 + # uefi | bios + --boot) + params[boot]="$2" + shift 2 + ;; + # amd64 + # none | gnome + --desktop) + params[desktop]="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Error: Invalid argument" + exit 1 + ;; + esac + done - if [ $# -gt 2 ]; then - branch=$3 - fi - - if [ $# -gt 4 ]; then - image_link="$4" - image_checksum="$5" - fi - - # Initialize the variables string + # Reset the global vars string vars="" + # Iterate over all keys in the params array + for key in "${!params[@]}"; do + # If the value for this key is not empty, add it to vars + if [ -n "${params[$key]}" ]; then + vars="$vars -var $key=${params[$key]}" + fi + done - # Add the pack variable if it is defined - if [ -n "${pack}" ]; then - vars="$vars -var pack=${pack}" - fi + export vars - # Add the github_user variable if it is defined - if [ -n "${github_user}" ]; then - vars="$vars -var github_user=${github_user}" - fi - - # Add the branch variable if it is defined - if [ -n "${branch}" ]; then - vars="$vars -var branch=${branch}" - fi - - # Add the image_link variable if it is defined - if [ -n "${image_link}" ]; then - vars="$vars -var image_link=${image_link}" - fi - - # Add the image_checksum variable if it is defined - if [ -n "${image_checksum}" ]; then - vars="$vars -var image_checksum=${image_checksum}" - fi }