Merge bitcoin/bitcoin#26976: ci: Cache package manager install step

fa486de212 ci: Cache package manager install step (MarcoFalke)

Pull request description:

  Use the local podman or docker image cache to skip the slow `apt` step

ACKs for top commit:
  jamesob:
    ACK fa486de212 ([`jamesob/ackr/26976.1.MarcoFalke.ci_cache_package_manager`](https://github.com/jamesob/bitcoin/tree/ackr/26976.1.MarcoFalke.ci_cache_package_manager))

Tree-SHA512: 3495346c6c862b63296d2691cc492bf52a0a99ee7fae798887c792609904546013eba788045cd508a5f669f2c52e3479c122c18a5275c87af38237a1b5c9da17

ci/test/01_base_install.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+CI_EXEC_ROOT () { bash -c "$*"; }
+export -f CI_EXEC_ROOT
+
+if [ -n "$DPKG_ADD_ARCH" ]; then
+  CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH"
+fi
+
+if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
+  ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release
+  ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$CI_BASE_PACKAGES" "$PACKAGES"
+elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
+  if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then
+    # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages.
+    # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated
+    # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools
+    # package.
+    # TODO: drop this once we can use newer images in GCE
+    CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc
+  fi
+  if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then
+    CI_EXEC_ROOT echo "${APPEND_APT_SOURCES_LIST}" \>\> /etc/apt/sources.list
+  fi
+  ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update
+  ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES"
+fi

ci/test/04_install.sh

@@ -33,7 +33,12 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
 
   # the name isn't important, so long as we use the same UID
   LOCAL_USER=nonroot
-  ${CI_RETRY_EXE} docker pull "$CI_IMAGE_NAME_TAG"
+  DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \
+      --file "${BASE_ROOT_DIR}/ci/test_imagefile" \
+      --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
+      --build-arg "FILE_ENV=${FILE_ENV}" \
+      --tag="${CONTAINER_NAME}" \
+      "${BASE_ROOT_DIR}"
 
   if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
     echo "Restart docker before run to stop and clear all containers started with --rm"
@@ -49,7 +54,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
                   -w $BASE_ROOT_DIR \
                   --env-file /tmp/env \
                   --name $CONTAINER_NAME \
-                  $CI_IMAGE_NAME_TAG)
+                  $CONTAINER_NAME)
   export CI_CONTAINER_ID
 
   # Create a non-root user inside the container which matches the local user.
@@ -63,6 +68,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
 else
   echo "Running on host system without docker wrapper"
+  "${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
 fi
 
 CI_EXEC () {
@@ -76,29 +82,6 @@ export -f CI_EXEC_ROOT
 
 CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
 
-if [ -n "$DPKG_ADD_ARCH" ]; then
-  CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH"
-fi
-
-if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
-  ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release
-  ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$CI_BASE_PACKAGES" "$PACKAGES"
-elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
-  if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then
-    # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages.
-    # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated
-    # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools
-    # package.
-    # TODO: drop this once we can use newer images in GCE
-    CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc
-  fi
-  if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then
-    CI_EXEC_ROOT echo "${APPEND_APT_SOURCES_LIST}" \>\> /etc/apt/sources.list
-  fi
-  ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update
-  ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES"
-fi
-
 if [ -n "$PIP_PACKAGES" ]; then
   if [ "$CI_OS_NAME" == "macos" ]; then
     sudo -H pip3 install --upgrade pip

ci/test_imagefile

@@ -0,0 +1,10 @@
+ARG CI_IMAGE_NAME_TAG
+FROM ${CI_IMAGE_NAME_TAG}
+
+ARG FILE_ENV
+ENV FILE_ENV=${FILE_ENV}
+
+COPY ./ci/retry/retry /usr/bin/retry
+COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/
+
+RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"]