diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 75b47af0ed3..3014714a44c 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -8,10 +8,18 @@ export LC_ALL=C.UTF-8 set -ex -# The root dir. +# The source root dir, usually from git, usually read-only. # The ci system copies this folder. -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR +BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_READ_ONLY_DIR +# The destination root dir inside the container. +if [ -z "${DANGER_RUN_CI_ON_HOST}" ] ; then + # This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR + export BASE_ROOT_DIR="/ci_container_base" +else + # This folder is equal to BASE_READ_ONLY_DIR and is read-write + export BASE_ROOT_DIR="${BASE_READ_ONLY_DIR}" +fi # The depends dir. # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 89006bf95ef..007bb4bedfd 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index a96ed30dd3c..1d39ef9c483 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 83213b28566..c9a496e6ab6 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -42,33 +42,35 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 "${BASE_SCRATCH_DIR}"/msan/llvm-project + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 /msan/llvm-project - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm + cmake -G Ninja -B /msan/clang_build/ \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S /msan/llvm-project/llvm - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes + ninja -C /msan/clang_build/ "$MAKEJOBS" + ninja -C /msan/clang_build/ install-runtimes - update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 - update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 - update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/llvm-symbolizer 100 + update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100 + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ - -DLIBCXX_ENABLE_DEBUG_MODE=ON \ - -DLIBCXX_ENABLE_ASSERTIONS=ON \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes + cmake -G Ninja -B /msan/cxx_build/ \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_ENABLE_DEBUG_MODE=ON \ + -DLIBCXX_ENABLE_ASSERTIONS=ON \ + -S /msan/llvm-project/runtimes - ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" + ninja -C /msan/cxx_build/ "$MAKEJOBS" fi if [[ "${RUN_TIDY}" == "true" ]]; then diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 7ecf00097ce..99e16bcb98e 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,10 +6,6 @@ export LC_ALL=C.UTF-8 -# Create folders that are mounted into the docker -mkdir -p "${CCACHE_DIR}" -mkdir -p "${PREVIOUS_RELEASES_DIR}" - export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" @@ -23,11 +19,11 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env echo "Creating $CI_IMAGE_NAME_TAG container to run in" DOCKER_BUILDKIT=1 docker build \ - --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ --build-arg "FILE_ENV=${FILE_ENV}" \ --tag="${CONTAINER_NAME}" \ - "${BASE_ROOT_DIR}" + "${BASE_READ_ONLY_DIR}" docker volume create "${CONTAINER_NAME}_ccache" || true docker volume create "${CONTAINER_NAME}_depends" || true docker volume create "${CONTAINER_NAME}_previous_releases" || true @@ -41,7 +37,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # shellcheck disable=SC2086 CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$BASE_READ_ONLY_DIR,dst=/ro_base,readonly \ --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ @@ -52,6 +48,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" + echo "Create missing folders" + mkdir -p "${CCACHE_DIR}" + mkdir -p "${PREVIOUS_RELEASES_DIR}" fi CI_EXEC () {