mirrors/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-20 14:05:23 +01:00
Code Issues Projects Releases Wiki Activity

Merge #17423: ci: Make ci system read-only on the git work tree

Browse source 
fa7523d3aa ci: Extend docs (MarcoFalke)
fa493ef088 ci: Make ci system read-only on the git work tree (MarcoFalke)
fab1333292 ci: Remove git from required packages on host (MarcoFalke)
fa00393bce ci: Make all filesystem operations inside docker (MarcoFalke)

Pull request description:

  Running the ci completely in a docker, without leaving any traces on the host system is not possible right now because the ccache and depends dir needs to be propagated back and picked up by the host for caching.

  Fixes #17372

ACKs for top commit:
  JeremyRubin:
    tested ACK fa7523d3aa

Tree-SHA512: 4bce1a0f883bcbdb34abf409bdbc80d420c5da2045d2f9c5536ac433f9e5b490f23df084546c8c049f688b487572bbfc4f9c4029e9e672f4d9279739d066ed2e
This commit is contained in:
MarcoFalke 2019-11-20 16:41:45 -05:00
commit 41162c0ba0
No known key found for this signature in database
GPG key ID: D2EA4850E7528B25
6 changed files with 39 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
ci/README.md
View file

@ -8,11 +8,21 @@ and numbered according to which stage and lifecycle step it belongs to.
### Running a stage locally
Be aware that the tests will be built and run in-place, so please run at your own risk.
If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory.
While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories,
such as the depends cache or ccache, are mounted as read-write into the docker container. While it should be fine to run
the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and
testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci
system in a virtual machine with a Linux operating system of your choice.
To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
requires `docker` to be installed. To install all requirements on Ubuntu, run
```
sudo apt install docker.io bash git
sudo apt install docker.io bash
```
To run the default test stage,
@ -26,6 +36,3 @@ To run the test stage with a specific configuration,
```
FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
```
Be aware that the tests will be build and run in-place, so please run at your own risk.
If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.

2
ci/test/00_setup_env.sh
View file

@ -44,7 +44,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
export WINEDEBUG=${WINEDEBUG:-fixme-all}
export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
export GOAL=${GOAL:-install}
export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH

26
ci/test/04_install.sh
View file

@ -33,12 +33,6 @@ fi
mkdir -p "${BASE_SCRATCH_DIR}"
mkdir -p "${CCACHE_DIR}"
if [ ! -d ${DIR_QA_ASSETS} ]; then
git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
fi
export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
export ASAN_OPTIONS="detect_stack_use_after_return=1"
export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan"
@ -54,7 +48,13 @@ if [ -z "$RUN_CI_ON_HOST" ]; then
echo "Creating $DOCKER_NAME_TAG container to run in"
${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
--mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \
--mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
--mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \
-w $BASE_BUILD_DIR \
--env-file /tmp/env \
$DOCKER_NAME_TAG)
DOCKER_EXEC () {
docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
@ -83,6 +83,18 @@ if [ "$TRAVIS_OS_NAME" != "osx" ]; then
${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
fi
if [ ! -d ${DIR_QA_ASSETS} ]; then
DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
fi
export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
if [ -z "$RUN_CI_ON_HOST" ]; then
echo "Create $BASE_BUILD_DIR"
DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR
fi
if [ "$USE_BUSY_BOX" = "true" ]; then
echo "Setup to use BusyBox utils"
DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/

4
ci/test/05_before_script.sh
View file

@ -13,13 +13,13 @@ else
DOCKER_EXEC echo \> \$HOME/.bitcoin
fi
mkdir -p depends/SDKs depends/sdk-sources
DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources
if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
fi
if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
DOCKER_EXEC tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
fi
if [[ $HOST = *-mingw32 ]]; then
DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)

8
ci/test/06_script_a.sh
View file

@ -19,7 +19,9 @@ else
fi
END_FOLD
# Create folder on host and docker, so that `cd` works
mkdir -p build
DOCKER_EXEC mkdir -p build
# Temporarily disable errexit, because Travis macOS fails without error message
set +o errexit
@ -27,10 +29,12 @@ cd build || (echo "could not enter build directory"; exit 1)
set -o errexit
BEGIN_FOLD configure
DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
END_FOLD
BEGIN_FOLD distdir
# Create folder on host and docker, so that `cd` works
mkdir -p "bitcoin-$HOST"
DOCKER_EXEC make distdir VERSION=$HOST
END_FOLD
@ -39,7 +43,7 @@ cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1)
set -o errexit
BEGIN_FOLD configure
DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
END_FOLD
set -o errtrace

4
ci/test/06_script_b.sh
View file

@ -48,7 +48,3 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then
DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
END_FOLD
fi
set +o errexit
cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)
set -o errexit