ci: Remove dependence on travis, use it as fallback env

.travis.yml

@@ -40,22 +40,10 @@ stages:
   - extended-lint
 env:
   global:
-    - MAKEJOBS=-j3
-    - RUN_UNIT_TESTS=true
-    - RUN_FUNCTIONAL_TESTS=true
-    - RUN_FUZZ_TESTS=false
-    - DOCKER_NAME_TAG=ubuntu:18.04
-    - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
-    - CCACHE_SIZE=100M
-    - CCACHE_TEMPDIR=/tmp/.ccache-temp
-    - CCACHE_COMPRESS=1
-    - CCACHE_DIR=$HOME/.ccache
-    - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
-    - SDK_URL=https://bitcoincore.org/depends-sources/sdks
-    - WINEDEBUG=fixme-all
-    - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
+    - CI_RETRY_EXE="travis_retry"
     - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved."
 before_install:
+  - set -o errexit; source ./ci/test/00_setup_env.sh
   - set -o errexit; source ./ci/test/03_before_install.sh
 install:
   - set -o errexit; source ./ci/test/04_install.sh
@@ -71,7 +59,6 @@ script:
   - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
 after_script:
   - echo $TRAVIS_COMMIT_RANGE
-  - echo $TRAVIS_COMMIT_LOG
 jobs:
   include:
 

ci/README.md

@@ -3,6 +3,22 @@
 This directory contains scripts for each build step in each build stage.
 
 Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
-[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle).  Every script in here is named
+[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
 and numbered according to which stage and lifecycle step it belongs to.
 
+### Running a stage locally
+
+To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
+requires `docker` to be installed. To install all requirements on Ubuntu, run
+
+```
+sudo apt install docker.io ccache bash git
+```
+
+To run the test stage,
+
+```
+./ci/test_run_all.sh
+```
+
+Be aware that the tests will be build and run in-place, so please run at your own risk.

ci/retry/README.md

@@ -35,7 +35,7 @@ Help:
         -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
         -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
         -f, --fail="script +cmds"        Fail Script: run in case of final failure
-    
+
 ### Examples
 
 No problem:

ci/retry/retry

@@ -4,7 +4,7 @@ GETOPT_BIN=$IN_GETOPT_BIN
 GETOPT_BIN=${GETOPT_BIN:-getopt}
 
 __sleep_amount() {
-  if [ -n "$constant_sleep" ]; then 
+  if [ -n "$constant_sleep" ]; then
     sleep_time=$constant_sleep
   else
     #TODO: check for awk
@@ -159,5 +159,5 @@ EOF
   done
 
   retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
-  
+
 fi

ci/test/00_setup_env.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+echo "Setting default values in env"
+
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
+export MAKEJOBS=${MAKEJOBS:--j3}
+export HOST=${HOST:-x86_64-unknown-linux-gnu}
+export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
+export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
+export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
+export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
+export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID}
+export CCACHE_SIZE=${CCACHE_SIZE:-100M}
+export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
+export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}
+export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
+export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
+export WINEDEBUG=${WINEDEBUG:-fixme-all}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export GOAL=${GOAL:-install}
+export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
+export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
+export CI_RETRY_EXE=${CI_RETRY_EXE:retry}

ci/test/04_install.sh

@@ -9,29 +9,32 @@ export LC_ALL=C.UTF-8
 free -m -h
 echo "Number of CPUs (nproc): $(nproc)"
 
-travis_retry docker pull "$DOCKER_NAME_TAG"
+ccache echo "Creating ccache dir if it didn't already exist"
 
-export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets
-git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN}
-export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/
+${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
 
-mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
+if [ ! -d ${DIR_QA_ASSETS} ]; then
+  git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+fi
+export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+
+mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
 export ASAN_OPTIONS=""
-export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"
-export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${TRAVIS_BUILD_DIR}/sanitizer-output/tsan"
-export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
+export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
+export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan"
+export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
 env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"
 elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
   DOCKER_ADMIN="--cap-add SYS_PTRACE"
 fi
-DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
+DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
 
 DOCKER_EXEC () {
   docker exec $DOCKER_ID bash -c "cd $PWD && $*"
 }
 
-travis_retry DOCKER_EXEC apt-get update
-travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
+${CI_RETRY_EXE} DOCKER_EXEC apt-get update
+${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
 

ci/test/06_script_a.sh

@@ -6,11 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-TRAVIS_COMMIT_LOG=$(git log --format=fuller -1)
-export TRAVIS_COMMIT_LOG
-
-OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
-BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
+BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
 if [ -z "$NO_DEPENDS" ]; then
   DOCKER_EXEC ccache --max-size=$CCACHE_SIZE
 fi
@@ -23,7 +19,7 @@ else
 fi
 END_FOLD
 
-mkdir build
+mkdir -p build
 cd build || (echo "could not enter build directory"; exit 1)
 
 BEGIN_FOLD configure
@@ -41,10 +37,10 @@ DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOI
 END_FOLD
 
 set -o errtrace
-trap 'DOCKER_EXEC "cat ${TRAVIS_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR
+trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR
 
 BEGIN_FOLD build
 DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false )
 END_FOLD
 
-cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1)
+cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)

ci/test/06_script_b.sh

@@ -10,7 +10,7 @@ cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST";
 
 if [ "$RUN_UNIT_TESTS" = "true" ]; then
   BEGIN_FOLD unit-tests
-  DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
+  DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
   END_FOLD
 fi
 
@@ -26,4 +26,4 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   END_FOLD
 fi
 
-cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1)
+cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)

ci/test_run_all.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+echo "Setting default values in env"
+
+set -o errexit; source ./ci/test/00_setup_env.sh
+set -o errexit; source ./ci/test/03_before_install.sh
+set -o errexit; source ./ci/test/04_install.sh
+set -o errexit; source ./ci/test/05_before_script.sh
+set -o errexit; source ./ci/test/06_script_a.sh
+set -o errexit; source ./ci/test/06_script_b.sh