diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ccc22d7e..400b624bf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,7 @@ jobs: smoke-test: name: Smoke Test ${{ matrix.cfg }} runs-on: ubuntu-20.04 + timeout-minutes: 300 env: DEVELOPER: 1 VALGRIND: 0 @@ -98,15 +99,8 @@ jobs: proto-test: name: Protocol Test Config runs-on: ubuntu-20.04 + timeout-minutes: 300 needs: [smoke-test] - env: - DEVELOPER: 1 - EXPERIMENTAL_FEATURES: 1 - COMPAT: 0 - PYTEST_PAR: 2 - TEST_CMD: "make check-protos" - TEST_GROUP: 1 - TEST_GROUP_COUNT: 1 strategy: fail-fast: true matrix: @@ -116,27 +110,23 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2.0.0 - - - name: Setup Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Install dependencies + - name: Build and run run: | - bash -x .github/scripts/setup.sh - - - name: Build - env: - ARCH: ${{ matrix.arch }} - COMPILER: ${{ matrix.compiler }} - DB: ${{ matrix.db }} - NETWORK: ${{ matrix.network }} - TARGET_HOST: ${{ matrix.TARGET_HOST }} - VALGRIND: ${{ matrix.valgrind }} - run: | - bash -x .github/scripts/build.sh - + docker build -f contrib/docker/Dockerfile.ubuntu -t cln-ci-ubuntu . + docker run -e ARCH=${{ matrix.arch }} \ + -e COMPILER=${{ matrix.compiler }} \ + -e DB=${{ matrix.db }} \ + -e NETWORK=${{ matrix.network }} \ + -e TARGET_HOST=${{ matrix.TARGET_HOST }} \ + -e VALGRIND=${{ matrix.valgrind }} \ + -e DEVELOPER=1 \ + -e EXPERIMENTAL_FEATURES=1 \ + -e COMPA=0 \ + -e PYTEST_PAR=2 \ + -e TEST_CMD="make check-protos" \ + -e TEST_GROUP=1 \ + -e TEST_GROUP_COUNT=1 \ + cln-ci-ubuntu - name: Upload Unit Test Results if: always() uses: actions/upload-artifact@v2 diff --git a/contrib/docker/Dockerfile.ubuntu b/contrib/docker/Dockerfile.ubuntu new file mode 100644 index 000000000..35d1f930c --- /dev/null +++ b/contrib/docker/Dockerfile.ubuntu @@ -0,0 +1,36 @@ +FROM ubuntu:20.04 +LABEL mantainer="Vincenzo Palazzo vincenzopalazzodev@gmail.com" + +WORKDIR /work + +COPY . . + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANGUAGE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV TZ="Europe/London" + +RUN apt-get -qq update && \ + apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ + sudo \ + locales \ + tzdata + +RUN ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime + +RUN locale-gen en_US.UTF-8 && dpkg-reconfigure --frontend noninteractive tzdata + +COPY . . + +# install package for pytho cryptography lib +# https://cryptography.io/en/latest/installation/#debian-ubuntu + +RUN apt-get -qq update && \ + apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ + build-essential libssl-dev libffi-dev \ + python3-dev cargo + +RUN ls -la + +CMD ["./contrib/docker/scripts/entrypoint.sh"] diff --git a/contrib/docker/scripts/build.sh b/contrib/docker/scripts/build.sh new file mode 100755 index 000000000..05cece10f --- /dev/null +++ b/contrib/docker/scripts/build.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +echo "Running in $(pwd)" +export ARCH=${ARCH:-64} +export BOLTDIR=lightning-rfc +export CC=${COMPILER:-gcc} +export COMPAT=${COMPAT:-1} +export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0} +export DEVELOPER=${DEVELOPER:-1} +export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0} +export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH" +export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}" +export PYTEST_PAR=${PYTEST_PAR:-10} +export PYTEST_SENTRY_ALWAYS_REPORT=1 +export SLOW_MACHINE=1 +export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"} +export TEST_DB_PROVIDER=${TEST_DB_PROVIDER:-"sqlite3"} +export TEST_NETWORK=${NETWORK:-"regtest"} +export TIMEOUT=900 +export VALGRIND=${VALGRIND:-0} +export FUZZING=${FUZZING:-0} +export LIGHTNINGD_POSTGRES_NO_VACUUM=1 + +pip3 install --user poetry +poetry config virtualenvs.create false --local +poetry install + +git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc +git submodule update --init --recursive + +./configure CC="$CC" +cat config.vars + +cat << EOF > pytest.ini +[pytest] +addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 +markers = + slow_test: marks tests as slow (deselect with '-m "not slow_test"') +EOF + +if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ] +then + export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/ + export MAKE_HOST="$TARGET_HOST" + export BUILD=x86_64-pc-linux-gnu + export AR="$TARGET_HOST"-ar + export AS="$TARGET_HOST"-as + export CC="$TARGET_HOST"-gcc + export CXX="$TARGET_HOST"-g++ + export LD="$TARGET_HOST"-ld + export STRIP="$TARGET_HOST"-strip + export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static + + wget -q https://zlib.net/zlib-1.2.11.tar.gz + tar xf zlib-1.2.11.tar.gz + cd zlib-1.2.11 || exit 1 + ./configure --prefix="$QEMU_LD_PREFIX" + make + sudo make install + cd .. || exit 1 + rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11 + + wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip + unzip -q sqlite-src-3260000.zip + cd sqlite-src-3260000 || exit 1 + automake --add-missing --force-missing --copy || true + ./configure --disable-tcl \ + --enable-static \ + --disable-readline \ + --disable-threadsafe \ + --disable-load-extension \ + --host="$TARGET_HOST" \ + --prefix="$QEMU_LD_PREFIX" + make + sudo make install + cd .. || exit 1 + rm sqlite-src-3260000.zip + rm -rf sqlite-src-3260000 + + wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz + tar xf gmp-6.1.2.tar.xz + cd gmp-6.1.2 || exit 1 + ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST" + make + sudo make install + cd .. + rm gmp-6.1.2.tar.xz + rm -rf gmp-6.1.2 + + ./configure CC="$TARGET_HOST-gcc" --enable-static + + make -j32 CC="$TARGET_HOST-gcc" > /dev/null +else + eatmydata make -j32 + # shellcheck disable=SC2086 + eatmydata $TEST_CMD +fi diff --git a/contrib/docker/scripts/entrypoint.sh b/contrib/docker/scripts/entrypoint.sh new file mode 100755 index 000000000..83f2410c4 --- /dev/null +++ b/contrib/docker/scripts/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd contrib/docker/scripts && \ + ./setup.sh && ./build.sh diff --git a/contrib/docker/scripts/setup.sh b/contrib/docker/scripts/setup.sh new file mode 100755 index 000000000..208571e1d --- /dev/null +++ b/contrib/docker/scripts/setup.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +export BITCOIN_VERSION=0.20.1 +export ELEMENTS_VERSION=0.18.1.8 +export RUST_VERSION=nightly +export TZ="Europe/London" + +sudo useradd -ms /bin/bash tester + +sudo apt-get update -qq + +sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ + autoconf \ + automake \ + binfmt-support \ + build-essential \ + clang \ + cppcheck \ + docbook-xml \ + eatmydata \ + gcc-aarch64-linux-gnu \ + gcc-arm-linux-gnueabihf \ + gcc-arm-none-eabi \ + gettext \ + git \ + libc6-dev-arm64-cross \ + libc6-dev-armhf-cross \ + libgmp-dev \ + libpq-dev \ + libprotobuf-c-dev \ + libsqlite3-dev \ + libtool \ + libxml2-utils \ + locales \ + net-tools \ + postgresql \ + python-pkg-resources \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools \ + qemu \ + qemu-system-arm \ + qemu-user-static \ + shellcheck \ + software-properties-common \ + sudo \ + tcl \ + unzip \ + valgrind \ + wget \ + xsltproc \ + zlib1g-dev + + +echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester +sudo chmod 0440 /etc/sudoers.d/tester + +( + cd /tmp/ || exit 1 + wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 + wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 + tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 + tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 + sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin + sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin + rm -rf \ + bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \ + bitcoin-$BITCOIN_VERSION \ + elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \ + elements-$ELEMENTS_VERSION +) + +if [ "$RUST" == "1" ]; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ + -y --default-toolchain ${RUST_VERSION} +fi