core-lightning/.github/workflows/ci.yaml

---
name: Continuous Integration
on: [push, pull_request]
jobs:
  build-and-test:
    name: Test DEV=${{ matrix.DEVELOPER }} VAL=${{ matrix.VALGRIND }} EXP=${{ matrix.EXPERIMENTAL_FEATURES }}
    env:
      DEVELOPER: 1
      VALGRIND: 0
      EXPERIMENTAL_FEATURES: 0
      COMPAT: 1
    strategy:
      fail-fast: false
      matrix:
        include:
          - {TEST_CMD: "make", NO_PYTHON: 1}
          - {TEST_CMD: "make check-source check-units installcheck", COPTFLAGS: "-O3"}
          - {ARCH: 32, TEST_CMD: "make check-source check-units installcheck", DEVELOPER: 0}
          - {TEST_CMD: "make check-source check-units installcheck", EXPERIMENTAL_FEATURES: 1}

          # All of the following will just run `make pytest`
          - {COMPILER: clang}
          - {COMPILER: gcc, TEST_CHECK_DBSTMTS: 1}
          - {DEVELOPER: 0, COMPAT: 0, TEST_GROUP: 1, TEST_GROUP_COUNT: 2}
          - {DEVELOPER: 0, COMPAT: 0, TEST_GROUP: 2, TEST_GROUP_COUNT: 2}

          # All of these are pretty slo since they run under valgrind
          - {VALGRIND: 1, TEST_GROUP: 1,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 2,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 3,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 4,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 5,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 6,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 7,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 8,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 9,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {VALGRIND: 1, TEST_GROUP: 10, TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}

          # Various other configurations
          - {NETWORK: liquid-regtest}
          - {DB: postgres, PYTEST_PAR: 2}

          # The cross-compiled versions
          - {ARCH: arm32v7, TARGET_HOST: arm-linux-gnueabihf}
          - {ARCH: arm64v8, TARGET_HOST: aarch64-linux-gnu}
          - {EXPERIMENTAL_FEATURES: 1}
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v2.0.0
        
      - name: Set up Python 3.6
        uses: actions/setup-python@v2
        with:
          python-version: 3.6

      - name: Install dependencies
        run: |
          export DEBIAN_FRONTEND=noninteractive
          export BITCOIN_VERSION=0.20.1
          export ELEMENTS_VERSION=0.18.1.8
          sudo useradd -ms /bin/bash tester
          sudo apt-get update -qq

          sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
          autoconf \
          automake \
          binfmt-support \
          build-essential \
          clang \
          cppcheck \
          docbook-xml \
          eatmydata \
          gcc-aarch64-linux-gnu \
          gcc-arm-linux-gnueabihf \
          gcc-arm-none-eabi \
          gettext \
          git \
          libc6-dev-arm64-cross \
          libc6-dev-armhf-cross \
          libgmp-dev \
          libpq-dev \
          libprotobuf-c-dev \
          libsqlite3-dev \
          libtool \
          libxml2-utils \
          locales \
          net-tools \
          postgresql \
          python-pkg-resources \
          python3 \
          python3-dev \
          python3-pip \
          python3-setuptools \
          qemu \
          qemu-system-arm \
          qemu-user-static \
          shellcheck \
          software-properties-common \
          sudo \
          tcl \
          unzip \
          valgrind \
          wget \
          xsltproc \
          zlib1g-dev

          echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
          sudo chmod 0440 /etc/sudoers.d/tester

          cd /tmp/ && \
          wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \
          wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \
          tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \
          tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \
          sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin && \
          sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin && \
          rm -rf \
          bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
          bitcoin-$BITCOIN_VERSION \
          elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
          elements-$ELEMENTS_VERSION          

      - name: Build
        env:
          VALGRIND: ${{ matrix.VALGRIND }}
          DEVELOPER: ${{ matrix.DEVELOPER }}
          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
          COMPILER: ${{ matrix.COMPILER }}
          ARCH: ${{ matrix.ARCH }}
          COMPAT: ${{ matrix.COMPAT }}
          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
          NETWORK: ${{ matrix.NETWORK }}
          TEST_CMD: ${{ matrix.TEST_CMD }}
          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
          TEST_GROUP: ${{ matrix.TEST_GROUP }}
        run: |
          echo Running in $(pwd)
          export ARCH=${ARCH:-64}
          export BOLTDIR=lightning-rfc
          export CC=${COMPILER:-gcc}
          export COMPAT=${COMPAT:-1}
          export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
          export DEVELOPER=${DEVELOPER:-1}
          export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
          export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
          export PYTEST_OPTS="--maxfail=5 --reruns=2 --suppress-no-test-exit-code ${PYTEST_OPTS}"
          export PYTEST_PAR=${PYTEST_PAR:-10}
          export PYTEST_SENTRY_ALWAYS_REPORT=1
          export SLOW_MACHINE=1
          export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
          export TEST_DB_PROVIDER=${DB:-"sqlite3"}
          export TEST_NETWORK=${NETWORK:-"regtest"}
          export TIMEOUT=900
          export VALGRIND=${VALGRIND:-0}

          env

          pip3 install --user -U \
          -r requirements.txt \
          -r contrib/pyln-client/requirements.txt \
          -r contrib/pyln-proto/requirements.txt \
          -r contrib/pyln-testing/requirements.txt

          # Install utilities that aren't dependencies, but make running
          # tests easier/feasible on CI
          pip3 install --user \
          blinker \
          pytest-sentry \
          pytest-rerunfailures==9.1.1 \
          pytest-test-groups==1.0.3 \
          pytest-custom-exit-code==0.3.0

          git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc

          git submodule init
          git submodule update
          (cd external/libwally-core/src/; git submodule init; git submodule update)

          ./configure CC="$CC"
          cat config.vars

          if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
          then
          export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
          export MAKE_HOST="$TARGET_HOST"
          export BUILD=x86_64-pc-linux-gnu
          export AR="$TARGET_HOST"-ar
          export AS="$TARGET_HOST"-as
          export CC="$TARGET_HOST"-gcc
          export CXX="$TARGET_HOST"-g++
          export LD="$TARGET_HOST"-ld
          export STRIP="$TARGET_HOST"-strip
          export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static

          wget -q https://zlib.net/zlib-1.2.11.tar.gz \
          && tar xf zlib-1.2.11.tar.gz \
          && cd zlib-1.2.11 \
          && ./configure --prefix="$QEMU_LD_PREFIX" \
          && make \
          && sudo make install
          cd .. && rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11

          wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip \
          && unzip -q sqlite-src-3260000.zip \
          && cd sqlite-src-3260000 \
          && automake --add-missing --force-missing --copy || true \
          && ./configure --disable-tcl --enable-static --disable-readline --disable-threadsafe --disable-load-extension --host="$TARGET_HOST" --prefix="$QEMU_LD_PREFIX" \
          && make \
          && sudo make install
          cd .. && rm sqlite-src-3260000.zip && rm -rf sqlite-src-3260000

          wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
          && tar xf gmp-6.1.2.tar.xz \
          && cd gmp-6.1.2 \
          && ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST" \
          && make \
          && sudo make install
          cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2

          ./configure CC="$TARGET_HOST-gcc" --enable-static

          make -j32 CC="$TARGET_HOST-gcc" > /dev/null
          else
          eatmydata make -j32
          eatmydata $TEST_CMD
          fi