core-lightning/.github/workflows/ci.yaml

---
name: Continuous Integration
on: [push, pull_request]
jobs:
  build-and-test:
    name: Test config ${{ matrix.cfg }}
    env:
      DEVELOPER: 1
      VALGRIND: 0
      EXPERIMENTAL_FEATURES: 0
      COMPAT: 1
    strategy:
      fail-fast: false
      matrix:
        include:
          # CFG us just an enumeration so we can refer to the configurations more easily
          - {CFG:  1, TEST_CMD: "make", NO_PYTHON: 1}
          - {CFG:  2, TEST_CMD: "make check-source check-units installcheck", COPTFLAGS: "-O3"}
          - {CFG:  3, ARCH: 32, TEST_CMD: "make check-source check-units installcheck", DEVELOPER: 0}
          - {CFG:  4, TEST_CMD: "make check-source check-units installcheck", EXPERIMENTAL_FEATURES: 1}

          # All of the following will just run `make pytest`
          - {CFG:  5, COMPILER: clang}
          - {CFG:  6, COMPILER: gcc, TEST_CHECK_DBSTMTS: 1}
          - {CFG:  7, DEVELOPER: 0, COMPAT: 0, TEST_GROUP: 1, TEST_GROUP_COUNT: 2}
          - {CFG:  8, DEVELOPER: 0, COMPAT: 0, TEST_GROUP: 2, TEST_GROUP_COUNT: 2}

          # All of these are pretty slo since they run under valgrind
          - {CFG:  9, VALGRIND: 1, TEST_GROUP: 1,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 10, VALGRIND: 1, TEST_GROUP: 2,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 11, VALGRIND: 1, TEST_GROUP: 3,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 12, VALGRIND: 1, TEST_GROUP: 4,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 13, VALGRIND: 1, TEST_GROUP: 5,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 14, VALGRIND: 1, TEST_GROUP: 6,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 15, VALGRIND: 1, TEST_GROUP: 7,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 16, VALGRIND: 1, TEST_GROUP: 8,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 17, VALGRIND: 1, TEST_GROUP: 9,  TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}
          - {CFG: 18, VALGRIND: 1, TEST_GROUP: 10, TEST_GROUP_COUNT: 10, PYTEST_PAR: 3}

          # Various other configurations
          - {CFG: 19, NETWORK: liquid-regtest}
          - {CFG: 20, DB: postgres, PYTEST_PAR: 2}

          # The cross-compiled versions
          - {CFG: 21, ARCH: arm32v7, TARGET_HOST: arm-linux-gnueabihf}
          - {CFG: 22, ARCH: arm64v8, TARGET_HOST: aarch64-linux-gnu}
          - {CFG: 23, EXPERIMENTAL_FEATURES: 1}
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v2.0.0

      - name: Set up Python 3.6
        uses: actions/setup-python@v2
        with:
          python-version: 3.6

      - name: Install dependencies
        run: |
          export DEBIAN_FRONTEND=noninteractive
          export BITCOIN_VERSION=0.20.1
          export ELEMENTS_VERSION=0.18.1.8
          sudo useradd -ms /bin/bash tester
          sudo apt-get update -qq

          sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
          autoconf \
          automake \
          binfmt-support \
          build-essential \
          clang \
          cppcheck \
          docbook-xml \
          eatmydata \
          gcc-aarch64-linux-gnu \
          gcc-arm-linux-gnueabihf \
          gcc-arm-none-eabi \
          gettext \
          git \
          libc6-dev-arm64-cross \
          libc6-dev-armhf-cross \
          libgmp-dev \
          libpq-dev \
          libprotobuf-c-dev \
          libsqlite3-dev \
          libtool \
          libxml2-utils \
          locales \
          net-tools \
          postgresql \
          python-pkg-resources \
          python3 \
          python3-dev \
          python3-pip \
          python3-setuptools \
          qemu \
          qemu-system-arm \
          qemu-user-static \
          shellcheck \
          software-properties-common \
          sudo \
          tcl \
          unzip \
          valgrind \
          wget \
          xsltproc \
          zlib1g-dev

          echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
          sudo chmod 0440 /etc/sudoers.d/tester

          cd /tmp/ && \
          wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \
          wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \
          tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \
          tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \
          sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin && \
          sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin && \
          rm -rf \
          bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
          bitcoin-$BITCOIN_VERSION \
          elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
          elements-$ELEMENTS_VERSION

      - name: Build
        env:
          VALGRIND: ${{ matrix.VALGRIND }}
          DEVELOPER: ${{ matrix.DEVELOPER }}
          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
          COMPILER: ${{ matrix.COMPILER }}
          ARCH: ${{ matrix.ARCH }}
          COMPAT: ${{ matrix.COMPAT }}
          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
          NETWORK: ${{ matrix.NETWORK }}
          TEST_CMD: ${{ matrix.TEST_CMD }}
          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
          TEST_GROUP: ${{ matrix.TEST_GROUP }}
        run: |
          echo Running in $(pwd)
          export ARCH=${ARCH:-64}
          export BOLTDIR=lightning-rfc
          export CC=${COMPILER:-gcc}
          export COMPAT=${COMPAT:-1}
          export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
          export DEVELOPER=${DEVELOPER:-1}
          export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
          export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
          export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}"
          export PYTEST_PAR=${PYTEST_PAR:-10}
          export PYTEST_SENTRY_ALWAYS_REPORT=1
          export SLOW_MACHINE=1
          export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
          export TEST_DB_PROVIDER=${DB:-"sqlite3"}
          export TEST_NETWORK=${NETWORK:-"regtest"}
          export TIMEOUT=900
          export VALGRIND=${VALGRIND:-0}

          env

          pip3 install --user -U \
            -r requirements.txt \
            -r contrib/pyln-client/requirements.txt \
            -r contrib/pyln-proto/requirements.txt \
            -r contrib/pyln-testing/requirements.txt

          # Install utilities that aren't dependencies, but make
          # running tests easier/feasible on CI (and pytest which
          # keeps breaking the rerunfailures plugin).
          pip3 install --user -U \
            flaky \
            blinker \
            pytest-sentry \
            pytest-test-groups==1.0.3 \
            pytest-custom-exit-code==0.3.0 \
            mypy==0.790

          git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc

          git submodule init
          git submodule update
          (cd external/libwally-core/src/; git submodule init; git submodule update)

          ./configure CC="$CC"
          cat config.vars

          cat << 'EOF' > pytest.ini
          [pytest]
          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml
          markers =
              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
          EOF

          if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
          then
            export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
            export MAKE_HOST="$TARGET_HOST"
            export BUILD=x86_64-pc-linux-gnu
            export AR="$TARGET_HOST"-ar
            export AS="$TARGET_HOST"-as
            export CC="$TARGET_HOST"-gcc
            export CXX="$TARGET_HOST"-g++
            export LD="$TARGET_HOST"-ld
            export STRIP="$TARGET_HOST"-strip
            export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static

            wget -q https://zlib.net/zlib-1.2.11.tar.gz
            tar xf zlib-1.2.11.tar.gz
            cd zlib-1.2.11
            ./configure --prefix="$QEMU_LD_PREFIX"
            make
            sudo make install
            cd ..
            rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11

            wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
            unzip -q sqlite-src-3260000.zip
            cd sqlite-src-3260000
            automake --add-missing --force-missing --copy || true
            ./configure --disable-tcl \
              --enable-static \
              --disable-readline \
              --disable-threadsafe \
              --disable-load-extension \
              --host="$TARGET_HOST" \
              --prefix="$QEMU_LD_PREFIX"
            make
            sudo make install
            cd ..
            rm sqlite-src-3260000.zip
            rm -rf sqlite-src-3260000

            wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
            tar xf gmp-6.1.2.tar.xz
            cd gmp-6.1.2
            ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST"
            make
            sudo make install
            cd ..
            rm gmp-6.1.2.tar.xz
            rm -rf gmp-6.1.2

            ./configure CC="$TARGET_HOST-gcc" --enable-static

            make -j32 CC="$TARGET_HOST-gcc" > /dev/null
          else
            eatmydata make -j32
            eatmydata $TEST_CMD
          fi

      - name: Upload Unit Test Results
        if: always()
        uses: actions/upload-artifact@v2
        with:
          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
          path: report.xml