[CI] improve on github workflows (#1889)

* [CI] making a proper github CI workflow

.github/actions/prepare/action.yml

@@ -0,0 +1,67 @@
+name: prepare
+
+inputs:
+  python-version:
+    description: "Python Version"
+    required: true
+    default: "3.9"
+  poetry-version:
+    description: "Poetry Version"
+    default: "1.5.1"
+  node-version:
+    description: "Node Version"
+    default: "18.x"
+  npm:
+    description: "use npm"
+    default: false
+    type: boolean
+
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+        # cache poetry install via pip
+        cache: "pip"
+
+    - name: Set up Poetry ${{ inputs.poetry-version }}
+      uses: abatilo/actions-poetry@v2
+      with:
+        poetry-version: ${{ inputs.poetry-version }}
+
+    - name: Setup a local virtual environment (if no poetry.toml file)
+      shell: bash
+      run: |
+        poetry config virtualenvs.create true --local
+        poetry config virtualenvs.in-project true --local
+
+    - uses: actions/cache@v3
+      name: Define a cache for the virtual environment based on the dependencies lock file
+      with:
+        path: ./.venv
+        key: venv-${{ hashFiles('poetry.lock') }}
+
+    - name: Install the project dependencies
+      shell: bash
+      run: poetry install
+
+    - name: Use Node.js ${{ inputs.node-version }}
+      if: ${{ (inputs.npm == 'true') }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ inputs.node-version }}
+
+    - uses: actions/cache@v3
+      if: ${{ (inputs.npm == 'true') }}
+      name: Define a cache for the npm based on the dependencies lock file
+      with:
+        path: ./node_modules
+        key: npm-${{ hashFiles('package-lock.json') }}
+
+    - name: Install npm packages
+      if: ${{ (inputs.npm == 'true') }}
+      shell: bash
+      run: npm install

.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: LNbits CI
+on: [push, pull_request]
+
+jobs:
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+
+  tests:
+    needs: [ lint ]
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+        db-url: ["", "postgres://lnbits:lnbits@0.0.0.0:5432/lnbits"]
+    uses: ./.github/workflows/tests.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+
+  migrations:
+    needs: [ lint ]
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    uses: ./.github/workflows/tests.yml
+    with:
+      make: test-migration
+      db-name: migration
+
+  openapi:
+    needs: [ lint ]
+    uses: ./.github/workflows/make.yml
+    with:
+      make: openapi
+
+  # docker:
+  #   uses: ./.github/workflows/docker.yml
+
+  regtest:
+    needs: [ lint ]
+    uses: ./.github/workflows/regtest.yml
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+        backend-wallet-class: ["LndRestWallet", "LndWallet", "CoreLightningWallet", "CoreLightningRestWallet", "LNbitsWallet", "EclairWallet"]
+    with:
+      python-version: ${{ matrix.python-version }}
+      backend-wallet-class: ${{ matrix.backend-wallet-class }}

.github/workflows/docker.yml

@@ -0,0 +1,20 @@
+name: docker
+
+on:
+  workflow_call:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+            context: .
+            push: false
+            tags: lnbitsdocker/lnbits-legend:latest
+            cache-from: type=registry,ref=lnbitsdocker/lnbits-legend:latest
+            cache-to: type=inline

.github/workflows/lint.yml

@@ -0,0 +1,27 @@
+name: lint
+on:
+  workflow_call:
+
+jobs:
+  black:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: checkblack
+  ruff:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: checkruff
+  mypy:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: mypy
+  pyright:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: pyright
+      npm: true
+  prettier:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: prettier
+      npm: true

.github/workflows/linting.yml

@@ -1,36 +0,0 @@
-name: linter
-on: [push, pull_request]
-jobs:
-  checks:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-
-      - name: Install python packages
-        run: poetry install
-      - name: Check black
-        run: make checkblack
-      - name: Check ruff
-        run: make checkruff
-      - name: Check mypy
-        run: make mypy
-
-      - name: Install node packages
-        run: npm i
-      - name: Run tests
-        run: make pyright
-      - name: Check prettier
-        run: make checkprettier

.github/workflows/make.yml

@@ -0,0 +1,31 @@
+name: make
+
+on:
+  workflow_call:
+    inputs:
+      make:
+        description: "make command that is run"
+        required: true
+        type: string
+      npm:
+        description: "use npm install"
+        default: false
+        type: boolean
+
+jobs:
+  make:
+    name: ${{ inputs.make }} (${{ matrix.python-version }})
+    strategy:
+      matrix:
+        os-version: ["ubuntu-latest"]
+        python-version: ["3.9", "3.10"]
+        node-version: ["18.x"]
+    runs-on: ${{ matrix.os-version }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/prepare
+        with:
+          python-version: ${{ matrix.python-version }}
+          node-version: ${{ matrix.node-version }}
+          npm: ${{ inputs.npm }}
+      - run: make ${{ inputs.make }}

.github/workflows/migrations.yml

@@ -1,44 +0,0 @@
-name: migrations
-
-on: [pull_request]
-
-jobs:
-  sqlite-to-postgres:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:latest
-        env:
-          POSTGRES_USER: lnbits
-          POSTGRES_PASSWORD: lnbits
-          POSTGRES_DB: migration
-        ports:
-          # maps tcp port 5432 on service container to the host
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-          sudo apt install unzip
-      - name: Run migrations
-        run: |
-          make test-migration

.github/workflows/openapi.yml

@@ -1,27 +0,0 @@
-name: openapi
-
-on: [push, pull_request]
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        run: make openapi

.github/workflows/regtest.yml

@@ -1,266 +1,79 @@
 name: regtest
 
-on: [push, pull_request]
+on:
+  workflow_call:
+    inputs:
+      make:
+        default: test
+        type: string
+      python-version:
+        default: "3.9"
+        type: string
+      os-version:
+        default: "ubuntu-latest"
+        type: string
+      backend-wallet-class:
+        required: true
+        type: string
 
 jobs:
-  LndRestWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
+  regtest:
+    runs-on: ${{ inputs.os-version }}
+
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
+
+      - name: Set up Docker Buildx
+        if: ${{ inputs.backend-wallet-class == 'LNbitsWallet' }}
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push
+        if: ${{ inputs.backend-wallet-class == 'LNbitsWallet' }}
+        uses: docker/build-push-action@v4
         with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+            context: .
+            push: false
+            tags: lnbitsdocker/lnbits-legend:latest
+            cache-from: type=registry,ref=lnbitsdocker/lnbits-legend:latest
+            cache-to: type=inline
+
       - name: Setup Regtest
         run: |
-          docker build -t lnbitsdocker/lnbits-legend .
           git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
           cd docker
           chmod +x ./tests
           ./tests
           sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
+
+      - uses: ./.github/actions/prepare
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Create fake admin
+        if: ${{ inputs.backend-wallet-class == 'LNbitsWallet' }}
+        run: docker exec lnbits-legend-lnbits-1 poetry run python tools/create_fake_admin.py
+
+      - name: Run Tests
         env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: LndRestWallet
+          LNBITS_DATABASE_URL: ${{ inputs.db-url }}
+          LNBITS_BACKEND_WALLET_CLASS: ${{ inputs.backend-wallet-class }}
           LND_REST_ENDPOINT: https://localhost:8081/
           LND_REST_CERT: ./docker/data/lnd-3/tls.cert
           LND_REST_MACAROON: ./docker/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  LndWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: LndWallet
           LND_GRPC_ENDPOINT: localhost
           LND_GRPC_PORT: 10009
           LND_GRPC_CERT: docker/data/lnd-3/tls.cert
           LND_GRPC_MACAROON: docker/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  CoreLightningWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: CoreLightningWallet
           CORELIGHTNING_RPC: ./docker/data/clightning-1/regtest/lightning-rpc
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  CoreLightningRestWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: CoreLightningRestWallet
           CORELIGHTNING_REST_URL: https://localhost:3001
           CORELIGHTNING_REST_MACAROON: ./docker/data/clightning-2-rest/access.macaroon
           CORELIGHTNING_REST_CERT: ./docker/data/clightning-2-rest/certificate.pem
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  LNbitsWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-          docker exec lnbits-legend-lnbits-1 /bin/bash -c "poetry run python tools/create_fake_admin.py"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: LNbitsWallet
           LNBITS_ENDPOINT: http://localhost:5001
           LNBITS_KEY: "d08a3313322a4514af75d488bcc27eee"
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  EclairWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: EclairWallet
           ECLAIR_URL: http://127.0.0.1:8082
           ECLAIR_PASS: lnbits
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
+        run: make test-real-wallet
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/tests.yml

@@ -1,69 +1,56 @@
 name: tests
 
-on: [push, pull_request]
+on:
+  workflow_call:
+    inputs:
+      make:
+        default: test
+        type: string
+      python-version:
+        default: "3.9"
+        type: string
+      os-version:
+        default: "ubuntu-latest"
+        type: string
+      db-url:
+        default: ""
+        type: string
+      db-name:
+        default: "lnbits"
+        type: string
 
 jobs:
-  sqlite:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-        poetry-version: ["1.5.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        run: make test
-  postgres:
-    runs-on: ubuntu-latest
+  tests:
+    runs-on: ${{ inputs.os-version }}
+
     services:
       postgres:
         image: postgres:latest
         env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
+          POSTGRES_USER: lnbits
+          POSTGRES_PASSWORD: lnbits
+          POSTGRES_DB: ${{ inputs.db-name }}
         ports:
-          # maps tcp port 5432 on service container to the host
           - 5432:5432
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-        poetry-version: ["1.5.1"]
+
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
+
+      - uses: ./.github/actions/prepare
         with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
+          python-version: ${{ inputs.python-version }}
+
+      - name: Run Tests
         env:
-          LNBITS_DATABASE_URL: postgres://postgres:postgres@0.0.0.0:5432/postgres
-        run: make test
+          LNBITS_DATABASE_URL: ${{ inputs.db-url }}
+          LNBITS_BACKEND_WALLET_CLASS: FakeWallet
+        run: make ${{ inputs.make }}
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

Makefile

@@ -68,7 +68,6 @@ migration:
 
 openapi:
 	LNBITS_BACKEND_WALLET_CLASS="FakeWallet" \
-	FAKE_WALLET_SECRET="ToTheMoon1" \
 	LNBITS_DATA_FOLDER="./tests/data" \
 	PYTHONUNBUFFERED=1 \
 	HOST=0.0.0.0 \