Merge remote-tracking branch 'origin/FastAPI' into FastAPI

.github/workflows/tests.yml

@@ -3,11 +3,11 @@ name: tests
 on: [push, pull_request]
 
 jobs:
-  build:
+  unit:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -15,22 +15,44 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        env: 
+          VIRTUAL_ENV: ./venv
+          PATH: ${{ env.VIRTUAL_ENV }}/bin:${{ env.PATH }}
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Test with pytest
-        env:
-          LNBITS_BACKEND_WALLET_CLASS: LNPayWallet
-          LNBITS_FORCE_HTTPS: 0
-          LNPAY_API_ENDPOINT: https://api.lnpay.co/v1/
-          LNPAY_API_KEY: sak_gG5pSFZhFgOLHm26a8hcWvXKt98yd
-          LNPAY_ADMIN_KEY: waka_HqWfOoNE0TPqmQHSYErbF4n9
-          LNPAY_INVOICE_KEY: waki_ZqFEbhrTyopuPlOZButZUw
-          LNPAY_READ_KEY: wakr_6IyTaNrvSeu3jbojSWt4ou6h
-        run: |
-          pip install pytest pytest-cov
-          pytest --cov=lnbits --cov-report=xml
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          file: ./coverage.xml
+          python -m venv ${{ env.VIRTUAL_ENV }}
+          ./venv/bin/python -m pip install --upgrade pip
+          ./venv/bin/pip install -r requirements.txt
+          ./venv/bin/pip install pytest pytest-asyncio requests trio mock
+      - name: Run tests
+        run: make test
+  # build:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       python-version: [3.7, 3.8]
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Set up Python ${{ matrix.python-version }}
+  #       uses: actions/setup-python@v1
+  #       with:
+  #         python-version: ${{ matrix.python-version }}
+  #     - name: Install dependencies
+  #       run: |
+  #         python -m pip install --upgrade pip
+  #         pip install -r requirements.txt
+  #     - name: Test with pytest
+  #       env:
+  #         LNBITS_BACKEND_WALLET_CLASS: LNPayWallet
+  #         LNBITS_FORCE_HTTPS: 0
+  #         LNPAY_API_ENDPOINT: https://api.lnpay.co/v1/
+  #         LNPAY_API_KEY: sak_gG5pSFZhFgOLHm26a8hcWvXKt98yd
+  #         LNPAY_ADMIN_KEY: waka_HqWfOoNE0TPqmQHSYErbF4n9
+  #         LNPAY_INVOICE_KEY: waki_ZqFEbhrTyopuPlOZButZUw
+  #         LNPAY_READ_KEY: wakr_6IyTaNrvSeu3jbojSWt4ou6h
+  #       run: |
+  #         pip install pytest pytest-cov
+  #         pytest --cov=lnbits --cov-report=xml
+  #     - name: Upload coverage to Codecov
+  #       uses: codecov/codecov-action@v1
+  #       with:
+  #         file: ./coverage.xml

.gitignore

@@ -15,6 +15,7 @@ __pycache__
 .webassets-cache
 htmlcov
 test-reports
+tests/data
 
 *.swo
 *.swp

Dockerfile

@@ -9,6 +9,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 # Install build deps
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends build-essential
+RUN python -m pip install --upgrade pip
 
 # Install runtime deps
 COPY requirements.txt /tmp/requirements.txt
@@ -18,7 +19,7 @@ RUN pip install -r /tmp/requirements.txt
 RUN pip install pylightning
 
 # Install LND specific deps
-RUN pip install lndgrpc purerpc
+RUN pip install lndgrpc
 
 # Production image
 FROM python:3.7-slim as lnbits
@@ -31,18 +32,10 @@ ENV VIRTUAL_ENV="/opt/venv"
 COPY --from=builder --chown=1000:1000 $VIRTUAL_ENV $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# Setup Quart
-ENV QUART_APP="lnbits.app:create_app()"
-ENV QUART_ENV="development"
-ENV QUART_DEBUG="true"
-
-# App
-ENV LNBITS_BIND="0.0.0.0:5000"
-
 # Copy in app source
 WORKDIR /app
 COPY --chown=1000:1000 lnbits /app/lnbits
 
 EXPOSE 5000
 
-CMD quart assets && quart migrate && hypercorn -k trio --bind $LNBITS_BIND 'lnbits.app:create_app()'
+CMD ["uvicorn", "lnbits.__main__:app", "--port", "5000", "--host", "0.0.0.0"]

Makefile

@@ -1,3 +1,5 @@
+.PHONY: test
+
 all: format check requirements.txt
 
 format: prettier black
@@ -26,3 +28,10 @@ Pipfile.lock: Pipfile
 
 requirements.txt: Pipfile.lock
 	cat Pipfile.lock | jq -r '.default | map_values(.version) | to_entries | map("\(.key)\(.value)") | join("\n")' > requirements.txt
+
+test:
+	rm -rf ./tests/data
+	mkdir -p ./tests/data
+	LNBITS_DATA_FOLDER="./tests/data" \
+	PYTHONUNBUFFERED=1 \
+	./venv/bin/pytest -s

docs/devs/development.md

@@ -10,3 +10,17 @@ For developers
 ==============
 
 Thanks for contributing :)
+
+
+Tests
+=====
+
+This project has unit tests that help prevent regressions. Before you can run the tests, you must install a few dependencies:
+```bash
+./venv/bin/pip install pytest pytest-asyncio requests trio mock
+```
+
+Then to run the tests:
+```bash
+make test
+```

lnbits/extensions/bleskomat/exchange_rates.py

@@ -65,15 +65,16 @@ async def fetch_fiat_exchange_rate(currency: str, provider: str):
     }
 
     url = exchange_rate_providers[provider]["api_url"]
-    for key in replacements.keys():
-        url = url.replace("{" + key + "}", replacements[key])
+    if url:
+        for key in replacements.keys():
+            url = url.replace("{" + key + "}", replacements[key])
+        async with httpx.AsyncClient() as client:
+            r = await client.get(url)
+            r.raise_for_status()
+            data = r.json()
+    else:
+        data = {}
 
     getter = exchange_rate_providers[provider]["getter"]
-
-    async with httpx.AsyncClient() as client:
-        r = await client.get(url)
-        r.raise_for_status()
-        data = r.json()
-        rate = float(getter(data, replacements))
-
+    rate = float(getter(data, replacements))
     return rate

lnbits/extensions/bleskomat/lnurl_api.py

@@ -121,8 +121,8 @@ async def api_bleskomat_lnurl(req: Request):
 
     except LnurlHttpError as e:
         return {"status": "ERROR", "reason": str(e)}
-    except Exception:
-        traceback.print_exc()
+    except Exception as e:
+        print(str(e))
         return {"status": "ERROR", "reason": "Unexpected error"}
 
     return {"status": "OK"}

lnbits/extensions/bleskomat/models.py

@@ -124,7 +124,8 @@ class BleskomatLnurl(BaseModel):
                     )
                 except (ValueError, PermissionError, PaymentFailure) as e:
                     raise LnurlValidationError("Failed to pay invoice: " + str(e))
-                except Exception:
+                except Exception as e:
+                    print(str(e))
                     raise LnurlValidationError("Unexpected error")
 
     async def use(self, conn) -> bool:

lnbits/wallets/lndrest.py

@@ -117,8 +117,9 @@ class LndRestWallet(Wallet):
         data = r.json()
         payment_hash = data["payment_hash"]
         checking_id = payment_hash
+        fee_msat = int(data["payment_route"]["total_fees_msat"])
         preimage = base64.b64decode(data["payment_preimage"]).hex()
-        return PaymentResponse(True, checking_id, 0, preimage, None)
+        return PaymentResponse(True, checking_id, fee_msat, preimage, None)
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
         checking_id = checking_id.replace("_", "/")

pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
-trio_mode = true
+filterwarnings =
+    ignore::pytest.PytestCacheWarning

tests/conftest.py

@@ -1,12 +1,28 @@
+import asyncio
 import pytest
-
+from httpx import AsyncClient
 from lnbits.app import create_app
+from lnbits.commands import migrate_databases
+from lnbits.settings import HOST, PORT
+import tests.mocks
 
+# use session scope to run once before and once after all tests
+@pytest.fixture(scope="session")
+def app():
+    # yield and pass the app to the test
+    app = create_app()
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(migrate_databases())
+    yield app
+    # get the current event loop and gracefully stop any running tasks
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(loop.shutdown_asyncgens())
+    loop.close()
 
 @pytest.fixture
-async def client():
-    app = create_app()
-    app.config["TESTING"] = True
-
-    async with app.test_client() as client:
-        yield client
+async def client(app):
+    client = AsyncClient(app=app, base_url=f'http://{HOST}:{PORT}')
+    # yield and pass the client to the test
+    yield client
+    # close the async client after the test has finished
+    await client.aclose()

tests/core/test_views.py

@@ -1,6 +0,0 @@
-import pytest
-
-
-async def test_homepage(client):
-    r = await client.get("/")
-    assert b"Add a new wallet" in await r.get_data()

tests/core/views/test_generic.py

@@ -0,0 +1,7 @@
+import pytest
+from tests.conftest import client
+
+@pytest.mark.asyncio
+async def test_core_views_generic(client):
+    response = await client.get("/")
+    assert response.status_code == 200

tests/extensions/bleskomat/conftest.py

@@ -0,0 +1,57 @@
+import json
+import pytest
+import secrets
+from lnbits.core.crud import create_account, create_wallet
+from lnbits.extensions.bleskomat.crud import create_bleskomat, create_bleskomat_lnurl
+from lnbits.extensions.bleskomat.models import CreateBleskomat
+from lnbits.extensions.bleskomat.helpers import generate_bleskomat_lnurl_secret, generate_bleskomat_lnurl_signature, prepare_lnurl_params, query_to_signing_payload
+from lnbits.extensions.bleskomat.exchange_rates import exchange_rate_providers
+
+exchange_rate_providers["dummy"] = {
+    "name": "dummy",
+    "domain": None,
+    "api_url": None,
+    "getter": lambda data, replacements: str(1e8),# 1 BTC = 100000000 sats
+}
+
+@pytest.fixture
+async def bleskomat():
+    user = await create_account()
+    wallet = await create_wallet(user_id=user.id, wallet_name="bleskomat_test")
+    data = CreateBleskomat(
+        name="Test Bleskomat",
+        fiat_currency="EUR",
+        exchange_rate_provider="dummy",
+        fee="0"
+    )
+    bleskomat = await create_bleskomat(data=data, wallet_id=wallet.id)
+    return bleskomat
+
+@pytest.fixture
+async def lnurl(bleskomat):
+    query = {
+        "tag": "withdrawRequest",
+        "nonce": secrets.token_hex(10),
+        "tag": "withdrawRequest",
+        "minWithdrawable": "50000",
+        "maxWithdrawable": "50000",
+        "defaultDescription": "test valid sig",
+    }
+    tag = query["tag"]
+    params = prepare_lnurl_params(tag, query)
+    payload = query_to_signing_payload(query)
+    signature = generate_bleskomat_lnurl_signature(
+        payload=payload,
+        api_key_secret=bleskomat.api_key_secret,
+        api_key_encoding=bleskomat.api_key_encoding
+    )
+    secret = generate_bleskomat_lnurl_secret(bleskomat.api_key_id, signature)
+    params = json.JSONEncoder().encode(params)
+    lnurl = await create_bleskomat_lnurl(
+        bleskomat=bleskomat, secret=secret, tag=tag, params=params, uses=1
+    )
+    return {
+        "bleskomat": bleskomat,
+        "lnurl": lnurl,
+        "secret": secret,
+    }

tests/extensions/bleskomat/test_lnurl_api.py

@@ -0,0 +1,120 @@
+import pytest
+import secrets
+from lnbits.core.crud import get_wallet
+from lnbits.settings import HOST, PORT
+from lnbits.extensions.bleskomat.crud import get_bleskomat_lnurl
+from lnbits.extensions.bleskomat.helpers import generate_bleskomat_lnurl_signature, query_to_signing_payload
+from tests.conftest import client
+from tests.helpers import credit_wallet
+from tests.extensions.bleskomat.conftest import bleskomat, lnurl
+from tests.mocks import WALLET
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_missing_secret(client):
+    response = await client.get("/bleskomat/u")
+    assert response.status_code == 200
+    assert response.json() == {"status": "ERROR", "reason": "Missing secret"}
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_invalid_secret(client):
+    response = await client.get("/bleskomat/u?k1=invalid-secret")
+    assert response.status_code == 200
+    assert response.json() == {"status": "ERROR", "reason": "Invalid secret"}
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_unknown_api_key(client):
+    query = {
+        "id": "does-not-exist",
+        "nonce": secrets.token_hex(10),
+        "tag": "withdrawRequest",
+        "minWithdrawable": "1",
+        "maxWithdrawable": "1",
+        "defaultDescription": "",
+        "f": "EUR",
+    }
+    payload = query_to_signing_payload(query)
+    signature = "xxx"# not checked, so doesn't matter
+    response = await client.get(f'/bleskomat/u?{payload}&signature={signature}')
+    assert response.status_code == 200
+    assert response.json() == {"status": "ERROR", "reason": "Unknown API key"}
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_invalid_signature(client, bleskomat):
+    query = {
+        "id": bleskomat.api_key_id,
+        "nonce": secrets.token_hex(10),
+        "tag": "withdrawRequest",
+        "minWithdrawable": "1",
+        "maxWithdrawable": "1",
+        "defaultDescription": "",
+        "f": "EUR",
+    }
+    payload = query_to_signing_payload(query)
+    signature = "invalid"
+    response = await client.get(f'/bleskomat/u?{payload}&signature={signature}')
+    assert response.status_code == 200
+    assert response.json() == {"status": "ERROR", "reason": "Invalid API key signature"}
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_valid_signature(client, bleskomat):
+    query = {
+        "id": bleskomat.api_key_id,
+        "nonce": secrets.token_hex(10),
+        "tag": "withdrawRequest",
+        "minWithdrawable": "1",
+        "maxWithdrawable": "1",
+        "defaultDescription": "test valid sig",
+        "f": "EUR",# tests use the dummy exchange rate provider
+    }
+    payload = query_to_signing_payload(query)
+    signature = generate_bleskomat_lnurl_signature(
+        payload=payload,
+        api_key_secret=bleskomat.api_key_secret,
+        api_key_encoding=bleskomat.api_key_encoding
+    )
+    response = await client.get(f'/bleskomat/u?{payload}&signature={signature}')
+    assert response.status_code == 200
+    data = response.json()
+    assert data["tag"] == "withdrawRequest"
+    assert data["minWithdrawable"] == 1000
+    assert data["maxWithdrawable"] == 1000
+    assert data["defaultDescription"] == "test valid sig"
+    assert data["callback"] == f'http://{HOST}:{PORT}/bleskomat/u'
+    k1 = data["k1"]
+    lnurl = await get_bleskomat_lnurl(secret=k1)
+    assert lnurl
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_action_insufficient_balance(client, lnurl):
+    bleskomat = lnurl["bleskomat"]
+    secret = lnurl["secret"]
+    pr = "lntb500n1pseq44upp5xqd38rgad72lnlh4gl339njlrsl3ykep82j6gj4g02dkule7k54qdqqcqzpgxqyz5vqsp5h0zgewuxdxcl2rnlumh6g520t4fr05rgudakpxm789xgjekha75s9qyyssq5vhwsy9knhfeqg0wn6hcnppwmum8fs3g3jxkgw45havgfl6evchjsz3s8e8kr6eyacz02szdhs7v5lg0m7wehd5rpf6yg8480cddjlqpae52xu"
+    WALLET.pay_invoice.reset_mock()
+    response = await client.get(f'/bleskomat/u?k1={secret}&pr={pr}')
+    assert response.status_code == 200
+    assert response.json() == {"status": "ERROR", "reason": "Failed to pay invoice: Insufficient balance."}
+    wallet = await get_wallet(bleskomat.wallet)
+    assert wallet.balance_msat == 0
+    bleskomat_lnurl = await get_bleskomat_lnurl(secret)
+    assert bleskomat_lnurl.has_uses_remaining() == True
+    WALLET.pay_invoice.assert_not_called()
+
+@pytest.mark.asyncio
+async def test_bleskomat_lnurl_api_action_success(client, lnurl):
+    bleskomat = lnurl["bleskomat"]
+    secret = lnurl["secret"]
+    pr = "lntb500n1pseq44upp5xqd38rgad72lnlh4gl339njlrsl3ykep82j6gj4g02dkule7k54qdqqcqzpgxqyz5vqsp5h0zgewuxdxcl2rnlumh6g520t4fr05rgudakpxm789xgjekha75s9qyyssq5vhwsy9knhfeqg0wn6hcnppwmum8fs3g3jxkgw45havgfl6evchjsz3s8e8kr6eyacz02szdhs7v5lg0m7wehd5rpf6yg8480cddjlqpae52xu"
+    await credit_wallet(
+        wallet_id=bleskomat.wallet,
+        amount=100000,
+    )
+    wallet = await get_wallet(bleskomat.wallet)
+    assert wallet.balance_msat == 100000
+    WALLET.pay_invoice.reset_mock()
+    response = await client.get(f'/bleskomat/u?k1={secret}&pr={pr}')
+    assert response.json() == {"status": "OK"}
+    wallet = await get_wallet(bleskomat.wallet)
+    assert wallet.balance_msat == 50000
+    bleskomat_lnurl = await get_bleskomat_lnurl(secret)
+    assert bleskomat_lnurl.has_uses_remaining() == False
+    WALLET.pay_invoice.assert_called_once_with(pr)

tests/helpers.py

@@ -0,0 +1,19 @@
+import hashlib
+import secrets
+from lnbits.core.crud import create_payment
+
+async def credit_wallet(wallet_id: str, amount: int):
+    preimage = secrets.token_hex(32)
+    m = hashlib.sha256()
+    m.update(f"{preimage}".encode())
+    payment_hash = m.hexdigest()
+    await create_payment(
+        wallet_id=wallet_id,
+        payment_request="",
+        payment_hash=payment_hash,
+        checking_id=payment_hash,
+        preimage=preimage,
+        memo="",
+        amount=amount,# msat
+        pending=False,# not pending, so it will increase the wallet's balance
+    )

tests/mocks.py

@@ -0,0 +1,36 @@
+from mock import AsyncMock
+from lnbits import bolt11
+from lnbits.wallets.base import (
+    StatusResponse,
+    InvoiceResponse,
+    PaymentResponse,
+    PaymentStatus,
+    Wallet,
+)
+from lnbits.settings import WALLET
+
+WALLET.status = AsyncMock(return_value=StatusResponse(
+    "",# no error
+    1000000,# msats
+))
+WALLET.create_invoice = AsyncMock(return_value=InvoiceResponse(
+    True,# ok
+    "6621aafbdd7709ca6eea6203f362d64bd7cb2911baa91311a176b3ecaf2274bd",# checking_id (i.e. payment_hash)
+    "lntb1u1psezhgspp5vcs6477awuyu5mh2vgplxckkf0tuk2g3h253xydpw6e7etezwj7sdqqcqzpgxqyz5vqsp5dxpw8zs77hw5pla4wz4mfujllyxtlpu443auur2uxqdrs8q2h56q9qyyssq65zk30ylmygvv5y4tuwalnf3ttnqjn57ef6rmcqg0s53akem560jh8ptemjcmytn3lrlatw4hv9smg88exv3v4f4lqnp96s0psdrhxsp6pp75q",# payment_request
+    "",# no error
+))
+def pay_invoice_side_effect(payment_request: str):
+    invoice = bolt11.decode(payment_request)
+    return PaymentResponse(
+        True,# ok
+        invoice.payment_hash,# checking_id (i.e. payment_hash)
+        0,# fee_msat
+        "",# no error
+    )
+WALLET.pay_invoice = AsyncMock(side_effect=pay_invoice_side_effect)
+WALLET.get_invoice_status = AsyncMock(return_value=PaymentStatus(
+    True,# paid
+))
+WALLET.get_payment_status = AsyncMock(return_value=PaymentStatus(
+    True,# paid
+))