From e607ab7a3e00721662aeb583016cda9964cdc053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Fri, 19 Apr 2024 13:22:06 +0200 Subject: [PATCH] test: restructure tests (#2444) unit, api, wallets * only run test-api for migration --- .github/workflows/ci.yml | 32 +- .github/workflows/regtest.yml | 12 +- .github/workflows/tests.yml | 12 +- Makefile | 27 +- tests/{core => api}/__init__.py | 0 tests/{core/views => api}/test_admin_api.py | 0 tests/{core/views => api}/test_api.py | 397 +----------------- tests/{core/views => api}/test_generic.py | 0 tests/{core/views => api}/test_public_api.py | 0 tests/conftest.py | 16 - tests/helpers.py | 131 +----- tests/{core/views => regtest}/__init__.py | 0 tests/regtest/conftest.py | 17 + tests/regtest/helpers.py | 130 ++++++ tests/regtest/test_real_invoice.py | 390 +++++++++++++++++ .../test_x_node_api.py} | 4 +- tests/unit/__init__.py | 0 tests/{core => unit}/test_cache.py | 0 tests/{core => unit}/test_db.py | 0 tests/{core => unit}/test_db_fetch_page.py | 0 tests/{core => unit}/test_helpers_query.py | 0 21 files changed, 605 insertions(+), 563 deletions(-) rename tests/{core => api}/__init__.py (100%) rename tests/{core/views => api}/test_admin_api.py (100%) rename tests/{core/views => api}/test_api.py (55%) rename tests/{core/views => api}/test_generic.py (100%) rename tests/{core/views => api}/test_public_api.py (100%) rename tests/{core/views => regtest}/__init__.py (100%) create mode 100644 tests/regtest/conftest.py create mode 100644 tests/regtest/helpers.py create mode 100644 tests/regtest/test_real_invoice.py rename tests/{core/views/test_node_api.py => regtest/test_x_node_api.py} (99%) create mode 100644 tests/unit/__init__.py rename tests/{core => unit}/test_cache.py (100%) rename tests/{core => unit}/test_db.py (100%) rename tests/{core => unit}/test_db_fetch_page.py (100%) rename tests/{core => unit}/test_helpers_query.py (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 569f58de5..fe3c2c23a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: lint: uses: ./.github/workflows/lint.yml - tests: + test-api: needs: [ lint ] strategy: matrix: @@ -20,6 +20,35 @@ jobs: db-url: ["", "postgres://lnbits:lnbits@0.0.0.0:5432/lnbits"] uses: ./.github/workflows/tests.yml with: + custom-pytest: "poetry run pytest tests/api" + python-version: ${{ matrix.python-version }} + db-url: ${{ matrix.db-url }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + test-wallets: + needs: [ lint ] + strategy: + matrix: + python-version: ["3.9", "3.10"] + db-url: ["", "postgres://lnbits:lnbits@0.0.0.0:5432/lnbits"] + uses: ./.github/workflows/tests.yml + with: + custom-pytest: "poetry run pytest tests/wallets" + python-version: ${{ matrix.python-version }} + db-url: ${{ matrix.db-url }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + test-unit: + needs: [ lint ] + strategy: + matrix: + python-version: ["3.9", "3.10"] + db-url: ["", "postgres://lnbits:lnbits@0.0.0.0:5432/lnbits"] + uses: ./.github/workflows/tests.yml + with: + custom-pytest: "poetry run pytest tests/unit" python-version: ${{ matrix.python-version }} db-url: ${{ matrix.db-url }} secrets: @@ -48,6 +77,7 @@ jobs: python-version: ["3.9"] backend-wallet-class: ["LndRestWallet", "LndWallet", "CoreLightningWallet", "CoreLightningRestWallet", "LNbitsWallet", "EclairWallet"] with: + custom-pytest: "poetry run pytest tests/regtest" python-version: ${{ matrix.python-version }} backend-wallet-class: ${{ matrix.backend-wallet-class }} secrets: diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index 87deff7da..2c5bda798 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -3,8 +3,9 @@ name: regtest on: workflow_call: inputs: - make: - default: test + custom-pytest: + description: "Custom pytest arguments" + required: true type: string python-version: default: "3.9" @@ -76,15 +77,14 @@ jobs: LNBITS_KEY: "d08a3313322a4514af75d488bcc27eee" ECLAIR_URL: http://127.0.0.1:8082 ECLAIR_PASS: lnbits - LNBITS_DATA_FOLDER: "./tests/data" PYTHONUNBUFFERED: 1 DEBUG: true with: - verbose: false + verbose: true job-summary: true emoji: false - click-to-expand: false - custom-pytest: poetry run pytest + click-to-expand: true + custom-pytest: ${{ inputs.custom-pytest }} report-title: "regtest (${{ inputs.python-version }}, ${{ inputs.backend-wallet-class }}" - name: Upload coverage to Codecov diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 00c50ce78..45e0bf433 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,6 +3,10 @@ name: tests on: workflow_call: inputs: + custom-pytest: + description: "Custom pytest arguments" + required: true + type: string python-version: default: "3.9" type: string @@ -50,16 +54,14 @@ jobs: env: LNBITS_DATABASE_URL: ${{ inputs.db-url }} LNBITS_BACKEND_WALLET_CLASS: FakeWallet - FAKE_WALLET_SECRET: "ToTheMoon1" - LNBITS_DATA_FOLDER: "./tests/data" PYTHONUNBUFFERED: 1 DEBUG: true with: - verbose: false + verbose: true job-summary: true emoji: false - click-to-expand: false - custom-pytest: poetry run pytest + click-to-expand: true + custom-pytest: ${{ inputs.custom-pytest }} report-title: "test (${{ inputs.python-version }}, ${{ inputs.db-url }})" - name: Upload coverage to Codecov diff --git a/Makefile b/Makefile index bacee95c5..eff6642e5 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ format: prettier black ruff check: mypy pyright checkblack checkruff checkprettier checkbundle +test: test-unit test-wallets test-api test-regtest + prettier: poetry run ./node_modules/.bin/prettier --write lnbits @@ -36,23 +38,32 @@ checkeditorconfig: dev: poetry run lnbits --reload -test: +test-wallets: LNBITS_BACKEND_WALLET_CLASS="FakeWallet" \ - FAKE_WALLET_SECRET="ToTheMoon1" \ - LNBITS_DATA_FOLDER="./tests/data" \ PYTHONUNBUFFERED=1 \ DEBUG=true \ - poetry run pytest + poetry run pytest tests/wallets -test-real-wallet: - LNBITS_DATA_FOLDER="./tests/data" \ +test-unit: + LNBITS_BACKEND_WALLET_CLASS="FakeWallet" \ PYTHONUNBUFFERED=1 \ DEBUG=true \ - poetry run pytest + poetry run pytest tests/unit + +test-api: + LNBITS_BACKEND_WALLET_CLASS="FakeWallet" \ + PYTHONUNBUFFERED=1 \ + DEBUG=true \ + poetry run pytest tests/api + +test-regtest: + PYTHONUNBUFFERED=1 \ + DEBUG=true \ + poetry run pytest tests/regtest test-migration: LNBITS_ADMIN_UI=True \ - make test + make test-api HOST=0.0.0.0 \ PORT=5002 \ LNBITS_DATA_FOLDER="./tests/data" \ diff --git a/tests/core/__init__.py b/tests/api/__init__.py similarity index 100% rename from tests/core/__init__.py rename to tests/api/__init__.py diff --git a/tests/core/views/test_admin_api.py b/tests/api/test_admin_api.py similarity index 100% rename from tests/core/views/test_admin_api.py rename to tests/api/test_admin_api.py diff --git a/tests/core/views/test_api.py b/tests/api/test_api.py similarity index 55% rename from tests/core/views/test_api.py rename to tests/api/test_api.py index 7a1cda89c..a8e6a09a3 100644 --- a/tests/core/views/test_api.py +++ b/tests/api/test_api.py @@ -1,29 +1,16 @@ -import asyncio import hashlib import pytest from lnbits import bolt11 -from lnbits.core.crud import get_standalone_payment, update_payment_details from lnbits.core.models import CreateInvoice, Payment -from lnbits.core.services import fee_reserve_total -from lnbits.core.views.admin_api import api_auditor from lnbits.core.views.payment_api import api_payment from lnbits.settings import settings -from lnbits.wallets import get_funding_source -from ...helpers import ( - cancel_invoice, +from ..helpers import ( get_random_invoice_data, - get_real_invoice, - is_fake, - is_regtest, - pay_real_invoice, - settle_invoice, ) -funding_source = get_funding_source() - # create account POST /api/v1/account @pytest.mark.asyncio @@ -339,9 +326,6 @@ async def test_get_payments_paginated(client, adminkey_headers_from, fake_paymen @pytest.mark.asyncio -@pytest.mark.skipif( - is_regtest, reason="payments wont be confirmed rightaway in regtest" -) async def test_get_payments_history(client, adminkey_headers_from, fake_payments): fake_data, filters = fake_payments @@ -406,11 +390,6 @@ async def test_api_payment_with_key(invoice, inkey_headers_from): # check POST /api/v1/payments: invoice creation with a description hash -@pytest.mark.skipif( - funding_source.__class__.__name__ - in ["CoreLightningWallet", "CoreLightningRestWallet"], - reason="wallet does not support description_hash", -) @pytest.mark.asyncio async def test_create_invoice_with_description_hash(client, inkey_headers_to): data = await get_random_invoice_data() @@ -428,10 +407,6 @@ async def test_create_invoice_with_description_hash(client, inkey_headers_to): return invoice -@pytest.mark.skipif( - funding_source.__class__.__name__ in ["CoreLightningRestWallet"], - reason="wallet does not support unhashed_description", -) @pytest.mark.asyncio async def test_create_invoice_with_unhashed_description(client, inkey_headers_to): data = await get_random_invoice_data() @@ -507,373 +482,3 @@ async def test_fiat_tracking(client, adminkey_headers_from): assert payment["extra"]["wallet_fiat_currency"] == "EUR" assert payment["extra"]["wallet_fiat_amount"] != payment["amount"] assert payment["extra"]["wallet_fiat_rate"] - - -async def get_node_balance_sats(): - audit = await api_auditor() - return audit["node_balance_msats"] / 1000 - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_pay_real_invoice( - client, real_invoice, adminkey_headers_from, inkey_headers_from, from_wallet_ws -): - prev_balance = await get_node_balance_sats() - response = await client.post( - "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from - ) - assert response.status_code < 300 - invoice = response.json() - assert len(invoice["payment_hash"]) == 64 - assert len(invoice["checking_id"]) > 0 - - data = from_wallet_ws.receive_json() - assert "wallet_balance" in data - payment = Payment(**data["payment"]) - assert payment.payment_hash == invoice["payment_hash"] - - # check the payment status - response = await client.get( - f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from - ) - assert response.status_code < 300 - payment_status = response.json() - assert payment_status["paid"] - - funding_source = get_funding_source() - status = await funding_source.get_payment_status(invoice["payment_hash"]) - assert status.paid - - await asyncio.sleep(1) - balance = await get_node_balance_sats() - assert prev_balance - balance == 100 - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_from): - prev_balance = await get_node_balance_sats() - create_invoice = CreateInvoice(out=False, amount=1000, memo="test") - response = await client.post( - "/api/v1/payments", - json=create_invoice.dict(), - headers=adminkey_headers_from, - ) - assert response.status_code < 300 - invoice = response.json() - - response = await client.get( - f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from - ) - assert response.status_code < 300 - payment_status = response.json() - assert not payment_status["paid"] - - async def listen(): - found_checking_id = False - async for checking_id in get_funding_source().paid_invoices_stream(): - if checking_id == invoice["checking_id"]: - found_checking_id = True - return - assert found_checking_id - - task = asyncio.create_task(listen()) - await asyncio.sleep(1) - pay_real_invoice(invoice["payment_request"]) - await asyncio.wait_for(task, timeout=10) - - response = await client.get( - f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from - ) - assert response.status_code < 300 - payment_status = response.json() - assert payment_status["paid"] - - await asyncio.sleep(1) - balance = await get_node_balance_sats() - assert balance - prev_balance == create_invoice.amount - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_pay_real_invoice_set_pending_and_check_state( - client, real_invoice, adminkey_headers_from, inkey_headers_from -): - """ - 1. We create an invoice - 2. We pay it - 3. We verify that the inoice was paid - 4. We set the invoice to pending in the database - 5. We recheck the state of the invoice - 6. We verify that the invoice is paid - """ - response = await client.post( - "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from - ) - assert response.status_code < 300 - invoice = response.json() - assert len(invoice["payment_hash"]) == 64 - assert len(invoice["checking_id"]) > 0 - - # check the payment status - response = await api_payment( - invoice["payment_hash"], inkey_headers_from["X-Api-Key"] - ) - assert response["paid"] - - # make sure that the backend also thinks it's paid - funding_source = get_funding_source() - status = await funding_source.get_payment_status(invoice["payment_hash"]) - assert status.paid - - # get the outgoing payment from the db - payment = await get_standalone_payment(invoice["payment_hash"]) - assert payment - assert payment.pending is False - - # set the outgoing invoice to pending - await update_payment_details(payment.checking_id, pending=True) - - payment_pending = await get_standalone_payment(invoice["payment_hash"]) - assert payment_pending - assert payment_pending.pending is True - - # check the outgoing payment status - await payment.check_status() - - payment_not_pending = await get_standalone_payment(invoice["payment_hash"]) - assert payment_not_pending - assert payment_not_pending.pending is False - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_pay_hold_invoice_check_pending( - client, hold_invoice, adminkey_headers_from -): - preimage, invoice = hold_invoice - task = asyncio.create_task( - client.post( - "/api/v1/payments", - json={"bolt11": invoice["payment_request"]}, - headers=adminkey_headers_from, - ) - ) - await asyncio.sleep(1) - - # get payment hash from the invoice - invoice_obj = bolt11.decode(invoice["payment_request"]) - - payment_db = await get_standalone_payment(invoice_obj.payment_hash) - - assert payment_db - assert payment_db.pending is True - - settle_invoice(preimage) - - response = await task - assert response.status_code < 300 - - # check if paid - - await asyncio.sleep(1) - - payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash) - - assert payment_db_after_settlement - assert payment_db_after_settlement.pending is False - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_pay_hold_invoice_check_pending_and_fail( - client, hold_invoice, adminkey_headers_from -): - preimage, invoice = hold_invoice - task = asyncio.create_task( - client.post( - "/api/v1/payments", - json={"bolt11": invoice["payment_request"]}, - headers=adminkey_headers_from, - ) - ) - await asyncio.sleep(1) - - # get payment hash from the invoice - invoice_obj = bolt11.decode(invoice["payment_request"]) - - payment_db = await get_standalone_payment(invoice_obj.payment_hash) - - assert payment_db - assert payment_db.pending is True - - preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest() - - # cancel the hodl invoice - assert preimage_hash == invoice_obj.payment_hash - cancel_invoice(preimage_hash) - - response = await task - assert response.status_code > 300 # should error - - await asyncio.sleep(1) - - # payment should not be in database anymore - payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash) - assert payment_db_after_settlement is None - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_pay_hold_invoice_check_pending_and_fail_cancel_payment_task_in_meantime( - client, hold_invoice, adminkey_headers_from -): - preimage, invoice = hold_invoice - task = asyncio.create_task( - client.post( - "/api/v1/payments", - json={"bolt11": invoice["payment_request"]}, - headers=adminkey_headers_from, - ) - ) - await asyncio.sleep(1) - - # get payment hash from the invoice - invoice_obj = bolt11.decode(invoice["payment_request"]) - - payment_db = await get_standalone_payment(invoice_obj.payment_hash) - - assert payment_db - assert payment_db.pending is True - - # cancel payment task, this simulates the client dropping the connection - task.cancel() - - preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest() - - assert preimage_hash == invoice_obj.payment_hash - cancel_invoice(preimage_hash) - - # check if paid - await asyncio.sleep(1) - - # payment should still be in db - payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash) - assert payment_db_after_settlement is not None - - # status should still be available and be False - status = await payment_db.check_status() - assert not status.paid - - # now the payment should be gone after the status check - # payment_db_after_status_check = await get_standalone_payment( - # invoice_obj.payment_hash - # ) - # assert payment_db_after_status_check is None - - -@pytest.mark.asyncio -@pytest.mark.skipif(is_fake, reason="this only works in regtest") -async def test_receive_real_invoice_set_pending_and_check_state( - client, adminkey_headers_from, inkey_headers_from -): - """ - 1. We create a real invoice - 2. We pay it from our wallet - 3. We check that the inoice was paid with the backend - 4. We set the invoice to pending in the database - 5. We recheck the state of the invoice with the backend - 6. We verify that the invoice is now marked as paid in the database - """ - create_invoice = CreateInvoice(out=False, amount=1000, memo="test") - response = await client.post( - "/api/v1/payments", - json=create_invoice.dict(), - headers=adminkey_headers_from, - ) - assert response.status_code < 300 - invoice = response.json() - response = await api_payment( - invoice["payment_hash"], inkey_headers_from["X-Api-Key"] - ) - assert not response["paid"] - - async def listen(): - found_checking_id = False - async for checking_id in get_funding_source().paid_invoices_stream(): - if checking_id == invoice["checking_id"]: - found_checking_id = True - return - assert found_checking_id - - task = asyncio.create_task(listen()) - await asyncio.sleep(1) - pay_real_invoice(invoice["payment_request"]) - await asyncio.wait_for(task, timeout=10) - response = await api_payment( - invoice["payment_hash"], inkey_headers_from["X-Api-Key"] - ) - assert response["paid"] - - # get the incoming payment from the db - payment = await get_standalone_payment(invoice["payment_hash"], incoming=True) - assert payment - assert payment.pending is False - - # set the incoming invoice to pending - await update_payment_details(payment.checking_id, pending=True) - - payment_pending = await get_standalone_payment( - invoice["payment_hash"], incoming=True - ) - assert payment_pending - assert payment_pending.pending is True - - # check the incoming payment status - await payment.check_status() - - payment_not_pending = await get_standalone_payment( - invoice["payment_hash"], incoming=True - ) - assert payment_not_pending - assert payment_not_pending.pending is False - - # verify we get the same result if we use the checking_id to look up the payment - payment_by_checking_id = await get_standalone_payment( - payment_not_pending.checking_id, incoming=True - ) - - assert payment_by_checking_id - assert payment_by_checking_id.pending is False - assert payment_by_checking_id.bolt11 == payment_not_pending.bolt11 - assert payment_by_checking_id.payment_hash == payment_not_pending.payment_hash - - -@pytest.mark.asyncio -async def test_check_fee_reserve(client, adminkey_headers_from): - # if regtest, create a real invoice, otherwise create an internal invoice - # call /api/v1/payments/fee-reserve?invoice=... with it and check if the fee reserve - # is correct - payment_request = "" - if is_regtest: - real_invoice = get_real_invoice(1000) - payment_request = real_invoice["payment_request"] - - else: - create_invoice = CreateInvoice(out=False, amount=1000, memo="test") - response = await client.post( - "/api/v1/payments", - json=create_invoice.dict(), - headers=adminkey_headers_from, - ) - assert response.status_code < 300 - invoice = response.json() - payment_request = invoice["payment_request"] - - response = await client.get( - f"/api/v1/payments/fee-reserve?invoice={payment_request}", - ) - assert response.status_code < 300 - fee_reserve = response.json() - assert fee_reserve["fee_reserve"] == fee_reserve_total(1000_000) diff --git a/tests/core/views/test_generic.py b/tests/api/test_generic.py similarity index 100% rename from tests/core/views/test_generic.py rename to tests/api/test_generic.py diff --git a/tests/core/views/test_public_api.py b/tests/api/test_public_api.py similarity index 100% rename from tests/core/views/test_public_api.py rename to tests/api/test_public_api.py diff --git a/tests/conftest.py b/tests/conftest.py index d98956053..0418b00e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,9 +26,7 @@ from lnbits.db import DB_TYPE, SQLITE, Database from lnbits.settings import settings from tests.helpers import ( clean_database, - get_hold_invoice, get_random_invoice_data, - get_real_invoice, ) # override settings for tests @@ -182,13 +180,6 @@ async def invoice(to_wallet): del invoice -@pytest_asyncio.fixture(scope="function") -async def real_invoice(): - invoice = get_real_invoice(100) - yield {"bolt11": invoice["payment_request"]} - del invoice - - @pytest_asyncio.fixture(scope="session") async def fake_payments(client, adminkey_headers_from): # Because sqlite only stores timestamps with milliseconds @@ -212,10 +203,3 @@ async def fake_payments(client, adminkey_headers_from): params = {"time[ge]": ts, "time[le]": time()} return fake_data, params - - -@pytest_asyncio.fixture(scope="function") -async def hold_invoice(): - invoice = get_hold_invoice(100) - yield invoice - del invoice diff --git a/tests/helpers.py b/tests/helpers.py index 7dcaffd76..59a109f03 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,13 +1,7 @@ -import hashlib -import json -import os import random import string -import time -from subprocess import PIPE, Popen, TimeoutExpired -from typing import Optional, Tuple +from typing import Optional -from loguru import logger from psycopg2 import connect from psycopg2.errors import InvalidCatalogName @@ -39,129 +33,6 @@ is_fake: bool = funding_source.__class__.__name__ == "FakeWallet" is_regtest: bool = not is_fake -docker_lightning_cli = [ - "docker", - "exec", - "lnbits-lnd-1-1", - "lncli", - "--network", - "regtest", - "--rpcserver=lnd-1", -] - -docker_bitcoin_cli = [ - "docker", - "exec", - "lnbits-bitcoind-1-1" "bitcoin-cli", - "-rpcuser=lnbits", - "-rpcpassword=lnbits", - "-regtest", -] - - -docker_lightning_unconnected_cli = [ - "docker", - "exec", - "lnbits-lnd-2-1", - "lncli", - "--network", - "regtest", - "--rpcserver=lnd-2", -] - - -def run_cmd(cmd: list) -> str: - timeout = 20 - process = Popen(cmd, stdout=PIPE, stderr=PIPE) - - def process_communication(comm): - stdout, stderr = comm - output = stdout.decode("utf-8").strip() - error = stderr.decode("utf-8").strip() - return output, error - - try: - now = time.time() - output, error = process_communication(process.communicate(timeout=timeout)) - took = time.time() - now - logger.debug(f"ran command output: {output}, error: {error}, took: {took}s") - return output - except TimeoutExpired: - process.kill() - output, error = process_communication(process.communicate()) - logger.error(f"timeout command: {cmd}, output: {output}, error: {error}") - raise - - -def run_cmd_json(cmd: list) -> dict: - output = run_cmd(cmd) - try: - return json.loads(output) if output else {} - except json.decoder.JSONDecodeError: - logger.error(f"failed to decode json from cmd `{cmd}`: {output}") - raise - - -def get_hold_invoice(sats: int) -> Tuple[str, dict]: - preimage = os.urandom(32) - preimage_hash = hashlib.sha256(preimage).hexdigest() - cmd = docker_lightning_cli.copy() - cmd.extend(["addholdinvoice", preimage_hash, str(sats)]) - json = run_cmd_json(cmd) - return preimage.hex(), json - - -def settle_invoice(preimage: str) -> str: - cmd = docker_lightning_cli.copy() - cmd.extend(["settleinvoice", preimage]) - return run_cmd(cmd) - - -def cancel_invoice(preimage_hash: str) -> str: - cmd = docker_lightning_cli.copy() - cmd.extend(["cancelinvoice", preimage_hash]) - return run_cmd(cmd) - - -def get_real_invoice(sats: int) -> dict: - cmd = docker_lightning_cli.copy() - cmd.extend(["addinvoice", str(sats)]) - return run_cmd_json(cmd) - - -def pay_real_invoice(invoice: str) -> str: - cmd = docker_lightning_cli.copy() - cmd.extend(["payinvoice", "--force", invoice]) - return run_cmd(cmd) - - -def mine_blocks(blocks: int = 1) -> str: - cmd = docker_bitcoin_cli.copy() - cmd.extend(["-generate", str(blocks)]) - return run_cmd(cmd) - - -def get_unconnected_node_uri() -> str: - cmd = docker_lightning_unconnected_cli.copy() - cmd.append("getinfo") - info = run_cmd_json(cmd) - pubkey = info["identity_pubkey"] - return f"{pubkey}@lnd-2:9735" - - -def create_onchain_address(address_type: str = "bech32") -> str: - cmd = docker_bitcoin_cli.copy() - cmd.extend(["getnewaddress", address_type]) - return run_cmd(cmd) - - -def pay_onchain(address: str, sats: int) -> str: - btc = sats * 0.00000001 - cmd = docker_bitcoin_cli.copy() - cmd.extend(["sendtoaddress", address, str(btc)]) - return run_cmd(cmd) - - def clean_database(settings): if DB_TYPE == POSTGRES: conn = connect(settings.lnbits_database_url) diff --git a/tests/core/views/__init__.py b/tests/regtest/__init__.py similarity index 100% rename from tests/core/views/__init__.py rename to tests/regtest/__init__.py diff --git a/tests/regtest/conftest.py b/tests/regtest/conftest.py new file mode 100644 index 000000000..f0134a224 --- /dev/null +++ b/tests/regtest/conftest.py @@ -0,0 +1,17 @@ +import pytest_asyncio + +from .helpers import get_hold_invoice, get_real_invoice + + +@pytest_asyncio.fixture(scope="function") +async def hold_invoice(): + invoice = get_hold_invoice(100) + yield invoice + del invoice + + +@pytest_asyncio.fixture(scope="function") +async def real_invoice(): + invoice = get_real_invoice(100) + yield {"bolt11": invoice["payment_request"]} + del invoice diff --git a/tests/regtest/helpers.py b/tests/regtest/helpers.py new file mode 100644 index 000000000..ec39b7593 --- /dev/null +++ b/tests/regtest/helpers.py @@ -0,0 +1,130 @@ +import hashlib +import json +import os +import time +from subprocess import PIPE, Popen, TimeoutExpired +from typing import Tuple + +from loguru import logger + +docker_lightning_cli = [ + "docker", + "exec", + "lnbits-lnd-1-1", + "lncli", + "--network", + "regtest", + "--rpcserver=lnd-1", +] + +docker_bitcoin_cli = [ + "docker", + "exec", + "lnbits-bitcoind-1-1" "bitcoin-cli", + "-rpcuser=lnbits", + "-rpcpassword=lnbits", + "-regtest", +] + + +docker_lightning_unconnected_cli = [ + "docker", + "exec", + "lnbits-lnd-2-1", + "lncli", + "--network", + "regtest", + "--rpcserver=lnd-2", +] + + +def run_cmd(cmd: list) -> str: + timeout = 20 + process = Popen(cmd, stdout=PIPE, stderr=PIPE) + + def process_communication(comm): + stdout, stderr = comm + output = stdout.decode("utf-8").strip() + error = stderr.decode("utf-8").strip() + return output, error + + try: + now = time.time() + output, error = process_communication(process.communicate(timeout=timeout)) + took = time.time() - now + logger.debug(f"ran command output: {output}, error: {error}, took: {took}s") + return output + except TimeoutExpired: + process.kill() + output, error = process_communication(process.communicate()) + logger.error(f"timeout command: {cmd}, output: {output}, error: {error}") + raise + + +def run_cmd_json(cmd: list) -> dict: + output = run_cmd(cmd) + try: + return json.loads(output) if output else {} + except json.decoder.JSONDecodeError: + logger.error(f"failed to decode json from cmd `{cmd}`: {output}") + raise + + +def get_hold_invoice(sats: int) -> Tuple[str, dict]: + preimage = os.urandom(32) + preimage_hash = hashlib.sha256(preimage).hexdigest() + cmd = docker_lightning_cli.copy() + cmd.extend(["addholdinvoice", preimage_hash, str(sats)]) + json = run_cmd_json(cmd) + return preimage.hex(), json + + +def settle_invoice(preimage: str) -> str: + cmd = docker_lightning_cli.copy() + cmd.extend(["settleinvoice", preimage]) + return run_cmd(cmd) + + +def cancel_invoice(preimage_hash: str) -> str: + cmd = docker_lightning_cli.copy() + cmd.extend(["cancelinvoice", preimage_hash]) + return run_cmd(cmd) + + +def get_real_invoice(sats: int) -> dict: + cmd = docker_lightning_cli.copy() + cmd.extend(["addinvoice", str(sats)]) + return run_cmd_json(cmd) + + +def pay_real_invoice(invoice: str) -> str: + cmd = docker_lightning_cli.copy() + cmd.extend(["payinvoice", "--force", invoice]) + return run_cmd(cmd) + + +def mine_blocks(blocks: int = 1) -> str: + cmd = docker_bitcoin_cli.copy() + cmd.extend(["-generate", str(blocks)]) + return run_cmd(cmd) + + +def get_unconnected_node_uri() -> str: + cmd = docker_lightning_unconnected_cli.copy() + cmd.append("getinfo") + info = run_cmd_json(cmd) + pubkey = info["identity_pubkey"] + return f"{pubkey}@lnd-2:9735" + + +def create_onchain_address(address_type: str = "bech32") -> str: + cmd = docker_bitcoin_cli.copy() + cmd.extend(["getnewaddress", address_type]) + return run_cmd(cmd) + + +def pay_onchain(address: str, sats: int) -> str: + btc = sats * 0.00000001 + cmd = docker_bitcoin_cli.copy() + cmd.extend(["sendtoaddress", address, str(btc)]) + return run_cmd(cmd) diff --git a/tests/regtest/test_real_invoice.py b/tests/regtest/test_real_invoice.py new file mode 100644 index 000000000..bdf043b0f --- /dev/null +++ b/tests/regtest/test_real_invoice.py @@ -0,0 +1,390 @@ +import asyncio +import hashlib + +import pytest + +from lnbits import bolt11 +from lnbits.core.crud import get_standalone_payment, update_payment_details +from lnbits.core.models import CreateInvoice, Payment +from lnbits.core.services import fee_reserve_total +from lnbits.core.views.admin_api import api_auditor +from lnbits.core.views.payment_api import api_payment +from lnbits.wallets import get_funding_source + +from ..helpers import is_fake, is_regtest +from .helpers import ( + cancel_invoice, + get_real_invoice, + pay_real_invoice, + settle_invoice, +) + + +async def get_node_balance_sats(): + audit = await api_auditor() + return audit["node_balance_msats"] / 1000 + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_pay_real_invoice( + client, real_invoice, adminkey_headers_from, inkey_headers_from, from_wallet_ws +): + prev_balance = await get_node_balance_sats() + response = await client.post( + "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from + ) + assert response.status_code < 300 + invoice = response.json() + assert len(invoice["payment_hash"]) == 64 + assert len(invoice["checking_id"]) > 0 + + data = from_wallet_ws.receive_json() + assert "wallet_balance" in data + payment = Payment(**data["payment"]) + assert payment.payment_hash == invoice["payment_hash"] + + # check the payment status + response = await client.get( + f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from + ) + assert response.status_code < 300 + payment_status = response.json() + assert payment_status["paid"] + + funding_source = get_funding_source() + status = await funding_source.get_payment_status(invoice["payment_hash"]) + assert status.paid + + await asyncio.sleep(1) + balance = await get_node_balance_sats() + assert prev_balance - balance == 100 + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_from): + prev_balance = await get_node_balance_sats() + create_invoice = CreateInvoice(out=False, amount=1000, memo="test") + response = await client.post( + "/api/v1/payments", + json=create_invoice.dict(), + headers=adminkey_headers_from, + ) + assert response.status_code < 300 + invoice = response.json() + + response = await client.get( + f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from + ) + assert response.status_code < 300 + payment_status = response.json() + assert not payment_status["paid"] + + async def listen(): + found_checking_id = False + async for checking_id in get_funding_source().paid_invoices_stream(): + if checking_id == invoice["checking_id"]: + found_checking_id = True + return + assert found_checking_id + + task = asyncio.create_task(listen()) + await asyncio.sleep(1) + pay_real_invoice(invoice["payment_request"]) + await asyncio.wait_for(task, timeout=10) + + response = await client.get( + f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from + ) + assert response.status_code < 300 + payment_status = response.json() + assert payment_status["paid"] + + await asyncio.sleep(1) + balance = await get_node_balance_sats() + assert balance - prev_balance == create_invoice.amount + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_pay_real_invoice_set_pending_and_check_state( + client, real_invoice, adminkey_headers_from, inkey_headers_from +): + """ + 1. We create an invoice + 2. We pay it + 3. We verify that the inoice was paid + 4. We set the invoice to pending in the database + 5. We recheck the state of the invoice + 6. We verify that the invoice is paid + """ + response = await client.post( + "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from + ) + assert response.status_code < 300 + invoice = response.json() + assert len(invoice["payment_hash"]) == 64 + assert len(invoice["checking_id"]) > 0 + + # check the payment status + response = await api_payment( + invoice["payment_hash"], inkey_headers_from["X-Api-Key"] + ) + assert response["paid"] + + # make sure that the backend also thinks it's paid + funding_source = get_funding_source() + status = await funding_source.get_payment_status(invoice["payment_hash"]) + assert status.paid + + # get the outgoing payment from the db + payment = await get_standalone_payment(invoice["payment_hash"]) + assert payment + assert payment.pending is False + + # set the outgoing invoice to pending + await update_payment_details(payment.checking_id, pending=True) + + payment_pending = await get_standalone_payment(invoice["payment_hash"]) + assert payment_pending + assert payment_pending.pending is True + + # check the outgoing payment status + await payment.check_status() + + payment_not_pending = await get_standalone_payment(invoice["payment_hash"]) + assert payment_not_pending + assert payment_not_pending.pending is False + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_pay_hold_invoice_check_pending( + client, hold_invoice, adminkey_headers_from +): + preimage, invoice = hold_invoice + task = asyncio.create_task( + client.post( + "/api/v1/payments", + json={"bolt11": invoice["payment_request"]}, + headers=adminkey_headers_from, + ) + ) + await asyncio.sleep(1) + + # get payment hash from the invoice + invoice_obj = bolt11.decode(invoice["payment_request"]) + + payment_db = await get_standalone_payment(invoice_obj.payment_hash) + + assert payment_db + assert payment_db.pending is True + + settle_invoice(preimage) + + response = await task + assert response.status_code < 300 + + # check if paid + + await asyncio.sleep(1) + + payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash) + + assert payment_db_after_settlement + assert payment_db_after_settlement.pending is False + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_pay_hold_invoice_check_pending_and_fail( + client, hold_invoice, adminkey_headers_from +): + preimage, invoice = hold_invoice + task = asyncio.create_task( + client.post( + "/api/v1/payments", + json={"bolt11": invoice["payment_request"]}, + headers=adminkey_headers_from, + ) + ) + await asyncio.sleep(1) + + # get payment hash from the invoice + invoice_obj = bolt11.decode(invoice["payment_request"]) + + payment_db = await get_standalone_payment(invoice_obj.payment_hash) + + assert payment_db + assert payment_db.pending is True + + preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest() + + # cancel the hodl invoice + assert preimage_hash == invoice_obj.payment_hash + cancel_invoice(preimage_hash) + + response = await task + assert response.status_code > 300 # should error + + await asyncio.sleep(1) + + # payment should not be in database anymore + payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash) + assert payment_db_after_settlement is None + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_pay_hold_invoice_check_pending_and_fail_cancel_payment_task_in_meantime( + client, hold_invoice, adminkey_headers_from +): + preimage, invoice = hold_invoice + task = asyncio.create_task( + client.post( + "/api/v1/payments", + json={"bolt11": invoice["payment_request"]}, + headers=adminkey_headers_from, + ) + ) + await asyncio.sleep(1) + + # get payment hash from the invoice + invoice_obj = bolt11.decode(invoice["payment_request"]) + + payment_db = await get_standalone_payment(invoice_obj.payment_hash) + + assert payment_db + assert payment_db.pending is True + + # cancel payment task, this simulates the client dropping the connection + task.cancel() + + preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest() + + assert preimage_hash == invoice_obj.payment_hash + cancel_invoice(preimage_hash) + + # check if paid + await asyncio.sleep(1) + + # payment should still be in db + payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash) + assert payment_db_after_settlement is not None + + # status should still be available and be False + status = await payment_db.check_status() + assert not status.paid + + # now the payment should be gone after the status check + # payment_db_after_status_check = await get_standalone_payment( + # invoice_obj.payment_hash + # ) + # assert payment_db_after_status_check is None + + +@pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="this only works in regtest") +async def test_receive_real_invoice_set_pending_and_check_state( + client, adminkey_headers_from, inkey_headers_from +): + """ + 1. We create a real invoice + 2. We pay it from our wallet + 3. We check that the inoice was paid with the backend + 4. We set the invoice to pending in the database + 5. We recheck the state of the invoice with the backend + 6. We verify that the invoice is now marked as paid in the database + """ + create_invoice = CreateInvoice(out=False, amount=1000, memo="test") + response = await client.post( + "/api/v1/payments", + json=create_invoice.dict(), + headers=adminkey_headers_from, + ) + assert response.status_code < 300 + invoice = response.json() + response = await api_payment( + invoice["payment_hash"], inkey_headers_from["X-Api-Key"] + ) + assert not response["paid"] + + async def listen(): + found_checking_id = False + async for checking_id in get_funding_source().paid_invoices_stream(): + if checking_id == invoice["checking_id"]: + found_checking_id = True + return + assert found_checking_id + + task = asyncio.create_task(listen()) + await asyncio.sleep(1) + pay_real_invoice(invoice["payment_request"]) + await asyncio.wait_for(task, timeout=10) + response = await api_payment( + invoice["payment_hash"], inkey_headers_from["X-Api-Key"] + ) + assert response["paid"] + + # get the incoming payment from the db + payment = await get_standalone_payment(invoice["payment_hash"], incoming=True) + assert payment + assert payment.pending is False + + # set the incoming invoice to pending + await update_payment_details(payment.checking_id, pending=True) + + payment_pending = await get_standalone_payment( + invoice["payment_hash"], incoming=True + ) + assert payment_pending + assert payment_pending.pending is True + + # check the incoming payment status + await payment.check_status() + + payment_not_pending = await get_standalone_payment( + invoice["payment_hash"], incoming=True + ) + assert payment_not_pending + assert payment_not_pending.pending is False + + # verify we get the same result if we use the checking_id to look up the payment + payment_by_checking_id = await get_standalone_payment( + payment_not_pending.checking_id, incoming=True + ) + + assert payment_by_checking_id + assert payment_by_checking_id.pending is False + assert payment_by_checking_id.bolt11 == payment_not_pending.bolt11 + assert payment_by_checking_id.payment_hash == payment_not_pending.payment_hash + + +@pytest.mark.asyncio +async def test_check_fee_reserve(client, adminkey_headers_from): + # if regtest, create a real invoice, otherwise create an internal invoice + # call /api/v1/payments/fee-reserve?invoice=... with it and check if the fee reserve + # is correct + payment_request = "" + if is_regtest: + real_invoice = get_real_invoice(1000) + payment_request = real_invoice["payment_request"] + + else: + create_invoice = CreateInvoice(out=False, amount=1000, memo="test") + response = await client.post( + "/api/v1/payments", + json=create_invoice.dict(), + headers=adminkey_headers_from, + ) + assert response.status_code < 300 + invoice = response.json() + payment_request = invoice["payment_request"] + + response = await client.get( + f"/api/v1/payments/fee-reserve?invoice={payment_request}", + ) + assert response.status_code < 300 + fee_reserve = response.json() + assert fee_reserve["fee_reserve"] == fee_reserve_total(1000_000) diff --git a/tests/core/views/test_node_api.py b/tests/regtest/test_x_node_api.py similarity index 99% rename from tests/core/views/test_node_api.py rename to tests/regtest/test_x_node_api.py index c314e4d63..30a6b6e95 100644 --- a/tests/core/views/test_node_api.py +++ b/tests/regtest/test_x_node_api.py @@ -9,9 +9,11 @@ from lnbits import bolt11 from lnbits.nodes.base import ChannelPoint, ChannelState, NodeChannel from tests.conftest import pytest_asyncio, settings -from ...helpers import ( +from ..helpers import ( funding_source, get_random_invoice_data, +) +from .helpers import ( get_unconnected_node_uri, mine_blocks, ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/test_cache.py b/tests/unit/test_cache.py similarity index 100% rename from tests/core/test_cache.py rename to tests/unit/test_cache.py diff --git a/tests/core/test_db.py b/tests/unit/test_db.py similarity index 100% rename from tests/core/test_db.py rename to tests/unit/test_db.py diff --git a/tests/core/test_db_fetch_page.py b/tests/unit/test_db_fetch_page.py similarity index 100% rename from tests/core/test_db_fetch_page.py rename to tests/unit/test_db_fetch_page.py diff --git a/tests/core/test_helpers_query.py b/tests/unit/test_helpers_query.py similarity index 100% rename from tests/core/test_helpers_query.py rename to tests/unit/test_helpers_query.py