[REFACTOR] replace async_wrap with run_sync (#1858)

* replace async_wrap with run_sync

formatting

remove unused fn

properly raise exception, not for timeout though

* [TEST] proper startup and shutdown (#1860)

* add proper startup / shutdown in tests

* fix event loop issues

because uvloop was installed in server.py which is not the main entry point when tests are ran.
this caused the loops referenced by the locks and queues to be a different one than the one used to run the tests
(only an issue in python 3.9)

* give openapi more time, does not matter anyway, regtest takes way longer

---------

Co-authored-by: jacksn <jkranawetter05@gmail.com>

remove uvloop

* fix test

* dont touch pyproject

* fix: install uvloop in conftest

not using uvloop at all causes tests to take way longer

---------

Co-authored-by: jacksn <jkranawetter05@gmail.com>

Makefile

@@ -73,7 +73,7 @@ openapi:
 	HOST=0.0.0.0 \
 	PORT=5003 \
 	poetry run lnbits &
-	sleep 7
+	sleep 15
 	curl -s http://0.0.0.0:5003/openapi.json | poetry run openapi-spec-validator --errors=all -
 	# kill -9 %1
 

lnbits/server.py

@@ -4,13 +4,10 @@ from pathlib import Path
 
 import click
 import uvicorn
-import uvloop
 from uvicorn.supervisors import ChangeReload
 
 from lnbits.settings import set_cli_settings, settings
 
-uvloop.install()
-
 
 @click.command(
     context_settings=dict(

lnbits/wallets/corelightning.py

@@ -1,7 +1,6 @@
 import asyncio
 import random
-from functools import partial, wraps
+from typing import Any, AsyncGenerator, Optional
-from typing import AsyncGenerator, Optional
 
 from loguru import logger
 from pyln.client import LightningRpc, RpcError
@@ -19,23 +18,9 @@ from .base import (
 )
 
 
-def async_wrap(func):
+async def run_sync(func) -> Any:
-    @wraps(func)
+    loop = asyncio.get_event_loop()
-    async def run(*args, loop=None, executor=None, **kwargs):
+    return await loop.run_in_executor(None, func)
-        if loop is None:
-            loop = asyncio.get_event_loop()
-        partial_func = partial(func, *args, **kwargs)
-        return await loop.run_in_executor(executor, partial_func)
-
-    return run
-
-
-def _pay_invoice(ln, payload):
-    return ln.call("pay", payload)
-
-
-def _paid_invoices_stream(ln, last_pay_index):
-    return ln.waitanyinvoice(last_pay_index)
 
 
 class CoreLightningWallet(Wallet):
@@ -125,8 +110,7 @@ class CoreLightningWallet(Wallet):
             "exemptfee": 0,
         }
         try:
-            wrapped = async_wrap(_pay_invoice)
+            r = await run_sync(lambda: self.ln.call("pay", payload))
-            r = await wrapped(self.ln, payload)
         except RpcError as exc:
             try:
                 error_message = exc.error["attempts"][-1]["fail_reason"]  # type: ignore
@@ -193,10 +177,15 @@ class CoreLightningWallet(Wallet):
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
         while True:
             try:
-                wrapped = async_wrap(_paid_invoices_stream)
+                paid = await run_sync(
-                paid = await wrapped(self.ln, self.last_pay_index)
+                    lambda: self.ln.waitanyinvoice(self.last_pay_index, timeout=2)
+                )
                 self.last_pay_index = paid["pay_index"]
                 yield paid["payment_hash"]
+            except RpcError as exc:
+                # only raise if not a timeout
+                if exc.error["code"] != 904:  # type: ignore
+                    raise
             except Exception as exc:
                 logger.error(
                     f"lost connection to corelightning invoices stream: '{exc}', "

tests/conftest.py

@@ -1,12 +1,15 @@
 import asyncio
 
+import uvloop
+
+uvloop.install()  # noqa
+
 import pytest
 import pytest_asyncio
 from fastapi.testclient import TestClient
 from httpx import AsyncClient
 
 from lnbits.app import create_app
-from lnbits.commands import migrate_databases
 from lnbits.core.crud import create_account, create_wallet
 from lnbits.core.models import CreateInvoice
 from lnbits.core.services import update_wallet_balance
@@ -28,17 +31,11 @@ def event_loop():
 
 # use session scope to run once before and once after all tests
 @pytest_asyncio.fixture(scope="session")
-def app(event_loop):
+async def app():
     app = create_app()
-    # use redefined version of the event loop for scope="session"
+    await app.router.startup()
-    # loop = asyncio.get_event_loop()
-    loop = event_loop
-    loop.run_until_complete(migrate_databases())
     yield app
-    # # get the current event loop and gracefully stop any running tasks
+    await app.router.shutdown()
-    # loop = event_loop
-    loop.run_until_complete(loop.shutdown_asyncgens())
-    # loop.close()
 
 
 @pytest_asyncio.fixture(scope="session")
@@ -75,8 +72,10 @@ async def from_wallet(from_user):
     yield wallet
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
-def from_wallet_ws(from_wallet, test_client):
+async def from_wallet_ws(from_wallet, test_client):
+    # wait a bit in order to avoid receiving topup notification
+    await asyncio.sleep(0.1)
     with test_client.websocket_connect(f"/api/v1/ws/{from_wallet.id}") as ws:
         yield ws
 
@@ -98,8 +97,10 @@ async def to_wallet(to_user):
     yield wallet
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
-def to_wallet_ws(to_wallet, test_client):
+async def to_wallet_ws(to_wallet, test_client):
+    # wait a bit in order to avoid receiving topup notification
+    await asyncio.sleep(0.1)
     with test_client.websocket_connect(f"/api/v1/ws/{to_wallet.id}") as ws:
         yield ws
 

tests/core/views/test_api.py

@@ -132,9 +132,7 @@ async def test_create_invoice_custom_expiry(client, inkey_headers_to):
 
 # check POST /api/v1/payments: make payment
 @pytest.mark.asyncio
-async def test_pay_invoice(
+async def test_pay_invoice(client, from_wallet_ws, invoice, adminkey_headers_from):
-    client, from_wallet_ws, to_wallet_ws, invoice, adminkey_headers_from
-):
     data = {"out": True, "bolt11": invoice["payment_request"]}
     response = await client.post(
         "/api/v1/payments", json=data, headers=adminkey_headers_from