plugin: clnrest

plugin: clnrest

plugins/clnrest/README.md

@@ -0,0 +1,47 @@
+# CLNRest
+
+CLNRest is a lightweight Python-based core lightning plugin that transforms RPC calls into a REST service. By generating REST API endpoints, it enables the execution of Core Lightning's RPC methods behind the scenes and provides responses in JSON format.
+
+## Installation
+
+Install required packages with `pip install json5 flask flask_restx gunicorn pyln-client` or `pip install -r requirements.txt`.
+
+## Configuration
+
+- --rest-protocol: Specifies the REST server protocol. Default is HTTPS.
+- --rest-host: Defines the REST server host. Default is 127.0.0.1.
+- --rest-port: Sets the REST server port to listen to. Default is 3010.
+- --rest-certs: Defines the path for HTTPS cert & key. Default path is same as RPC file path to utilize gRPC's client certificate. If it is missing at the configured location, new identity (`client.pem` and `client-key.pem`) will be generated.
+
+## Plugin
+
+- It can be configured by adding `plugin=/<path>/clnrest/clnrest.py` to the Core Lightning's config file.
+
+## Server
+
+With the default configurations, the Swagger user interface will be available at https://127.0.0.1:3010/. The POST method requires `rune` and `nodeid` headers for authorization.
+
+### cURL
+Example curl command for GET:
+    `curl -k https://127.0.0.1:3010/v1/notifications`
+
+Example curl command for POST will also require `rune` and `nodeid` headers like below:
+    `curl -k -X POST 'https://127.0.0.1:3010/v1/getinfo' -H 'Rune: <node-rune>' -H 'Nodeid: <node-id>'`
+
+With `-k` or `--insecure` option curl proceeds with the connection even if the SSL certificate cannot be verified.
+This option should be used only when testing with self signed certificate.
+
+### Swagger
+<p float="left">
+    <img src="./.github/screenshots/Swagger.png" width="200" alt="Swagger Dashboard" />
+    <img src="./.github/screenshots/Swagger-auth.png" width="200" alt="Swagger Authorize" />
+    <img src="./.github/screenshots/Swagger-list-methods.png" width="200" alt="Swagger GET List Methods" />
+    <img src="./.github/screenshots/Swagger-rpc-method.png" width="200" alt="Swagger POST RPC Method" />
+</p>
+
+### Postman
+<p float="left">
+    <img src="./.github/screenshots/Postman.png" width="200" alt="Postman Headers">
+    <img src="./.github/screenshots/Postman-with-body.png" width="200" alt="Postman with JSON body">
+    <img src="./.github/screenshots/Postman-bkpr-plugin.png" width="200" alt="Postman bkpr plugin RPC">
+</p>

plugins/clnrest/clnrest.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# For --hidden-import gunicorn.glogging gunicorn.workers.sync
+from gunicorn import glogging  # noqa: F401
+from gunicorn.workers import sync  # noqa: F401
+
+from pathlib import Path
+from flask import Flask
+from flask_restx import Api
+from gunicorn.app.base import BaseApplication
+from multiprocessing import Process, cpu_count
+from utilities.generate_certs import generate_certs
+from utilities.shared import set_config
+from utilities.rpc_routes import rpcns
+from utilities.rpc_plugin import plugin
+
+jobs = {}
+
+
+def create_app():
+    app = Flask(__name__)
+    authorizations = {
+        "rune": {"type": "apiKey", "in": "header", "name": "Rune"},
+        "nodeid": {"type": "apiKey", "in": "header", "name": "Nodeid"}
+    }
+    api = Api(app, version="1.0", title="Core Lightning Rest", description="Core Lightning REST API Swagger", authorizations=authorizations, security=["rune", "nodeid"])
+    api.add_namespace(rpcns, path="/v1")
+    return app
+
+
+def set_application_options(plugin):
+    from utilities.shared import CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT
+    plugin.log(f"REST Server is starting at {REST_PROTOCOL}://{REST_HOST}:{REST_PORT}", "debug")
+    if REST_PROTOCOL == "http":
+        options = {
+            "bind": f"{REST_HOST}:{REST_PORT}",
+            "workers": cpu_count(),
+            "timeout": 60,
+            "loglevel": "warning",
+        }
+    else:
+        cert_file = Path(f"{CERTS_PATH}/client.pem")
+        key_file = Path(f"{CERTS_PATH}/client-key.pem")
+        if not cert_file.is_file() or not key_file.is_file():
+            plugin.log(f"Certificate not found at {CERTS_PATH}. Generating a new certificate!", "debug")
+            generate_certs(plugin, CERTS_PATH)
+        try:
+            plugin.log(f"Certs Path: {CERTS_PATH}", "debug")
+        except Exception as err:
+            raise Exception(f"{err}: Certificates do not exist at {CERTS_PATH}")
+        options = {
+            "bind": f"{REST_HOST}:{REST_PORT}",
+            "workers": cpu_count(),
+            "timeout": 60,
+            "loglevel": "warning",
+            "certfile": f"{CERTS_PATH}/client.pem",
+            "keyfile": f"{CERTS_PATH}/client-key.pem",
+        }
+    return options
+
+
+class CLNRestApplication(BaseApplication):
+    def __init__(self, app, options=None):
+        from utilities.shared import REST_PROTOCOL, REST_HOST, REST_PORT
+        self.application = app
+        self.options = options or {}
+        plugin.log(f"REST server running at {REST_PROTOCOL}://{REST_HOST}:{REST_PORT}", "info")
+        super().__init__()
+
+    def load_config(self):
+        config = {key: value for key, value in self.options.items()
+                  if key in self.cfg.settings and value is not None}
+        for key, value in config.items():
+            self.cfg.set(key.lower(), value)
+
+    def load(self):
+        return self.application
+
+
+def worker():
+    options = set_application_options(plugin)
+    app = create_app()
+    CLNRestApplication(app, options).run()
+
+
+def start_server():
+    from utilities.shared import REST_PORT
+    if REST_PORT in jobs:
+        return False, "server already running"
+
+    p = Process(
+        target=worker,
+        args=[],
+        name="server on port {}".format(REST_PORT),
+    )
+    p.daemon = True
+    jobs[REST_PORT] = p
+    p.start()
+    return True
+
+
+@plugin.init()
+def init(options, configuration, plugin):
+    set_config(options)
+    start_server()
+
+
+plugin.run()

plugins/clnrest/requirements.txt

@@ -0,0 +1,34 @@
+aniso8601==9.0.1
+asn1crypto==1.5.1
+attrs==23.1.0
+base58==2.1.1
+bitstring==3.1.9
+blinker==1.6.2
+cachelib==0.10.2
+cffi==1.15.1
+click==8.1.3
+coincurve==17.0.0
+cryptography==36.0.2
+Flask==2.3.2
+Flask-Cors==4.0.0
+flask-restx==1.1.0
+Flask-WTF==1.1.1
+gevent==22.10.2
+greenlet==2.0.2
+gunicorn==20.1.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+json5==0.9.14
+jsonschema==4.17.3
+MarkupSafe==2.1.3
+pycparser==2.21
+pyln-bolt7==1.0.246
+pyln-client==23.5
+pyln-proto==23.5
+pyrsistent==0.19.3
+PySocks==1.7.1
+pytz==2023.3
+Werkzeug==2.3.6
+WTForms==3.0.1
+zope.event==5.0
+zope.interface==6.0

plugins/clnrest/utilities/generate_certs.py

@@ -0,0 +1,36 @@
+import os
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.serialization import Encoding
+import datetime
+
+
+def generate_certs(plugin, certs_path):
+    # Generate key
+    key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    # Create the certs directory if it does not exist
+    os.makedirs(certs_path, exist_ok=True)
+    # Write key
+    with open(os.path.join(certs_path, "client-key.pem"), "wb") as f:
+        f.write(key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        ))
+    subject = issuer = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"Core Lightning")])
+    cert = (
+        x509.CertificateBuilder()
+        .subject_name(subject)
+        .issuer_name(issuer)
+        .public_key(key.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(datetime.datetime.utcnow())
+        .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=10 * 365))  # Ten years validity
+        .sign(key, hashes.SHA256())
+    )
+    with open(os.path.join(certs_path, "client.pem"), "wb") as f:
+        f.write(cert.public_bytes(Encoding.PEM))
+    plugin.log(f"Certificate Generated!", "debug")

plugins/clnrest/utilities/rpc_plugin.py

@@ -0,0 +1,23 @@
+import os
+import sys
+from multiprocessing import Manager
+from pyln.client import Plugin
+
+plugin = Plugin(autopatch=False)
+manager = Manager()
+queue = manager.Queue()
+
+plugin.add_option(name="rest-certs", default=os.getcwd(), description="Path for certificates (for https)", opt_type="string", deprecated=False)
+plugin.add_option(name="rest-protocol", default="https", description="REST server protocol", opt_type="string", deprecated=False)
+plugin.add_option(name="rest-host", default="127.0.0.1", description="REST server host", opt_type="string", deprecated=False)
+plugin.add_option(name="rest-port", default=3010, description="REST server port to listen", opt_type="int", deprecated=False)
+
+
+@plugin.subscribe("*")
+def on_any_notification(request, **kwargs):
+    plugin.log("Notification: {}".format(kwargs), "debug")
+    if request.method == 'shutdown':
+        # A plugin which subscribes to shutdown is expected to exit itself.
+        sys.exit(0)
+    else:
+        queue.put(str(kwargs) + "\n")

plugins/clnrest/utilities/rpc_routes.py

@@ -0,0 +1,72 @@
+import json5
+from flask import request, make_response, Response, stream_with_context
+from flask_restx import Namespace, Resource
+from .shared import call_rpc_method, verify_rune, process_help_response
+from .rpc_plugin import plugin
+
+methods_list = []
+rpcns = Namespace("RPCs")
+payload_model = rpcns.model("Payload", {}, None, False)
+
+
+@rpcns.route("/list-methods")
+class ListMethodsResource(Resource):
+    @rpcns.response(200, "Success")
+    @rpcns.response(500, "Server error")
+    def get(self):
+        """Get the list of all valid rpc methods, useful for Swagger to get human readable list without calling lightning-cli help"""
+        try:
+            help_response = call_rpc_method(plugin, "help", [])
+            html_content = process_help_response(help_response)
+            response = make_response(html_content)
+            response.headers["Content-Type"] = "text/html"
+            return response
+
+        except Exception as err:
+            plugin.log(f"Error: {err}", "error")
+            return json5.loads(str(err)), 500
+
+
+@rpcns.route("/<rpc_method>")
+class RpcMethodResource(Resource):
+    @rpcns.doc(security=[{"rune": [], "nodeid": []}])
+    @rpcns.doc(params={"rpc_method": (f"Name of the RPC method to be called")})
+    @rpcns.expect(payload_model, validate=False)
+    @rpcns.response(201, "Success")
+    @rpcns.response(500, "Server error")
+    def post(self, rpc_method):
+        """Call any valid core lightning method (check list-methods response)"""
+        try:
+            is_valid_rune = verify_rune(plugin, request)
+
+            if "error" in is_valid_rune:
+                plugin.log(f"Error: {is_valid_rune}", "error")
+                raise Exception(is_valid_rune)
+
+        except Exception as err:
+            return json5.loads(str(err)), 403
+
+        try:
+            if request.is_json:
+                payload = request.get_json()
+            else:
+                payload = request.form.to_dict()
+            return call_rpc_method(plugin, rpc_method, payload), 201
+
+        except Exception as err:
+            plugin.log(f"Error: {err}", "error")
+            return json5.loads(str(err)), 500
+
+
+@rpcns.route("/notifications")
+class NotificationsResource(Resource):
+    def get(self):
+        try:
+            def notifications_stream():
+                while True:
+                    from .rpc_plugin import queue
+                    yield queue.get()
+            return Response(stream_with_context(notifications_stream()), mimetype="text/event-stream")
+
+        except Exception as err:
+            return json5.loads(str(err)), 500

plugins/clnrest/utilities/shared.py

@@ -0,0 +1,71 @@
+import json5
+import re
+import json
+
+CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT = "", "", "", ""
+
+
+def set_config(options):
+    global CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT
+    CERTS_PATH = str(options["rest-certs"])
+    REST_PROTOCOL = str(options["rest-protocol"])
+    REST_HOST = str(options["rest-host"])
+    REST_PORT = int(options["rest-port"])
+
+
+def call_rpc_method(plugin, rpc_method, payload):
+    try:
+        response = plugin.rpc.call(rpc_method, payload)
+        if '"error":' in str(response).lower():
+            raise Exception(response)
+        else:
+            plugin.log(f"{response}", "debug")
+            if '"result":' in str(response).lower():
+                # Use json5.loads ONLY when necessary, as it increases processing time significantly
+                return json.loads(response)["result"]
+            else:
+                return response
+
+    except Exception as err:
+        plugin.log(f"Error: {err}", "error")
+        if "error" in str(err).lower():
+            match_err_obj = re.search(r'"error":\{.*?\}', str(err))
+            if match_err_obj is not None:
+                err = "{" + match_err_obj.group() + "}"
+            else:
+                match_err_str = re.search(r"error: \{.*?\}", str(err))
+                if match_err_str is not None:
+                    err = "{" + match_err_str.group() + "}"
+        raise Exception(err)
+
+
+def verify_rune(plugin, request):
+    rune = request.headers.get("rune", None)
+    nodeid = request.headers.get("nodeid", None)
+
+    if nodeid is None:
+        raise Exception('{ "error": {"code": 403, "message": "Not authorized: Missing nodeid"} }')
+
+    if rune is None:
+        raise Exception('{ "error": {"code": 403, "message": "Not authorized: Missing rune"} }')
+
+    if request.is_json:
+        rpc_params = request.get_json()
+    else:
+        rpc_params = request.form.to_dict()
+
+    return call_rpc_method(plugin, "checkrune", [rune, nodeid, request.view_args["rpc_method"], rpc_params])
+
+
+def process_help_response(help_response):
+    # Use json5.loads due to single quotes in response
+    processed_res = json5.loads(str(help_response))["help"]
+    line = "\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n\n"
+    processed_html_res = ""
+    for row in processed_res:
+        processed_html_res += f"Command: {row['command']}\n"
+        processed_html_res += f"Category: {row['category']}\n"
+        processed_html_res += f"Description: {row['description']}\n"
+        processed_html_res += f"Verbose: {row['verbose']}\n"
+        processed_html_res += line
+    return processed_html_res