From b8bcc7d13f50779fd54490681bf57317ffe431ee Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 30 Jun 2022 11:29:57 +0200 Subject: [PATCH] pytest: Add a new RPC interface to talk to grpc This allows us to re-use existing tests (assuming the call and fields are covered by `cln-rpc` and `cln-grpc`) to test the full roundtrip from test over the grpc interface to the json-rpc interface and back again. You can switch to the grpc interface by setting the `CLN_TEST_GRPC` environment variable to 1, but for now only very few shims are implemented (due to the non-generated nature of LightningRpc). --- contrib/pyln-testing/pyln/testing/grpc.py | 84 ++++++++++++++++++++++ contrib/pyln-testing/pyln/testing/utils.py | 64 +++++++++++++++-- tools/check-spelling.sh | 2 +- 3 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 contrib/pyln-testing/pyln/testing/grpc.py diff --git a/contrib/pyln-testing/pyln/testing/grpc.py b/contrib/pyln-testing/pyln/testing/grpc.py new file mode 100644 index 000000000..593f1ee09 --- /dev/null +++ b/contrib/pyln-testing/pyln/testing/grpc.py @@ -0,0 +1,84 @@ +"""A drop-in replacement for the JSON-RPC LightningRpc +""" + +from pyln.testing import node_pb2_grpc as pbgrpc +from pyln.testing import node_pb2 as pb +import grpc +import json +from google.protobuf.json_format import MessageToJson +from pyln.testing import grpc2py + + +DUMMY_CA_PEM = b"""-----BEGIN CERTIFICATE----- +MIIBcTCCARigAwIBAgIJAJhah1bqO05cMAoGCCqGSM49BAMCMBYxFDASBgNVBAMM +C2NsbiBSb290IENBMCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAW +MRQwEgYDVQQDDAtjbG4gUm9vdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BPF4JrGsOsksgsYM1NNdUdLESwOxkzyD75Rnj/g7sFEVYXewcmyB3MRGCBx2a3/7 +ft2Xu2ED6WigajaHlnSvfUyjTTBLMBkGA1UdEQQSMBCCA2NsboIJbG9jYWxob3N0 +MB0GA1UdDgQWBBRcTjvqVodamGirO6sX1rOR02LwXzAPBgNVHRMBAf8EBTADAQH/ +MAoGCCqGSM49BAMCA0cAMEQCICDvV5iFw/nmJdl6rlEEGAdBdZqjxD0tV6U/FvuL +7PycAiASEMtsFtpfiUvxveBkOGt7AN32GP/Z75l+GhYXh7L1ig== +-----END CERTIFICATE-----""" + + +DUMMY_CA_KEY_PEM = b"""-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgqbU7LQsRcvmI5vE5 +MBBNK3imhIU2jmAczgvLuBi/Ys+hRANCAATxeCaxrDrJLILGDNTTXVHSxEsDsZM8 +g++UZ4/4O7BRFWF3sHJsgdzERggcdmt/+37dl7thA+looGo2h5Z0r31M +-----END PRIVATE KEY-----""" + + +DUMMY_CLIENT_KEY_PEM = b"""-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgIEdQyKso8PaD1kiz +xxFEcKiTvTg+bej4Nc/GqnXipcGhRANCAARGoUNSnWx1qgt4RiVG8tOMX1vpKvhr +OLcUJ92T++kIFZchZvcTXwnlNiTAQg3ukL+RYyG5Q1PaYrYRVlOtl1T0 +-----END PRIVATE KEY-----""" + + +DUMMY_CLIENT_PEM = b"""-----BEGIN CERTIFICATE----- +MIIBRDCB7KADAgECAgkA8SsXq7IZfi8wCgYIKoZIzj0EAwIwFjEUMBIGA1UEAwwL +Y2xuIFJvb3QgQ0EwIBcNNzUwMTAxMDAwMDAwWhgPNDA5NjAxMDEwMDAwMDBaMBox +GDAWBgNVBAMMD2NsbiBncnBjIFNlcnZlcjBZMBMGByqGSM49AgEGCCqGSM49AwEH +A0IABEahQ1KdbHWqC3hGJUby04xfW+kq+Gs4txQn3ZP76QgVlyFm9xNfCeU2JMBC +De6Qv5FjIblDU9pithFWU62XVPSjHTAbMBkGA1UdEQQSMBCCA2NsboIJbG9jYWxo +b3N0MAoGCCqGSM49BAMCA0cAMEQCICTU/YAs35cb6DRdZNzO1YbEt77uEjcqMRca +Hh6kK99RAiAKOQOkGnoAICjBmBJeC/iC4/+hhhkWZtFgbC3Jg5JD0w== +-----END CERTIFICATE-----""" + + +class LightningGrpc(object): + def __init__( + self, + host: str, + port: int, + root_certificates: bytes = DUMMY_CA_PEM, + private_key: bytes = DUMMY_CLIENT_KEY_PEM, + certificate_chain: bytes = DUMMY_CLIENT_PEM, + ): + self.credentials = grpc.ssl_channel_credentials( + root_certificates=root_certificates, + private_key=private_key, + certificate_chain=certificate_chain, + ) + self.channel = grpc.secure_channel( + f"{host}:{port}", + self.credentials, + options=(("grpc.ssl_target_name_override", "cln"),), + ) + self.stub = pbgrpc.NodeStub(self.channel) + + def getinfo(self): + return grpc2py.getinfo2py( + self.stub.Getinfo(pb.GetinfoRequest()) + ) + + def connect(self, peer_id, host=None, port=None): + """ + Connect to {peer_id} at {host} and {port}. + """ + payload = pb.ConnectRequest( + id=peer_id, + host=host, + port=port + ) + return grpc2py.connect2py(self.stub.ConnectPeer(payload)) diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index a5f12632f..e9d903995 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -10,6 +10,7 @@ from collections import OrderedDict from decimal import Decimal from pyln.client import LightningRpc from pyln.client import Millisatoshi +from pyln.testing import grpc import ephemeral_port_reserve # type: ignore import json @@ -538,7 +539,15 @@ class ElementsD(BitcoinD): class LightningD(TailableProc): - def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, node_id=0): + def __init__( + self, + lightning_dir, + bitcoindproxy, + port=9735, + random_hsm=False, + node_id=0, + grpc_port=None + ): # We handle our own version of verbose, below. TailableProc.__init__(self, lightning_dir, verbose=False) self.executable = 'lightningd' @@ -564,6 +573,9 @@ class LightningD(TailableProc): 'bitcoin-datadir': lightning_dir, } + if grpc_port is not None: + opts['grpc-port'] = grpc_port + for k, v in opts.items(): self.opts[k] = v @@ -693,19 +705,22 @@ class LightningNode(object): self.allow_bad_gossip = allow_bad_gossip self.allow_warning = allow_warning self.db = db + self.lightning_dir = Path(lightning_dir) # Assume successful exit self.rc = 0 - socket_path = os.path.join(lightning_dir, TEST_NETWORK, "lightning-rpc").format(node_id) - self.rpc = PrettyPrintingLightningRpc(socket_path, self.executor, jsonschemas=jsonschemas) + # Ensure we have an RPC we can use to talk to the node + self._create_rpc(jsonschemas) self.gossip_store = GossipStore(Path(lightning_dir, TEST_NETWORK, "gossip_store")) self.daemon = LightningD( lightning_dir, bitcoindproxy=bitcoind.get_proxy(), - port=port, random_hsm=random_hsm, node_id=node_id + port=port, random_hsm=random_hsm, node_id=node_id, + grpc_port=self.grpc_port, ) + # If we have a disconnect string, dump it to a file for daemon. if disconnect: self.daemon.disconnect_file = os.path.join(lightning_dir, TEST_NETWORK, "dev_disconnect") @@ -751,6 +766,47 @@ class LightningNode(object): if SLOW_MACHINE: self.daemon.cmd_prefix += ['--read-inline-info=no'] + def _create_rpc(self, jsonschemas): + """Prepares anything related to the RPC. + """ + if os.environ.get('CLN_TEST_GRPC') == '1': + logging.info("Switching to GRPC based RPC for tests") + self._create_grpc_rpc() + else: + self._create_jsonrpc_rpc(jsonschemas) + + def _create_grpc_rpc(self): + self.grpc_port = reserve_unused_port() + d = self.lightning_dir / TEST_NETWORK + d.mkdir(parents=True, exist_ok=True) + + # Copy all the certificates and keys into place: + with (d / "ca.pem").open(mode='wb') as f: + f.write(grpc.DUMMY_CA_PEM) + + with (d / "ca-key.pem").open(mode='wb') as f: + f.write(grpc.DUMMY_CA_KEY_PEM) + + # Now the node will actually start up and use them, so we can + # create the RPC instance. + self.rpc = grpc.LightningGrpc( + host='localhost', + port=self.grpc_port, + root_certificates=grpc.DUMMY_CA_PEM, + private_key=grpc.DUMMY_CLIENT_KEY_PEM, + certificate_chain=grpc.DUMMY_CLIENT_PEM + ) + + def _create_jsonrpc_rpc(self, jsonschemas): + socket_path = self.lightning_dir / TEST_NETWORK / "lightning-rpc" + self.grpc_port = None + + self.rpc = PrettyPrintingLightningRpc( + str(socket_path), + self.executor, + jsonschemas=jsonschemas + ) + def connect(self, remote_node): self.rpc.connect(remote_node.info['id'], '127.0.0.1', remote_node.daemon.port) diff --git a/tools/check-spelling.sh b/tools/check-spelling.sh index 900fd87e4..c96764841 100755 --- a/tools/check-spelling.sh +++ b/tools/check-spelling.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihgtning|ligthning|lighnting|lightinng|lightnnig|lightnign' -- . ':!tools/check-spelling.sh' | grep -vE "highlighting"; then +if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihgtning|ligthning|lighnting|lightinng|lightnnig|lightnign' -- . ':!tools/check-spelling.sh' | grep -vE "highlighting|LightningGrpc"; then echo "Identified a likely misspelling of the word \"lightning\" (see above). Please fix." echo "Is this warning incorrect? Please teach tools/check-spelling.sh about the exciting new word." exit 1