From 8b84c8945b62ee545f8dd6f6fa66b8bd41129970 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Fri, 10 Nov 2023 13:24:46 +1030
Subject: [PATCH] clnrest: don't convert *_msat fields to strings.

We have a global JSON encoder hack, which means that any field ending in msat gets special treatment (so we can safely talk to lightningd, even if a field expects satoshi amounts, we are explicit).  However, requests uses the JSON parser and DOES NOT want this conversion when sending it out as an HTTP response!

The simplest local fix we could find was Shahana's suggestion to iterate and covert away from Millisatoshi(): the reverse of what our JSON encoder does.

Fixes: https://github.com/ElementsProject/lightning/issues/6848
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 plugins/clnrest/utilities/shared.py | 24 +++++++++++++++++++++++-
 tests/test_clnrest.py               | 22 ++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/plugins/clnrest/utilities/shared.py b/plugins/clnrest/utilities/shared.py
index 3b51369e1..f58e0026d 100644
--- a/plugins/clnrest/utilities/shared.py
+++ b/plugins/clnrest/utilities/shared.py
@@ -1,5 +1,6 @@
 import json5
 import ipaddress
+import pyln.client
 
 
 CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT, REST_CSP, REST_CORS_ORIGINS = "", "", "", "", "", []
@@ -64,8 +65,29 @@ def set_config(options):
     return None
 
 
+def convert_millisatoshis(item):
+    """
+    The global JSON encoder has been replaced (see
+    monkey_patch_json!)  by one that turns Millisatoshi class object
+    into strings ending in msat.  We do not want the http response
+    to be encoded like that!  pyln-client should probably not do that,
+    but meanwhile, convert them to integers.
+    """
+    if isinstance(item, dict):
+        ret = {}
+        for k in item:
+            ret[k] = convert_millisatoshis(item[k])
+    elif isinstance(item, list):
+        ret = [convert_millisatoshis(i) for i in item]
+    elif isinstance(item, pyln.client.Millisatoshi):
+        ret = int(item)
+    else:
+        ret = item
+    return ret
+
+
 def call_rpc_method(plugin, rpc_method, payload):
-    return plugin.rpc.call(rpc_method, payload)
+    return convert_millisatoshis(plugin.rpc.call(rpc_method, payload))
 
 
 def verify_rune(plugin, rune, rpc_method, rpc_params):
diff --git a/tests/test_clnrest.py b/tests/test_clnrest.py
index da9ee654e..4b692e9f7 100644
--- a/tests/test_clnrest.py
+++ b/tests/test_clnrest.py
@@ -225,6 +225,28 @@ def test_clnrest_rpc_method(node_factory):
     assert 'bolt11' in response.json()
 
 
+def test_clnrest_large_response(node_factory):
+    """Test a large reply still works (and msat fields are integers!)"""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+    http_session = http_session_with_retry()
+
+    # Add 500 invoices, test list
+    NUM_INVOICES = 500
+    for i in range(NUM_INVOICES):
+        l1.rpc.invoice(amount_msat=100, label=str(i), description="inv")
+
+    rune = l1.rpc.createrune()['rune']
+    response = http_session.post(base_url + '/v1/listinvoices', headers={'Rune': rune},
+                                 verify=ca_cert)
+    # No, this doesn't return JSON, it *parses* it into a Python object!
+    resp = response.json()
+
+    # Make sure it hasn't turned msat fields into strings!
+    assert not isinstance(resp['invoices'][0]['amount_msat'], Millisatoshi)
+    assert len(resp['invoices']) == NUM_INVOICES
+
+
 # Tests for websocket are written separately to avoid flake8
 # to complain with the errors F811 like this "F811 redefinition of
 # unused 'message'".