From cc342fda83f07eb78187b5e4ab98366ebb0a268a Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Mon, 4 Mar 2019 13:30:35 +1030
Subject: [PATCH] pylightning: make sure UTF-8 gets passed intact.

1. We need to read in as a byte string, then decode into utf8 once we
   have a marker.  Otherwise we seem to mangle it horribly, and we
   might have a bad utf8 string anyway.

2. We need to suppress the JSON \u escapes on output.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 contrib/pylightning/lightning/plugin.py | 14 ++++++++------
 tests/test_plugin.py                    |  1 -
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/contrib/pylightning/lightning/plugin.py b/contrib/pylightning/lightning/plugin.py
index 3da175d70..bd8be1c3d 100644
--- a/contrib/pylightning/lightning/plugin.py
+++ b/contrib/pylightning/lightning/plugin.py
@@ -378,9 +378,11 @@ class Plugin(object):
             self.log(traceback.format_exc())
 
     def _write_locked(self, obj):
-        s = json.dumps(obj, cls=LightningRpc.LightningJSONEncoder) + "\n\n"
+        # ensure_ascii turns UTF-8 into \uXXXX so we need to suppress that,
+        # then utf8 ourselves.
+        s = bytes(json.dumps(obj, cls=LightningRpc.LightningJSONEncoder, ensure_ascii=False) + "\n\n", encoding='utf-8')
         with self.write_lock:
-            self.stdout.write(s)
+            self.stdout.buffer.write(s)
             self.stdout.flush()
 
     def notify(self, method, params):
@@ -415,7 +417,7 @@ class Plugin(object):
         for payload in msgs[:-1]:
             # Note that we use function annotations to do Millisatoshi conversions
             # in _exec_func, so we don't use LightningJSONDecoder here.
-            request = self._parse_request(json.loads(payload))
+            request = self._parse_request(json.loads(payload.decode('utf8')))
 
             # If this has an 'id'-field, it's a request and returns a
             # result. Otherwise it's a notification and it doesn't
@@ -428,11 +430,11 @@ class Plugin(object):
         return msgs[-1]
 
     def run(self):
-        partial = ""
-        for l in self.stdin:
+        partial = b""
+        for l in self.stdin.buffer:
             partial += l
 
-            msgs = partial.split('\n\n')
+            msgs = partial.split(b'\n\n')
             if len(msgs) < 2:
                 continue
 
diff --git a/tests/test_plugin.py b/tests/test_plugin.py
index d682fe149..cf278c86c 100644
--- a/tests/test_plugin.py
+++ b/tests/test_plugin.py
@@ -190,7 +190,6 @@ def test_async_rpcmethod(node_factory, executor):
     assert [r.result() for r in results] == [42] * len(results)
 
 
-@pytest.mark.xfail(strict=True)
 def test_utf8_passthrough(node_factory, executor):
     l1 = node_factory.get_node(options={'plugin': 'tests/plugins/utf8.py',
                                         'log-level': 'io'})