common: make json_parse_input API retry friendly.

The jsmn parser is a beautiful piece of code.  In particular, you can parse
part of a string, then continue where you left off.

We don't take advantage of this, however, meaning for large JSON objects
we parse them multiple times before finally having enough to complete.

Expose the parser state and tokens through the API, so the caller can pass
them in repeatedly.  For the moment, every caller is allocates each time
(except the unit tests).

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/json.c

@@ -539,60 +539,74 @@ validate_jsmn_parse_output(const jsmntok_t *p, const jsmntok_t *end)
 JSMN Result Validation Ends
 -----------------------------------------------------------------------------*/
 
-jsmntok_t *json_parse_input(const tal_t *ctx,
-			    const char *input, int len, bool *valid)
+void toks_reset(jsmntok_t *toks)
+{
+	assert(tal_count(toks) >= 1);
+	toks[0].type = JSMN_UNDEFINED;
+}
+
+jsmntok_t *toks_alloc(const tal_t *ctx)
+{
+	jsmntok_t *toks = tal_arr(ctx, jsmntok_t, 10);
+	toks_reset(toks);
+	return toks;
+}
+
+bool json_parse_input(jsmn_parser *parser,
+		      jsmntok_t **toks,
+		      const char *input, int len,
+		      bool *complete)
 {
-	jsmn_parser parser;
-	jsmntok_t *toks;
 	int ret;
 
-	toks = tal_arr(ctx, jsmntok_t, 10);
-	toks[0].type = JSMN_UNDEFINED;
-
-	jsmn_init(&parser);
 again:
-	ret = jsmn_parse(&parser, input, len, toks, tal_count(toks) - 1);
+	ret = jsmn_parse(parser, input, len, *toks, tal_count(*toks) - 1);
 
 	switch (ret) {
 	case JSMN_ERROR_INVAL:
-		*valid = false;
-		return tal_free(toks);
+		return false;
 	case JSMN_ERROR_NOMEM:
-		tal_resize(&toks, tal_count(toks) * 2);
+		tal_resize(toks, tal_count(*toks) * 2);
 		goto again;
 	}
 
 	/* Check whether we read at least one full root element, i.e., root
 	 * element has its end set. */
-	if (toks[0].type == JSMN_UNDEFINED || toks[0].end == -1) {
-		*valid = true;
-		return tal_free(toks);
+	if ((*toks)[0].type == JSMN_UNDEFINED || (*toks)[0].end == -1) {
+		*complete = false;
+		return true;
 	}
 
 	/* If we read a partial element at the end of the stream we'll get a
 	 * ret=JSMN_ERROR_PART, but due to the previous check we know we read at
 	 * least one full element, so count tokens that are part of this root
 	 * element. */
-	ret = json_next(toks) - toks;
+	ret = json_next(*toks) - *toks;
+
+	if (!validate_jsmn_parse_output(*toks, *toks + ret))
+		return false;
 
 	/* Cut to length and return. */
-	*valid = validate_jsmn_parse_output(toks, toks + ret);
-	tal_resize(&toks, ret + 1);
+	tal_resize(toks, ret + 1);
 	/* Make sure last one is always referenceable. */
-	toks[ret].type = -1;
-	toks[ret].start = toks[ret].end = toks[ret].size = 0;
+	(*toks)[ret].type = -1;
+	(*toks)[ret].start = (*toks)[ret].end = (*toks)[ret].size = 0;
 
-	return toks;
+	*complete = true;
+	return true;
 }
 
 jsmntok_t *json_parse_simple(const tal_t *ctx, const char *input, int len)
 {
-	bool valid;
-	jsmntok_t *toks;
+	bool complete;
+	jsmn_parser parser;
+	jsmntok_t *toks = toks_alloc(ctx);
 
-	toks = json_parse_input(ctx, input, len, &valid);
-	if (toks && !valid)
-		toks = tal_free(toks);
+	jsmn_init(&parser);
+
+	if (!json_parse_input(&parser, &toks, input, len, &complete)
+	    || !complete)
+		return tal_free(toks);
 	return toks;
 }
 

common/json.h

@@ -85,9 +85,35 @@ const jsmntok_t *json_get_member(const char *buffer, const jsmntok_t tok[],
 /* Get index'th array member. */
 const jsmntok_t *json_get_arr(const jsmntok_t tok[], size_t index);
 
-/* If input is complete and valid, return tokens. */
-jsmntok_t *json_parse_input(const tal_t *ctx,
-			    const char *input, int len, bool *valid);
+/* Allocate a starter array of tokens for json_parse_input */
+jsmntok_t *toks_alloc(const tal_t *ctx);
+
+/* Reset a token array to reuse it. */
+void toks_reset(jsmntok_t *toks);
+
+/**
+ * json_parse_input: parse and validate JSON.
+ * @parser: parser initialized with jsmn_init.
+ * @toks: tallocated array from toks_alloc()
+ * @input, @len: input string.
+ * @complete: set to true if the valid JSON is complete, or NULL if must be.
+ *
+ * This returns false if the JSON is invalid, true otherwise.
+ * If it returns true, *@complete indicates that (*@toks)[0] points to a
+ * valid, complete JSON element.  If @complete is NULL, then incomplete
+ * JSON returns false (i.e. is considered invalid).
+ *
+ * *@toks is resized to the complete set of tokens, with a dummy
+ * terminator (type == -1) at the end.
+ *
+ * If it returns true, and *@complete is false, you can append more
+ * data to @input and call it again (with the same perser) and the parser
+ * will continue where it left off.
+*/
+bool json_parse_input(jsmn_parser *parser,
+		      jsmntok_t **toks,
+		      const char *input, int len,
+		      bool *complete);
 
 /* Simplified version of above which parses only a complete, valid
  * JSON string */

common/test/run-json.c

@@ -104,34 +104,47 @@ static int test_json_tok_millionths(void)
 
 static void test_json_tok_size(void)
 {
-	const jsmntok_t *toks;
+	jsmntok_t *toks;
 	char *buf;
-	bool ok;
+	bool ok, complete;
+	jsmn_parser parser;
 
 	buf = "[\"e1\", [\"e2\", \"e3\"]]";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks = toks_alloc(tmpctx);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 2);
 	assert(toks[2].size == 2);
 
 	buf = "[[\"e1\", \"e2\"], \"e3\"]";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 2);
 	assert(toks[1].size == 2);
 
 	buf = "{\"e1\" : {\"e2\": 2, \"e3\": 3}}";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 1);
 	assert(toks[2].size == 2);
 
 	buf = "{\"e1\" : {\"e2\": 2, \"e3\": 3}, \"e4\" : {\"e5\": 5, \"e6\": 6}}";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 2);
 	assert(toks[2].size == 2);
@@ -139,7 +152,9 @@ static void test_json_tok_size(void)
 
 	/* This should *not* parse! (used to give toks[0]->size == 3!) */
 	buf = "{ \"\" \"\" \"\" }";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(!ok);
 
 	/* This should *not* parse! (used to give toks[0]->size == 2!) */

lightningd/jsonrpc.c

@@ -913,7 +913,8 @@ static struct io_plan *read_json(struct io_conn *conn,
 				 struct json_connection *jcon)
 {
 	jsmntok_t *toks;
-	bool valid;
+	bool complete;
+	jsmn_parser parser;
 
 	if (jcon->len_read)
 		log_io(jcon->log, LOG_IO_IN, NULL, "",
@@ -930,21 +931,18 @@ static struct io_plan *read_json(struct io_conn *conn,
 		return io_wait(conn, conn, read_json, jcon);
 	}
 
-	toks = json_parse_input(jcon->buffer, jcon->buffer, jcon->used, &valid);
-	if (!toks) {
-		if (!valid) {
-			log_unusual(jcon->log,
-				    "Invalid token in json input: '%.*s'",
-				    (int)jcon->used, jcon->buffer);
-			json_command_malformed(
-			    jcon, "null",
-			    "Invalid token in json input");
-			return io_halfclose(conn);
-		}
-		/* We need more. */
-		goto read_more;
+	toks = toks_alloc(jcon->buffer);
+	jsmn_init(&parser);
+	if (!json_parse_input(&parser, &toks, jcon->buffer, jcon->used,
+			      &complete)) {
+		json_command_malformed(jcon, "null",
+				       "Invalid token in json input");
+		return io_halfclose(conn);
 	}
 
+	if (!complete)
+		goto read_more;
+
 	/* Empty buffer? (eg. just whitespace). */
 	if (tal_count(toks) == 1) {
 		jcon->used = 0;

lightningd/plugin.c

@@ -386,10 +386,11 @@ static const char *plugin_read_json_one(struct plugin *plugin,
 					bool *complete,
 					bool *destroyed)
 {
-	bool valid;
-	const jsmntok_t *toks, *jrtok, *idtok;
+	jsmntok_t *toks;
+	const jsmntok_t *jrtok, *idtok;
 	struct plugin_destroyed *pd;
 	const char *err;
+	jsmn_parser parser;
 
 	*destroyed = false;
 	/* Note that in the case of 'plugin stop' this can free request (since
@@ -398,28 +399,30 @@ static const char *plugin_read_json_one(struct plugin *plugin,
 	/* FIXME: This could be done more efficiently by storing the
 	 * toks and doing an incremental parse, like lightning-cli
 	 * does. */
-	toks = json_parse_input(plugin->buffer, plugin->buffer, plugin->used,
-				&valid);
-	if (!toks) {
-		if (!valid) {
-			return tal_fmt(plugin,
-				       "Failed to parse JSON response '%.*s'",
-				       (int)plugin->used, plugin->buffer);
-		}
+	toks = toks_alloc(plugin);
+	jsmn_init(&parser);
+	if (!json_parse_input(&parser, &toks, plugin->buffer, plugin->used,
+			      complete)) {
+		return tal_fmt(plugin,
+			       "Failed to parse JSON response '%.*s'",
+			       (int)plugin->used, plugin->buffer);
+	}
+
+	if (!*complete) {
 		/* We need more. */
-		*complete = false;
+		tal_free(toks);
 		return NULL;
 	}
 
 	/* Empty buffer? (eg. just whitespace). */
 	if (tal_count(toks) == 1) {
+		tal_free(toks);
 		plugin->used = 0;
 		/* We need more. */
 		*complete = false;
 		return NULL;
 	}
 
-	*complete = true;
 	jrtok = json_get_member(plugin->buffer, toks, "jsonrpc");
 	idtok = json_get_member(plugin->buffer, toks, "id");
 

lightningd/test/run-jsonrpc.c

@@ -132,11 +132,12 @@ static int test_json_filter(void)
 	struct json_stream *result = new_json_stream(NULL, NULL, NULL);
 	jsmntok_t *toks;
 	const jsmntok_t *x;
-	bool valid;
+	bool valid, complete;
 	int i;
 	char *badstr = tal_arr(result, char, 256);
 	const char *str;
 	size_t len;
+	jsmn_parser parser;
 
 	/* Fill with junk, and nul-terminate (256 -> 0) */
 	for (i = 1; i < 257; i++)
@@ -150,9 +151,11 @@ static int test_json_filter(void)
 	str = json_out_contents(result->jout, &len);
 	str = tal_strndup(result, str, len);
 
-	toks = json_parse_input(str, str, strlen(str), &valid);
+	toks = toks_alloc(str);
+	jsmn_init(&parser);
+	valid = json_parse_input(&parser, &toks, str, strlen(str), &complete);
 	assert(valid);
-	assert(toks);
+	assert(complete);
 
 	assert(toks[0].type == JSMN_OBJECT);
 	x = json_get_member(str, toks, "x");
@@ -240,32 +243,54 @@ static void test_json_partial(void)
 /* Test that we can segment and parse a stream of json objects correctly. */
 static void test_json_stream(void)
 {
-	bool valid;
+	bool valid, complete;
 	char *input, *talstr;
 	jsmntok_t *toks;
+	jsmn_parser parser;
 
 	/* Multiple full messages in a single buffer (happens when buffer
 	 * boundary coincides with message boundary, or read returned after
 	 * timeout. */
 	input = "{\"x\":\"x\"}{\"y\":\"y\"}";
 	talstr = tal_strndup(NULL, input, strlen(input));
-	toks = json_parse_input(talstr, talstr, strlen(talstr), &valid);
-	assert(toks);
+	toks = toks_alloc(NULL);
+	jsmn_init(&parser);
+	valid = json_parse_input(&parser, &toks, talstr, strlen(talstr), &complete);
 	assert(tal_count(toks) == 4);
 	assert(toks[0].start == 0 && toks[0].end == 9);
 	assert(valid);
+	assert(complete);
 	tal_free(talstr);
 
 	/* Multiple messages, and the last one is partial, far more likely than
 	 * accidentally getting the boundaries to match. */
 	input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z";
 	talstr = tal_strndup(NULL, input, strlen(input));
-	toks = json_parse_input(talstr, talstr, strlen(talstr), &valid);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	valid = json_parse_input(&parser, &toks, talstr, strlen(talstr), &complete);
 	assert(toks);
 	assert(tal_count(toks) == 4);
 	assert(toks[0].start == 0 && toks[0].end == 9);
 	assert(valid);
+	assert(complete);
 	tal_free(talstr);
+
+	/* We can do this incrementally, too. */
+	toks_reset(toks);
+	input = "{\"x\":\"x\"}";
+	jsmn_init(&parser);
+	for (size_t i = 0; i <= strlen(input); i++) {
+		valid = json_parse_input(&parser, &toks, input, i, &complete);
+		assert(valid);
+		if (i == strlen(input))
+			assert(complete);
+		else
+			assert(!complete);
+	}
+	assert(tal_count(toks) == 4);
+	assert(toks[0].start == 0 && toks[0].end == 9);
+	tal_free(toks);
 }
 
 int main(void)

plugins/libplugin.c

@@ -1089,21 +1089,25 @@ static void ld_command_handle(struct plugin *plugin,
  */
 static bool ld_read_json_one(struct plugin *plugin)
 {
-	bool valid;
-	const jsmntok_t *toks;
+	bool complete;
+	jsmntok_t *toks;
 	struct command *cmd = tal(plugin, struct command);
+	jsmn_parser parser;
 
 	/* FIXME: This could be done more efficiently by storing the
 	 * toks and doing an incremental parse, like lightning-cli
 	 * does. */
-	toks = json_parse_input(NULL, plugin->buffer, plugin->used,
-				&valid);
-	if (!toks) {
-		if (!valid) {
-			plugin_err(plugin, "Failed to parse JSON response '%.*s'",
-				   (int)plugin->used, plugin->buffer);
-			return false;
-		}
+	toks = toks_alloc(NULL);
+	jsmn_init(&parser);
+	if (!json_parse_input(&parser, &toks, plugin->buffer, plugin->used,
+			      &complete)) {
+		tal_free(toks);
+		plugin_err(plugin, "Failed to parse JSON response '%.*s'",
+			   (int)plugin->used, plugin->buffer);
+		return false;
+	}
+
+	if (!complete) {
 		/* We need more. */
 		return false;
 	}