From 305795b01eea08f4cc7c176ca7ae523b76ececad Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 19 Oct 2018 11:47:48 +1030 Subject: [PATCH] common/json: move JSON creation routines into lightningd/ It's the only user of them, and it's going to get optimized. Signed-off-by: Rusty Russell gossip.pydiff --git a/common/test/run-json.c b/common/test/run-json.c index 956fdda35..db52d6b01 100644 --- cli/Makefile | 3 +- cli/test/Makefile | 1 - common/Makefile | 1 - common/json.c | 237 -------------------- common/json.h | 50 ----- common/test/run-json.c | 142 ------------ lightningd/Makefile | 2 +- lightningd/gossip_control.c | 2 +- lightningd/invoice.c | 2 +- lightningd/json.c | 237 ++++++++++++++++++++ lightningd/json.h | 51 +++++ {common => lightningd}/json_escaped.c | 2 +- {common => lightningd}/json_escaped.h | 6 +- lightningd/jsonrpc.c | 2 +- lightningd/lightningd.c | 1 + lightningd/lightningd.h | 1 - lightningd/options.c | 2 +- lightningd/peer_control.c | 1 - lightningd/peer_htlcs.c | 2 +- lightningd/test/Makefile | 1 + lightningd/test/run-invoice-select-inchan.c | 3 - lightningd/test/run-json_escaped.c | 46 ++++ lightningd/test/run-jsonrpc.c | 218 ++++++++++++++++++ lightningd/test/run-param.c | 16 +- wallet/db.c | 2 +- 25 files changed, 573 insertions(+), 458 deletions(-) rename {common => lightningd}/json_escaped.c (98%) rename {common => lightningd}/json_escaped.h (88%) create mode 100644 lightningd/test/run-json_escaped.c create mode 100644 lightningd/test/run-jsonrpc.c diff --git a/cli/Makefile b/cli/Makefile index 8dbfe603d..5f830d5f8 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -4,8 +4,7 @@ LIGHTNING_CLI_OBJS := $(LIGHTNING_CLI_SRC:.c=.o) LIGHTNING_CLI_COMMON_OBJS := \ common/configdir.o \ common/json.o \ - common/json_escaped.o \ - common/utils.o \ + common/utils.o \ common/version.o lightning-cli-all: cli/lightning-cli diff --git a/cli/test/Makefile b/cli/test/Makefile index abae776a8..5846cdff6 100644 --- a/cli/test/Makefile +++ b/cli/test/Makefile @@ -12,7 +12,6 @@ CLI_TEST_COMMON_OBJS := \ common/daemon_conn.o \ common/htlc_state.o \ common/json.o \ - common/json_escaped.o \ common/pseudorand.o \ common/memleak.o \ common/msg_queue.o \ diff --git a/common/Makefile b/common/Makefile index dd6bf5dfd..e7f098d05 100644 --- a/common/Makefile +++ b/common/Makefile @@ -25,7 +25,6 @@ COMMON_SRC_NOGEN := \ common/initial_commit_tx.c \ common/io_lock.c \ common/json.c \ - common/json_escaped.c \ common/key_derive.c \ common/keyset.c \ common/memleak.c \ diff --git a/common/json.c b/common/json.c index f741f7962..f3993e370 100644 --- a/common/json.c +++ b/common/json.c @@ -1,6 +1,5 @@ /* JSON core and helpers */ #include "json.h" -#include "json_escaped.h" #include #include #include @@ -11,14 +10,6 @@ #include #include -struct json_result { - /* tal_arr of types we're enclosed in. */ - jsmntype_t *wrapping; - - /* tal_count() of this is strlen() + 1 */ - char *s; -}; - const char *json_tok_contents(const char *buffer, const jsmntok_t *t) { if (t->type == JSMN_STRING) @@ -225,231 +216,3 @@ again: return toks; } - -static void result_append(struct json_result *res, const char *str) -{ - size_t len = tal_count(res->s) - 1; - - tal_resize(&res->s, len + strlen(str) + 1); - strcpy(res->s + len, str); -} - -static void PRINTF_FMT(2,3) -result_append_fmt(struct json_result *res, const char *fmt, ...) -{ - size_t len = tal_count(res->s) - 1, fmtlen; - va_list ap; - - va_start(ap, fmt); - fmtlen = vsnprintf(NULL, 0, fmt, ap); - va_end(ap); - - tal_resize(&res->s, len + fmtlen + 1); - va_start(ap, fmt); - vsprintf(res->s + len, fmt, ap); - va_end(ap); -} - -static bool result_ends_with(struct json_result *res, const char *str) -{ - size_t len = tal_count(res->s) - 1; - - if (strlen(str) > len) - return false; - return streq(res->s + len - strlen(str), str); -} - -static void check_fieldname(const struct json_result *result, - const char *fieldname) -{ - size_t n = tal_count(result->wrapping); - if (n == 0) - /* Can't have a fieldname if not in anything! */ - assert(!fieldname); - else if (result->wrapping[n-1] == JSMN_ARRAY) - /* No fieldnames in arrays. */ - assert(!fieldname); - else - /* Must have fieldnames in objects. */ - assert(fieldname); -} - -static void result_add_indent(struct json_result *result); - - static void json_start_member(struct json_result *result, const char *fieldname) -{ - /* Prepend comma if required. */ - if (result->s[0] - && !result_ends_with(result, "{") - && !result_ends_with(result, "[")) - result_append(result, ", \n"); - else - result_append(result, "\n"); - - result_add_indent(result); - - check_fieldname(result, fieldname); - if (fieldname) - result_append_fmt(result, "\"%s\": ", fieldname); -} - -static void result_add_indent(struct json_result *result) -{ - size_t i, indent = tal_count(result->wrapping); - - if (!indent) - return; - - for (i = 0; i < indent; i++) - result_append(result, " "); -} - -static void result_add_wrap(struct json_result *result, jsmntype_t type) -{ - size_t indent = tal_count(result->wrapping); - - tal_resize(&result->wrapping, indent+1); - result->wrapping[indent] = type; -} - -static void result_pop_wrap(struct json_result *result, jsmntype_t type) -{ - size_t indent = tal_count(result->wrapping); - - assert(indent); - assert(result->wrapping[indent-1] == type); - tal_resize(&result->wrapping, indent-1); -} - -void json_array_start(struct json_result *result, const char *fieldname) -{ - json_start_member(result, fieldname); - result_append(result, "["); - result_add_wrap(result, JSMN_ARRAY); -} - -void json_array_end(struct json_result *result) -{ - result_append(result, "\n"); - result_pop_wrap(result, JSMN_ARRAY); - result_add_indent(result); - result_append(result, "]"); -} - -void json_object_start(struct json_result *result, const char *fieldname) -{ - json_start_member(result, fieldname); - result_append(result, "{"); - result_add_wrap(result, JSMN_OBJECT); -} - -void json_object_end(struct json_result *result) -{ - result_append(result, "\n"); - result_pop_wrap(result, JSMN_OBJECT); - result_add_indent(result); - result_append(result, "}"); -} - -void json_add_num(struct json_result *result, const char *fieldname, unsigned int value) -{ - json_start_member(result, fieldname); - result_append_fmt(result, "%u", value); -} - -void json_add_double(struct json_result *result, const char *fieldname, double value) -{ - json_start_member(result, fieldname); - result_append_fmt(result, "%f", value); -} - -void json_add_u64(struct json_result *result, const char *fieldname, - uint64_t value) -{ - json_start_member(result, fieldname); - result_append_fmt(result, "%"PRIu64, value); -} - -void json_add_literal(struct json_result *result, const char *fieldname, - const char *literal, int len) -{ - json_start_member(result, fieldname); - result_append_fmt(result, "%.*s", len, literal); -} - -void json_add_string(struct json_result *result, const char *fieldname, const char *value) -{ - struct json_escaped *esc = json_partial_escape(NULL, value); - - json_start_member(result, fieldname); - result_append_fmt(result, "\"%s\"", esc->s); - tal_free(esc); -} - -void json_add_bool(struct json_result *result, const char *fieldname, bool value) -{ - json_start_member(result, fieldname); - result_append(result, value ? "true" : "false"); -} - -void json_add_hex(struct json_result *result, const char *fieldname, - const void *data, size_t len) -{ - char *hex = tal_arr(NULL, char, hex_str_size(len)); - - hex_encode(data, len, hex, hex_str_size(len)); - json_add_string(result, fieldname, hex); - tal_free(hex); -} - -void json_add_hex_talarr(struct json_result *result, - const char *fieldname, - const tal_t *data) -{ - json_add_hex(result, fieldname, data, tal_bytelen(data)); -} - -void json_add_object(struct json_result *result, ...) -{ - va_list ap; - const char *field; - - va_start(ap, result); - json_object_start(result, NULL); - while ((field = va_arg(ap, const char *)) != NULL) { - jsmntype_t type = va_arg(ap, jsmntype_t); - const char *value = va_arg(ap, const char *); - if (type == JSMN_STRING) - json_add_string(result, field, value); - else - json_add_literal(result, field, value, strlen(value)); - } - json_object_end(result); - va_end(ap); -} - -void json_add_escaped_string(struct json_result *result, const char *fieldname, - const struct json_escaped *esc TAKES) -{ - json_start_member(result, fieldname); - result_append_fmt(result, "\"%s\"", esc->s); - if (taken(esc)) - tal_free(esc); -} - -struct json_result *new_json_result(const tal_t *ctx) -{ - struct json_result *r = tal(ctx, struct json_result); - - /* Using tal_arr means that it has a valid count. */ - r->s = tal_arrz(r, char, 1); - r->wrapping = tal_arr(r, jsmntype_t, 0); - return r; -} - -const char *json_result_string(const struct json_result *result) -{ - assert(tal_count(result->wrapping) == 0); - assert(tal_count(result->s) == strlen(result->s) + 1); - return result->s; -} diff --git a/common/json.h b/common/json.h index c5e5b048e..e27929a29 100644 --- a/common/json.h +++ b/common/json.h @@ -12,7 +12,6 @@ # include struct json_escaped; -struct json_result; struct short_channel_id; /* Include " if it's a string. */ @@ -58,53 +57,4 @@ const jsmntok_t *json_get_arr(const jsmntok_t tok[], size_t index); /* If input is complete and valid, return tokens. */ jsmntok_t *json_parse_input(const char *input, int len, bool *valid); -/* Creating JSON strings */ - -/* '"fieldname" : [ ' or '[ ' if fieldname is NULL */ -void json_array_start(struct json_result *ptr, const char *fieldname); -/* '"fieldname" : { ' or '{ ' if fieldname is NULL */ -void json_object_start(struct json_result *ptr, const char *fieldname); -/* ' ], ' */ -void json_array_end(struct json_result *ptr); -/* ' }, ' */ -void json_object_end(struct json_result *ptr); - -struct json_result *new_json_result(const tal_t *ctx); - -/* '"fieldname" : "value"' or '"value"' if fieldname is NULL. Turns - * any non-printable chars into JSON escapes, but leaves existing escapes alone. - */ -void json_add_string(struct json_result *result, const char *fieldname, const char *value); - -/* '"fieldname" : "value"' or '"value"' if fieldname is NULL. String must - * already be JSON escaped as necessary. */ -void json_add_escaped_string(struct json_result *result, - const char *fieldname, - const struct json_escaped *esc TAKES); - -/* '"fieldname" : literal' or 'literal' if fieldname is NULL*/ -void json_add_literal(struct json_result *result, const char *fieldname, - const char *literal, int len); -/* '"fieldname" : value' or 'value' if fieldname is NULL */ -void json_add_double(struct json_result *result, const char *fieldname, - double value); -/* '"fieldname" : value' or 'value' if fieldname is NULL */ -void json_add_num(struct json_result *result, const char *fieldname, - unsigned int value); -/* '"fieldname" : value' or 'value' if fieldname is NULL */ -void json_add_u64(struct json_result *result, const char *fieldname, - uint64_t value); -/* '"fieldname" : true|false' or 'true|false' if fieldname is NULL */ -void json_add_bool(struct json_result *result, const char *fieldname, - bool value); -/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */ -void json_add_hex(struct json_result *result, const char *fieldname, - const void *data, size_t len); -/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */ -void json_add_hex_talarr(struct json_result *result, - const char *fieldname, - const tal_t *data); -void json_add_object(struct json_result *result, ...); - -const char *json_result_string(const struct json_result *result); #endif /* LIGHTNING_COMMON_JSON_H */ diff --git a/common/test/run-json.c b/common/test/run-json.c index d1b855ed4..db52d6b01 100644 --- a/common/test/run-json.c +++ b/common/test/run-json.c @@ -1,5 +1,4 @@ #include "../json.c" -#include "../json_escaped.c" #include #include @@ -39,152 +38,11 @@ static int test_json_tok_bitcoin_amount(void) return 0; } -static int test_json_filter(void) -{ - struct json_result *result = new_json_result(NULL); - jsmntok_t *toks; - const jsmntok_t *x; - bool valid; - int i; - char *badstr = tal_arr(result, char, 256); - const char *str; - - /* Fill with junk, and nul-terminate (256 -> 0) */ - for (i = 1; i < 257; i++) - badstr[i-1] = i; - - json_object_start(result, NULL); - json_add_string(result, "x", badstr); - json_object_end(result); - - /* Parse back in, make sure nothing crazy. */ - str = json_result_string(result); - - toks = json_parse_input(str, strlen(str), &valid); - assert(valid); - assert(toks); - - assert(toks[0].type == JSMN_OBJECT); - x = json_get_member(str, toks, "x"); - assert(x); - assert(x->type == JSMN_STRING); - - /* There are 7 one-letter escapes, and (32-5) \uXXXX escapes. */ - assert((x->end - x->start) == 255 + 7*1 + (32-5)*5); - /* No control characters. */ - for (i = x->start; i < x->end; i++) { - assert((unsigned)str[i] >= ' '); - assert((unsigned)str[i] != 127); - } - tal_free(result); - return 0; -} - -static void test_json_escape(void) -{ - int i; - - for (i = 1; i < 256; i++) { - char badstr[2]; - struct json_result *result = new_json_result(NULL); - struct json_escaped *esc; - - badstr[0] = i; - badstr[1] = 0; - - json_object_start(result, NULL); - esc = json_escape(NULL, badstr); - json_add_escaped_string(result, "x", take(esc)); - json_object_end(result); - - const char *str = json_result_string(result); - if (i == '\\' || i == '"' - || i == '\n' || i == '\r' || i == '\b' - || i == '\t' || i == '\f') - assert(strstarts(str, "\n{\n \"x\": \"\\")); - else if (i < 32 || i == 127) { - assert(strstarts(str, "\n{\n \"x\": \"\\u00")); - } else { - char expect[] = "\n{\n \"x\": \"?\"\n}"; - expect[11] = i; - assert(streq(str, expect)); - } - tal_free(result); - } -} - -static void test_json_partial(void) -{ - const tal_t *ctx = tal(NULL, char); - - assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\")); - assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\")); - assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\")); - assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\")); - assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n")); - assert(streq(json_partial_escape(ctx, "\n")->s, "\\n")); - assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\"")); - assert(streq(json_partial_escape(ctx, "\"")->s, "\\\"")); - assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t")); - assert(streq(json_partial_escape(ctx, "\t")->s, "\\t")); - assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b")); - assert(streq(json_partial_escape(ctx, "\b")->s, "\\b")); - assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r")); - assert(streq(json_partial_escape(ctx, "\r")->s, "\\r")); - assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f")); - assert(streq(json_partial_escape(ctx, "\f")->s, "\\f")); - /* You're allowed to escape / according to json.org. */ - assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/")); - assert(streq(json_partial_escape(ctx, "/")->s, "/")); - - assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF")); - assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx")); - - /* Unknown escapes should be escaped. */ - assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x")); - tal_free(ctx); -} - -/* Test that we can segment and parse a stream of json objects correctly. */ -static void test_json_stream(void) -{ - bool valid; - char *input, *talstr; - jsmntok_t *toks; - - /* Multiple full messages in a single buffer (happens when buffer - * boundary coincides with message boundary, or read returned after - * timeout. */ - input = "{\"x\":\"x\"}{\"y\":\"y\"}"; - talstr = tal_strndup(NULL, input, strlen(input)); - toks = json_parse_input(talstr, strlen(talstr), &valid); - assert(toks); - assert(tal_count(toks) == 4); - assert(toks[0].start == 0 && toks[0].end == 9); - assert(valid); - tal_free(talstr); - - /* Multiple messages, and the last one is partial, far more likely than - * accidentally getting the boundaries to match. */ - input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z"; - talstr = tal_strndup(NULL, input, strlen(input)); - toks = json_parse_input(talstr, strlen(talstr), &valid); - assert(toks); - assert(tal_count(toks) == 4); - assert(toks[0].start == 0 && toks[0].end == 9); - assert(valid); - tal_free(talstr); -} - int main(void) { setup_locale(); test_json_tok_bitcoin_amount(); - test_json_filter(); - test_json_escape(); - test_json_partial(); - test_json_stream(); assert(!taken_any()); take_cleanup(); } diff --git a/lightningd/Makefile b/lightningd/Makefile index ffee6cda7..8ff4cdd91 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -34,7 +34,6 @@ LIGHTNINGD_COMMON_OBJS := \ common/key_derive.o \ common/io_lock.o \ common/json.o \ - common/json_escaped.o \ common/memleak.o \ common/msg_queue.o \ common/permute_tx.o \ @@ -65,6 +64,7 @@ LIGHTNINGD_SRC := \ lightningd/htlc_end.c \ lightningd/invoice.c \ lightningd/json.c \ + lightningd/json_escaped.c \ lightningd/jsonrpc.c \ lightningd/lightningd.c \ lightningd/log.c \ diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 23ae2c7c1..fa405d6b0 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -23,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 34d464fac..18d223e03 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/json.c b/lightningd/json.c index b0ac53246..08ebf25d3 100644 --- a/lightningd/json.c +++ b/lightningd/json.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -477,3 +478,239 @@ bool json_tok_tok(struct command *cmd, const char *name, { return (*out = tok); } + +struct json_result { + /* tal_arr of types we're enclosed in. */ + jsmntype_t *wrapping; + + /* tal_count() of this is strlen() + 1 */ + char *s; +}; + +static void result_append(struct json_result *res, const char *str) +{ + size_t len = tal_count(res->s) - 1; + + tal_resize(&res->s, len + strlen(str) + 1); + strcpy(res->s + len, str); +} + +static void PRINTF_FMT(2,3) +result_append_fmt(struct json_result *res, const char *fmt, ...) +{ + size_t len = tal_count(res->s) - 1, fmtlen; + va_list ap; + + va_start(ap, fmt); + fmtlen = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + + tal_resize(&res->s, len + fmtlen + 1); + va_start(ap, fmt); + vsprintf(res->s + len, fmt, ap); + va_end(ap); +} + +static bool result_ends_with(struct json_result *res, const char *str) +{ + size_t len = tal_count(res->s) - 1; + + if (strlen(str) > len) + return false; + return streq(res->s + len - strlen(str), str); +} + +static void check_fieldname(const struct json_result *result, + const char *fieldname) +{ + size_t n = tal_count(result->wrapping); + if (n == 0) + /* Can't have a fieldname if not in anything! */ + assert(!fieldname); + else if (result->wrapping[n-1] == JSMN_ARRAY) + /* No fieldnames in arrays. */ + assert(!fieldname); + else + /* Must have fieldnames in objects. */ + assert(fieldname); +} + +static void result_add_indent(struct json_result *result); + + static void json_start_member(struct json_result *result, const char *fieldname) +{ + /* Prepend comma if required. */ + if (result->s[0] + && !result_ends_with(result, "{") + && !result_ends_with(result, "[")) + result_append(result, ", \n"); + else + result_append(result, "\n"); + + result_add_indent(result); + + check_fieldname(result, fieldname); + if (fieldname) + result_append_fmt(result, "\"%s\": ", fieldname); +} + +static void result_add_indent(struct json_result *result) +{ + size_t i, indent = tal_count(result->wrapping); + + if (!indent) + return; + + for (i = 0; i < indent; i++) + result_append(result, " "); +} + +static void result_add_wrap(struct json_result *result, jsmntype_t type) +{ + size_t indent = tal_count(result->wrapping); + + tal_resize(&result->wrapping, indent+1); + result->wrapping[indent] = type; +} + +static void result_pop_wrap(struct json_result *result, jsmntype_t type) +{ + size_t indent = tal_count(result->wrapping); + + assert(indent); + assert(result->wrapping[indent-1] == type); + tal_resize(&result->wrapping, indent-1); +} + +void json_array_start(struct json_result *result, const char *fieldname) +{ + json_start_member(result, fieldname); + result_append(result, "["); + result_add_wrap(result, JSMN_ARRAY); +} + +void json_array_end(struct json_result *result) +{ + result_append(result, "\n"); + result_pop_wrap(result, JSMN_ARRAY); + result_add_indent(result); + result_append(result, "]"); +} + +void json_object_start(struct json_result *result, const char *fieldname) +{ + json_start_member(result, fieldname); + result_append(result, "{"); + result_add_wrap(result, JSMN_OBJECT); +} + +void json_object_end(struct json_result *result) +{ + result_append(result, "\n"); + result_pop_wrap(result, JSMN_OBJECT); + result_add_indent(result); + result_append(result, "}"); +} + +void json_add_num(struct json_result *result, const char *fieldname, unsigned int value) +{ + json_start_member(result, fieldname); + result_append_fmt(result, "%u", value); +} + +void json_add_double(struct json_result *result, const char *fieldname, double value) +{ + json_start_member(result, fieldname); + result_append_fmt(result, "%f", value); +} + +void json_add_u64(struct json_result *result, const char *fieldname, + uint64_t value) +{ + json_start_member(result, fieldname); + result_append_fmt(result, "%"PRIu64, value); +} + +void json_add_literal(struct json_result *result, const char *fieldname, + const char *literal, int len) +{ + json_start_member(result, fieldname); + result_append_fmt(result, "%.*s", len, literal); +} + +void json_add_string(struct json_result *result, const char *fieldname, const char *value) +{ + struct json_escaped *esc = json_partial_escape(NULL, value); + + json_start_member(result, fieldname); + result_append_fmt(result, "\"%s\"", esc->s); + tal_free(esc); +} + +void json_add_bool(struct json_result *result, const char *fieldname, bool value) +{ + json_start_member(result, fieldname); + result_append(result, value ? "true" : "false"); +} + +void json_add_hex(struct json_result *result, const char *fieldname, + const void *data, size_t len) +{ + char *hex = tal_arr(NULL, char, hex_str_size(len)); + + hex_encode(data, len, hex, hex_str_size(len)); + json_add_string(result, fieldname, hex); + tal_free(hex); +} + +void json_add_hex_talarr(struct json_result *result, + const char *fieldname, + const tal_t *data) +{ + json_add_hex(result, fieldname, data, tal_bytelen(data)); +} + +void json_add_object(struct json_result *result, ...) +{ + va_list ap; + const char *field; + + va_start(ap, result); + json_object_start(result, NULL); + while ((field = va_arg(ap, const char *)) != NULL) { + jsmntype_t type = va_arg(ap, jsmntype_t); + const char *value = va_arg(ap, const char *); + if (type == JSMN_STRING) + json_add_string(result, field, value); + else + json_add_literal(result, field, value, strlen(value)); + } + json_object_end(result); + va_end(ap); +} + +void json_add_escaped_string(struct json_result *result, const char *fieldname, + const struct json_escaped *esc TAKES) +{ + json_start_member(result, fieldname); + result_append_fmt(result, "\"%s\"", esc->s); + if (taken(esc)) + tal_free(esc); +} + +struct json_result *new_json_result(const tal_t *ctx) +{ + struct json_result *r = tal(ctx, struct json_result); + + /* Using tal_arr means that it has a valid count. */ + r->s = tal_arrz(r, char, 1); + r->wrapping = tal_arr(r, jsmntype_t, 0); + return r; +} + +const char *json_result_string(const struct json_result *result) +{ + assert(tal_count(result->wrapping) == 0); + assert(tal_count(result->s) == strlen(result->s) + 1); + return result->s; +} diff --git a/lightningd/json.h b/lightningd/json.h index 02902f798..ed693b45e 100644 --- a/lightningd/json.h +++ b/lightningd/json.h @@ -6,6 +6,7 @@ #define LIGHTNING_LIGHTNINGD_JSON_H #include "config.h" #include +#include #include #include #include @@ -157,4 +158,54 @@ bool json_tok_tok(struct command *cmd, const char *name, const char *buffer, const jsmntok_t * tok, const jsmntok_t **out); + +/* Creating JSON strings */ + +/* '"fieldname" : [ ' or '[ ' if fieldname is NULL */ +void json_array_start(struct json_result *ptr, const char *fieldname); +/* '"fieldname" : { ' or '{ ' if fieldname is NULL */ +void json_object_start(struct json_result *ptr, const char *fieldname); +/* ' ], ' */ +void json_array_end(struct json_result *ptr); +/* ' }, ' */ +void json_object_end(struct json_result *ptr); + +struct json_result *new_json_result(const tal_t *ctx); + +/* '"fieldname" : "value"' or '"value"' if fieldname is NULL. Turns + * any non-printable chars into JSON escapes, but leaves existing escapes alone. + */ +void json_add_string(struct json_result *result, const char *fieldname, const char *value); + +/* '"fieldname" : "value"' or '"value"' if fieldname is NULL. String must + * already be JSON escaped as necessary. */ +void json_add_escaped_string(struct json_result *result, + const char *fieldname, + const struct json_escaped *esc TAKES); + +/* '"fieldname" : literal' or 'literal' if fieldname is NULL*/ +void json_add_literal(struct json_result *result, const char *fieldname, + const char *literal, int len); +/* '"fieldname" : value' or 'value' if fieldname is NULL */ +void json_add_double(struct json_result *result, const char *fieldname, + double value); +/* '"fieldname" : value' or 'value' if fieldname is NULL */ +void json_add_num(struct json_result *result, const char *fieldname, + unsigned int value); +/* '"fieldname" : value' or 'value' if fieldname is NULL */ +void json_add_u64(struct json_result *result, const char *fieldname, + uint64_t value); +/* '"fieldname" : true|false' or 'true|false' if fieldname is NULL */ +void json_add_bool(struct json_result *result, const char *fieldname, + bool value); +/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */ +void json_add_hex(struct json_result *result, const char *fieldname, + const void *data, size_t len); +/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */ +void json_add_hex_talarr(struct json_result *result, + const char *fieldname, + const tal_t *data); +void json_add_object(struct json_result *result, ...); + +const char *json_result_string(const struct json_result *result); #endif /* LIGHTNING_LIGHTNINGD_JSON_H */ diff --git a/common/json_escaped.c b/lightningd/json_escaped.c similarity index 98% rename from common/json_escaped.c rename to lightningd/json_escaped.c index d60654b92..5e62548e5 100644 --- a/common/json_escaped.c +++ b/lightningd/json_escaped.c @@ -1,4 +1,4 @@ -#include +#include #include struct json_escaped *json_escaped_string_(const tal_t *ctx, diff --git a/common/json_escaped.h b/lightningd/json_escaped.h similarity index 88% rename from common/json_escaped.h rename to lightningd/json_escaped.h index 0f742a86f..8ef77123d 100644 --- a/common/json_escaped.h +++ b/lightningd/json_escaped.h @@ -1,5 +1,5 @@ -#ifndef LIGHTNING_COMMON_JSON_ESCAPED_H -#define LIGHTNING_COMMON_JSON_ESCAPED_H +#ifndef LIGHTNING_LIGHTNINGD_JSON_ESCAPED_H +#define LIGHTNING_LIGHTNINGD_JSON_ESCAPED_H #include "config.h" #include @@ -32,4 +32,4 @@ struct json_escaped *json_escaped_string_(const tal_t *ctx, /* Be very careful here! Can fail! Doesn't handle \u: use UTF-8 please. */ const char *json_escaped_unescape(const tal_t *ctx, const struct json_escaped *esc); -#endif /* LIGHTNING_COMMON_JSON_ESCAPED_H */ +#endif /* LIGHTNING_LIGHTNINGD_JSON_ESCAPED_H */ diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index 148e1b35f..47e5d36c0 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 6f289565b..cc9820163 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -69,6 +69,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index ff3bcff1f..9a52dd8a6 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/lightningd/options.c b/lightningd/options.c index 9172bde53..b017e8246 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -20,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 2b7923b76..7c90083bf 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 2aff5a5f3..3b32e7b5f 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -13,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/test/Makefile b/lightningd/test/Makefile index 82243c824..60cc9e551 100644 --- a/lightningd/test/Makefile +++ b/lightningd/test/Makefile @@ -11,6 +11,7 @@ LIGHTNINGD_TEST_COMMON_OBJS := \ common/bech32.o \ common/daemon_conn.o \ common/htlc_state.o \ + common/json.o \ common/pseudorand.o \ common/memleak.o \ common/msg_queue.o \ diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index 41f35a1a7..a15108e01 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -156,9 +156,6 @@ void json_array_start(struct json_result *ptr UNNEEDED, const char *fieldname UN /* Generated stub for json_escape */ struct json_escaped *json_escape(const tal_t *ctx UNNEEDED, const char *str TAKES UNNEEDED) { fprintf(stderr, "json_escape called!\n"); abort(); } -/* Generated stub for json_next */ -const jsmntok_t *json_next(const jsmntok_t *tok UNNEEDED) -{ fprintf(stderr, "json_next called!\n"); abort(); } /* Generated stub for json_object_end */ void json_object_end(struct json_result *ptr UNNEEDED) { fprintf(stderr, "json_object_end called!\n"); abort(); } diff --git a/lightningd/test/run-json_escaped.c b/lightningd/test/run-json_escaped.c new file mode 100644 index 000000000..03feab5ad --- /dev/null +++ b/lightningd/test/run-json_escaped.c @@ -0,0 +1,46 @@ +#include "../json_escaped.c" +#include +#include +#include + +static void test_json_partial(void) +{ + const tal_t *ctx = tal(NULL, char); + + assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\")); + assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\")); + assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\")); + assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\")); + assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n")); + assert(streq(json_partial_escape(ctx, "\n")->s, "\\n")); + assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\"")); + assert(streq(json_partial_escape(ctx, "\"")->s, "\\\"")); + assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t")); + assert(streq(json_partial_escape(ctx, "\t")->s, "\\t")); + assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b")); + assert(streq(json_partial_escape(ctx, "\b")->s, "\\b")); + assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r")); + assert(streq(json_partial_escape(ctx, "\r")->s, "\\r")); + assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f")); + assert(streq(json_partial_escape(ctx, "\f")->s, "\\f")); + /* You're allowed to escape / according to json.org. */ + assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/")); + assert(streq(json_partial_escape(ctx, "/")->s, "/")); + + assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF")); + assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx")); + + /* Unknown escapes should be escaped. */ + assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x")); + tal_free(ctx); +} + + +int main(void) +{ + setup_locale(); + + test_json_partial(); + assert(!taken_any()); + take_cleanup(); +} diff --git a/lightningd/test/run-jsonrpc.c b/lightningd/test/run-jsonrpc.c new file mode 100644 index 000000000..500f73c20 --- /dev/null +++ b/lightningd/test/run-jsonrpc.c @@ -0,0 +1,218 @@ +#include "../json_escaped.c" +#include "../jsonrpc.c" +#include "../json.c" + +/* AUTOGENERATED MOCKS START */ +/* Generated stub for db_begin_transaction_ */ +void db_begin_transaction_(struct db *db UNNEEDED, const char *location UNNEEDED) +{ fprintf(stderr, "db_begin_transaction_ called!\n"); abort(); } +/* Generated stub for db_commit_transaction */ +void db_commit_transaction(struct db *db UNNEEDED) +{ fprintf(stderr, "db_commit_transaction called!\n"); abort(); } +/* Generated stub for fatal */ +void fatal(const char *fmt UNNEEDED, ...) +{ fprintf(stderr, "fatal called!\n"); abort(); } +/* Generated stub for feerate_from_style */ +u32 feerate_from_style(u32 feerate UNNEEDED, enum feerate_style style UNNEEDED) +{ fprintf(stderr, "feerate_from_style called!\n"); abort(); } +/* Generated stub for feerate_name */ +const char *feerate_name(enum feerate feerate UNNEEDED) +{ fprintf(stderr, "feerate_name called!\n"); abort(); } +/* Generated stub for fmt_wireaddr_without_port */ +char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED) +{ fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); } +/* Generated stub for get_block_height */ +u32 get_block_height(const struct chain_topology *topo UNNEEDED) +{ fprintf(stderr, "get_block_height called!\n"); abort(); } +/* Generated stub for get_chainparams */ +const struct chainparams *get_chainparams(const struct lightningd *ld UNNEEDED) +{ fprintf(stderr, "get_chainparams called!\n"); abort(); } +/* Generated stub for io_lock_acquire_out_ */ +struct io_plan *io_lock_acquire_out_(struct io_conn *conn UNNEEDED, struct io_lock *lock UNNEEDED, + struct io_plan *(*next)(struct io_conn * UNNEEDED, + void *) UNNEEDED, + void *arg UNNEEDED) +{ fprintf(stderr, "io_lock_acquire_out_ called!\n"); abort(); } +/* Generated stub for io_lock_new */ +struct io_lock *io_lock_new(const tal_t *ctx UNNEEDED) +{ fprintf(stderr, "io_lock_new called!\n"); abort(); } +/* Generated stub for io_lock_release */ +void io_lock_release(struct io_lock *lock UNNEEDED) +{ fprintf(stderr, "io_lock_release called!\n"); abort(); } +/* Generated stub for json_feerate_estimate */ +bool json_feerate_estimate(struct command *cmd UNNEEDED, + u32 **feerate_per_kw UNNEEDED, enum feerate feerate UNNEEDED) +{ fprintf(stderr, "json_feerate_estimate called!\n"); abort(); } +/* Generated stub for log_ */ +void log_(struct log *log UNNEEDED, enum log_level level UNNEEDED, const char *fmt UNNEEDED, ...) + +{ fprintf(stderr, "log_ called!\n"); abort(); } +/* Generated stub for log_io */ +void log_io(struct log *log UNNEEDED, enum log_level dir UNNEEDED, const char *comment UNNEEDED, + const void *data UNNEEDED, size_t len UNNEEDED) +{ fprintf(stderr, "log_io called!\n"); abort(); } +/* Generated stub for log_prefix */ +const char *log_prefix(const struct log *log UNNEEDED) +{ fprintf(stderr, "log_prefix called!\n"); abort(); } +/* Generated stub for new_log */ +struct log *new_log(const tal_t *ctx UNNEEDED, struct log_book *record UNNEEDED, const char *fmt UNNEEDED, ...) +{ fprintf(stderr, "new_log called!\n"); abort(); } +/* Generated stub for param */ +bool param(struct command *cmd UNNEEDED, const char *buffer UNNEEDED, + const jsmntok_t params[] UNNEEDED, ...) +{ fprintf(stderr, "param called!\n"); abort(); } +/* Generated stub for version */ +const char *version(void) +{ fprintf(stderr, "version called!\n"); abort(); } +/* AUTOGENERATED MOCKS END */ + +bool deprecated_apis; + +static int test_json_filter(void) +{ + struct json_result *result = new_json_result(NULL); + jsmntok_t *toks; + const jsmntok_t *x; + bool valid; + int i; + char *badstr = tal_arr(result, char, 256); + const char *str; + + /* Fill with junk, and nul-terminate (256 -> 0) */ + for (i = 1; i < 257; i++) + badstr[i-1] = i; + + json_object_start(result, NULL); + json_add_string(result, "x", badstr); + json_object_end(result); + + /* Parse back in, make sure nothing crazy. */ + str = json_result_string(result); + + toks = json_parse_input(str, strlen(str), &valid); + assert(valid); + assert(toks); + + assert(toks[0].type == JSMN_OBJECT); + x = json_get_member(str, toks, "x"); + assert(x); + assert(x->type == JSMN_STRING); + + /* There are 7 one-letter escapes, and (32-5) \uXXXX escapes. */ + assert((x->end - x->start) == 255 + 7*1 + (32-5)*5); + /* No control characters. */ + for (i = x->start; i < x->end; i++) { + assert((unsigned)str[i] >= ' '); + assert((unsigned)str[i] != 127); + } + tal_free(result); + return 0; +} + +static void test_json_escape(void) +{ + int i; + + for (i = 1; i < 256; i++) { + char badstr[2]; + struct json_result *result = new_json_result(NULL); + struct json_escaped *esc; + + badstr[0] = i; + badstr[1] = 0; + + json_object_start(result, NULL); + esc = json_escape(NULL, badstr); + json_add_escaped_string(result, "x", take(esc)); + json_object_end(result); + + const char *str = json_result_string(result); + if (i == '\\' || i == '"' + || i == '\n' || i == '\r' || i == '\b' + || i == '\t' || i == '\f') + assert(strstarts(str, "\n{\n \"x\": \"\\")); + else if (i < 32 || i == 127) { + assert(strstarts(str, "\n{\n \"x\": \"\\u00")); + } else { + char expect[] = "\n{\n \"x\": \"?\"\n}"; + expect[11] = i; + assert(streq(str, expect)); + } + tal_free(result); + } +} + +static void test_json_partial(void) +{ + const tal_t *ctx = tal(NULL, char); + + assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\")); + assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\")); + assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\")); + assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\")); + assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n")); + assert(streq(json_partial_escape(ctx, "\n")->s, "\\n")); + assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\"")); + assert(streq(json_partial_escape(ctx, "\"")->s, "\\\"")); + assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t")); + assert(streq(json_partial_escape(ctx, "\t")->s, "\\t")); + assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b")); + assert(streq(json_partial_escape(ctx, "\b")->s, "\\b")); + assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r")); + assert(streq(json_partial_escape(ctx, "\r")->s, "\\r")); + assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f")); + assert(streq(json_partial_escape(ctx, "\f")->s, "\\f")); + /* You're allowed to escape / according to json.org. */ + assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/")); + assert(streq(json_partial_escape(ctx, "/")->s, "/")); + + assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF")); + assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx")); + + /* Unknown escapes should be escaped. */ + assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x")); + tal_free(ctx); +} + +/* Test that we can segment and parse a stream of json objects correctly. */ +static void test_json_stream(void) +{ + bool valid; + char *input, *talstr; + jsmntok_t *toks; + + /* Multiple full messages in a single buffer (happens when buffer + * boundary coincides with message boundary, or read returned after + * timeout. */ + input = "{\"x\":\"x\"}{\"y\":\"y\"}"; + talstr = tal_strndup(NULL, input, strlen(input)); + toks = json_parse_input(talstr, strlen(talstr), &valid); + assert(toks); + assert(tal_count(toks) == 4); + assert(toks[0].start == 0 && toks[0].end == 9); + assert(valid); + tal_free(talstr); + + /* Multiple messages, and the last one is partial, far more likely than + * accidentally getting the boundaries to match. */ + input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z"; + talstr = tal_strndup(NULL, input, strlen(input)); + toks = json_parse_input(talstr, strlen(talstr), &valid); + assert(toks); + assert(tal_count(toks) == 4); + assert(toks[0].start == 0 && toks[0].end == 9); + assert(valid); + tal_free(talstr); +} + +int main(void) +{ + setup_locale(); + + test_json_filter(); + test_json_escape(); + test_json_partial(); + test_json_stream(); + assert(!taken_any()); + take_cleanup(); +} diff --git a/lightningd/test/run-param.c b/lightningd/test/run-param.c index 8e735fdc2..1ff2c98d3 100644 --- a/lightningd/test/run-param.c +++ b/lightningd/test/run-param.c @@ -1,14 +1,12 @@ -#include "config.h" -#include -#include -#include - -#include -#include -#include -#include +#include "../json.c" +#include "../json_escaped.c" +#include "../param.c" #include #include +#include +#include +#include +#include #include char *fail_msg = NULL; diff --git a/wallet/db.c b/wallet/db.c index a7523161f..6b4574e29 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -1,9 +1,9 @@ #include "db.h" #include -#include #include #include +#include #include #include