2018-11-20 02:46:32 +01:00
|
|
|
#include "../json_stream.c"
|
2018-10-19 03:17:48 +02:00
|
|
|
#include "../jsonrpc.c"
|
|
|
|
#include "../json.c"
|
|
|
|
|
|
|
|
/* AUTOGENERATED MOCKS START */
|
|
|
|
/* Generated stub for db_begin_transaction_ */
|
|
|
|
void db_begin_transaction_(struct db *db UNNEEDED, const char *location UNNEEDED)
|
|
|
|
{ fprintf(stderr, "db_begin_transaction_ called!\n"); abort(); }
|
|
|
|
/* Generated stub for db_commit_transaction */
|
|
|
|
void db_commit_transaction(struct db *db UNNEEDED)
|
|
|
|
{ fprintf(stderr, "db_commit_transaction called!\n"); abort(); }
|
|
|
|
/* Generated stub for fatal */
|
|
|
|
void fatal(const char *fmt UNNEEDED, ...)
|
|
|
|
{ fprintf(stderr, "fatal called!\n"); abort(); }
|
|
|
|
/* Generated stub for feerate_from_style */
|
|
|
|
u32 feerate_from_style(u32 feerate UNNEEDED, enum feerate_style style UNNEEDED)
|
|
|
|
{ fprintf(stderr, "feerate_from_style called!\n"); abort(); }
|
|
|
|
/* Generated stub for feerate_name */
|
|
|
|
const char *feerate_name(enum feerate feerate UNNEEDED)
|
|
|
|
{ fprintf(stderr, "feerate_name called!\n"); abort(); }
|
|
|
|
/* Generated stub for fmt_wireaddr_without_port */
|
|
|
|
char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED)
|
|
|
|
{ fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); }
|
common/node_id: new type.
Node ids are pubkeys, but we only use them as pubkeys for routing and checking
gossip messages. So we're packing and unpacking them constantly, and wasting
some space and time.
This introduces a new type, explicitly the SEC1 compressed encoding
(33 bytes). We ensure its validity when we load from the db, or get it
from JSON. We still use 'struct pubkey' for peer messages, which checks
validity.
Results from 5 runs, min-max(mean +/- stddev):
store_load_msec,vsz_kb,store_rewrite_sec,listnodes_sec,listchannels_sec,routing_sec,peer_write_all_sec
39475-39572(39518+/-36),2880732,41.150000-41.390000(41.298+/-0.085),2.260000-2.550000(2.336+/-0.11),44.390000-65.150000(58.648+/-7.5),32.740000-33.020000(32.89+/-0.093),44.130000-45.090000(44.566+/-0.32)
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
2019-04-08 08:34:06 +02:00
|
|
|
/* Generated stub for json_to_node_id */
|
|
|
|
bool json_to_node_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
|
|
|
|
struct node_id *id UNNEEDED)
|
|
|
|
{ fprintf(stderr, "json_to_node_id called!\n"); abort(); }
|
2019-01-15 04:54:27 +01:00
|
|
|
/* Generated stub for json_to_pubkey */
|
|
|
|
bool json_to_pubkey(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
|
|
|
|
struct pubkey *pubkey UNNEEDED)
|
|
|
|
{ fprintf(stderr, "json_to_pubkey called!\n"); abort(); }
|
|
|
|
/* Generated stub for json_to_short_channel_id */
|
|
|
|
bool json_to_short_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
|
2019-02-08 00:23:25 +01:00
|
|
|
struct short_channel_id *scid UNNEEDED,
|
|
|
|
bool may_be_deprecated_form UNNEEDED)
|
2019-01-15 04:54:27 +01:00
|
|
|
{ fprintf(stderr, "json_to_short_channel_id called!\n"); abort(); }
|
2019-06-05 07:28:53 +02:00
|
|
|
/* Generated stub for json_to_txid */
|
|
|
|
bool json_to_txid(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
|
|
|
|
struct bitcoin_txid *txid UNNEEDED)
|
|
|
|
{ fprintf(stderr, "json_to_txid called!\n"); abort(); }
|
2018-10-19 03:17:48 +02:00
|
|
|
/* Generated stub for log_ */
|
2019-06-06 10:26:42 +02:00
|
|
|
void log_(struct log *log UNNEEDED, enum log_level level UNNEEDED, bool call_notifier UNNEEDED, const char *fmt UNNEEDED, ...)
|
2018-10-19 03:17:48 +02:00
|
|
|
|
|
|
|
{ fprintf(stderr, "log_ called!\n"); abort(); }
|
|
|
|
/* Generated stub for log_io */
|
|
|
|
void log_io(struct log *log UNNEEDED, enum log_level dir UNNEEDED, const char *comment UNNEEDED,
|
|
|
|
const void *data UNNEEDED, size_t len UNNEEDED)
|
|
|
|
{ fprintf(stderr, "log_io called!\n"); abort(); }
|
|
|
|
/* Generated stub for log_prefix */
|
|
|
|
const char *log_prefix(const struct log *log UNNEEDED)
|
|
|
|
{ fprintf(stderr, "log_prefix called!\n"); abort(); }
|
2019-02-04 11:55:42 +01:00
|
|
|
/* Generated stub for memleak_remove_strmap_ */
|
|
|
|
void memleak_remove_strmap_(struct htable *memtable UNNEEDED, const struct strmap *m UNNEEDED)
|
|
|
|
{ fprintf(stderr, "memleak_remove_strmap_ called!\n"); abort(); }
|
2018-10-19 03:17:48 +02:00
|
|
|
/* Generated stub for new_log */
|
|
|
|
struct log *new_log(const tal_t *ctx UNNEEDED, struct log_book *record UNNEEDED, const char *fmt UNNEEDED, ...)
|
|
|
|
{ fprintf(stderr, "new_log called!\n"); abort(); }
|
2018-11-20 02:46:32 +01:00
|
|
|
/* Generated stub for new_reltimer_ */
|
|
|
|
struct oneshot *new_reltimer_(struct timers *timers UNNEEDED,
|
|
|
|
const tal_t *ctx UNNEEDED,
|
|
|
|
struct timerel expire UNNEEDED,
|
|
|
|
void (*cb)(void *) UNNEEDED, void *arg UNNEEDED)
|
|
|
|
{ fprintf(stderr, "new_reltimer_ called!\n"); abort(); }
|
2018-10-19 03:17:48 +02:00
|
|
|
/* Generated stub for param */
|
|
|
|
bool param(struct command *cmd UNNEEDED, const char *buffer UNNEEDED,
|
|
|
|
const jsmntok_t params[] UNNEEDED, ...)
|
|
|
|
{ fprintf(stderr, "param called!\n"); abort(); }
|
2018-12-16 05:50:06 +01:00
|
|
|
/* Generated stub for param_feerate_estimate */
|
|
|
|
struct command_result *param_feerate_estimate(struct command *cmd UNNEEDED,
|
|
|
|
u32 **feerate_per_kw UNNEEDED,
|
|
|
|
enum feerate feerate UNNEEDED)
|
|
|
|
{ fprintf(stderr, "param_feerate_estimate called!\n"); abort(); }
|
|
|
|
/* Generated stub for param_number */
|
|
|
|
struct command_result *param_number(struct command *cmd UNNEEDED, const char *name UNNEEDED,
|
|
|
|
const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
|
|
|
|
unsigned int **num UNNEEDED)
|
|
|
|
{ fprintf(stderr, "param_number called!\n"); abort(); }
|
|
|
|
/* Generated stub for param_sha256 */
|
|
|
|
struct command_result *param_sha256(struct command *cmd UNNEEDED, const char *name UNNEEDED,
|
|
|
|
const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
|
|
|
|
struct sha256 **hash UNNEEDED)
|
|
|
|
{ fprintf(stderr, "param_sha256 called!\n"); abort(); }
|
|
|
|
/* Generated stub for param_tok */
|
|
|
|
struct command_result *param_tok(struct command *cmd UNNEEDED, const char *name UNNEEDED,
|
|
|
|
const char *buffer UNNEEDED, const jsmntok_t * tok UNNEEDED,
|
|
|
|
const jsmntok_t **out UNNEEDED)
|
|
|
|
{ fprintf(stderr, "param_tok called!\n"); abort(); }
|
lightningd: hang up on clients if they make us run out of memory.
This happened with the 800M JSON for the MCP listchannels on the raspberry
pi, and tal calls abort() by default.
We switch to raw malloc here; we could override the error hook for
tal, but this is neater since we're doing low-level things anyway,
I tested it manually with this patch:
diff --git a/lightningd/json_stream.c b/lightningd/json_stream.c
index cec9f5771..206ba37c0 100644
--- a/lightningd/json_stream.c
+++ b/lightningd/json_stream.c
@@ -43,6 +43,14 @@ static void free_json_stream_membuf(struct json_stream *js)
free(membuf_cleanup(&js->outbuf));
}
+static void *membuf_realloc_hack(struct membuf *mb, void *rawelems,
+ size_t newsize)
+{
+ if (newsize > 1000000000)
+ return NULL;
+ return realloc(rawelems, newsize);
+}
+
struct json_stream *new_json_stream(const tal_t *ctx,
struct command *writer,
struct log *log)
@@ -53,7 +61,7 @@ struct json_stream *new_json_stream(const tal_t *ctx,
js->reader = NULL;
/* We don't use tal here, because we handle failure externally (tal
* helpfully aborts with a msg, which is usually right) */
- membuf_init(&js->outbuf, malloc(64), 64, membuf_realloc);
+ membuf_init(&js->outbuf, malloc(64), 64, membuf_realloc_hack);
tal_add_destructor(js, free_json_stream_membuf);
#if DEVELOPER
js->wrapping = tal_arr(js, jsmntype_t, 0);
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
2019-05-16 21:57:17 +02:00
|
|
|
/* Generated stub for send_backtrace */
|
|
|
|
void send_backtrace(const char *why UNNEEDED)
|
|
|
|
{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
|
2018-10-19 03:17:48 +02:00
|
|
|
/* AUTOGENERATED MOCKS END */
|
|
|
|
|
|
|
|
bool deprecated_apis;
|
|
|
|
|
|
|
|
static int test_json_filter(void)
|
|
|
|
{
|
2019-02-18 03:44:29 +01:00
|
|
|
struct json_stream *result = new_json_stream(NULL, NULL, NULL);
|
2018-10-19 03:17:48 +02:00
|
|
|
jsmntok_t *toks;
|
|
|
|
const jsmntok_t *x;
|
|
|
|
bool valid;
|
|
|
|
int i;
|
|
|
|
char *badstr = tal_arr(result, char, 256);
|
|
|
|
const char *str;
|
|
|
|
|
|
|
|
/* Fill with junk, and nul-terminate (256 -> 0) */
|
|
|
|
for (i = 1; i < 257; i++)
|
|
|
|
badstr[i-1] = i;
|
|
|
|
|
|
|
|
json_object_start(result, NULL);
|
|
|
|
json_add_string(result, "x", badstr);
|
|
|
|
json_object_end(result);
|
|
|
|
|
|
|
|
/* Parse back in, make sure nothing crazy. */
|
2018-11-20 02:46:32 +01:00
|
|
|
str = tal_strndup(result, membuf_elems(&result->outbuf),
|
|
|
|
membuf_num_elems(&result->outbuf));
|
2018-10-19 03:17:48 +02:00
|
|
|
|
2018-12-08 01:37:56 +01:00
|
|
|
toks = json_parse_input(str, str, strlen(str), &valid);
|
2018-10-19 03:17:48 +02:00
|
|
|
assert(valid);
|
|
|
|
assert(toks);
|
|
|
|
|
|
|
|
assert(toks[0].type == JSMN_OBJECT);
|
|
|
|
x = json_get_member(str, toks, "x");
|
|
|
|
assert(x);
|
|
|
|
assert(x->type == JSMN_STRING);
|
|
|
|
|
|
|
|
/* There are 7 one-letter escapes, and (32-5) \uXXXX escapes. */
|
|
|
|
assert((x->end - x->start) == 255 + 7*1 + (32-5)*5);
|
|
|
|
/* No control characters. */
|
|
|
|
for (i = x->start; i < x->end; i++) {
|
|
|
|
assert((unsigned)str[i] >= ' ');
|
|
|
|
assert((unsigned)str[i] != 127);
|
|
|
|
}
|
2018-11-20 02:46:32 +01:00
|
|
|
tal_free(result);
|
2018-10-19 03:17:48 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_json_escape(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 1; i < 256; i++) {
|
|
|
|
char badstr[2];
|
2019-02-18 03:44:29 +01:00
|
|
|
struct json_stream *result = new_json_stream(NULL, NULL, NULL);
|
2018-10-19 03:17:48 +02:00
|
|
|
struct json_escaped *esc;
|
|
|
|
|
|
|
|
badstr[0] = i;
|
|
|
|
badstr[1] = 0;
|
|
|
|
|
|
|
|
json_object_start(result, NULL);
|
|
|
|
esc = json_escape(NULL, badstr);
|
|
|
|
json_add_escaped_string(result, "x", take(esc));
|
|
|
|
json_object_end(result);
|
|
|
|
|
2018-11-20 02:46:32 +01:00
|
|
|
const char *str = tal_strndup(result, membuf_elems(&result->outbuf),
|
|
|
|
membuf_num_elems(&result->outbuf));
|
2018-10-19 03:17:48 +02:00
|
|
|
if (i == '\\' || i == '"'
|
|
|
|
|| i == '\n' || i == '\r' || i == '\b'
|
|
|
|
|| i == '\t' || i == '\f')
|
2019-04-08 11:58:44 +02:00
|
|
|
assert(strstarts(str, "{\"x\":\"\\"));
|
2018-10-19 03:17:48 +02:00
|
|
|
else if (i < 32 || i == 127) {
|
2019-04-08 11:58:44 +02:00
|
|
|
assert(strstarts(str, "{\"x\":\"\\u00"));
|
2018-10-19 03:17:48 +02:00
|
|
|
} else {
|
2019-04-08 11:58:44 +02:00
|
|
|
char expect[] = "{\"x\":\"?\"}";
|
|
|
|
expect[6] = i;
|
2018-10-19 03:17:48 +02:00
|
|
|
assert(streq(str, expect));
|
|
|
|
}
|
2018-11-20 02:46:32 +01:00
|
|
|
tal_free(result);
|
2018-10-19 03:17:48 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_json_partial(void)
|
|
|
|
{
|
|
|
|
const tal_t *ctx = tal(NULL, char);
|
|
|
|
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\n")->s, "\\n"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\""));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\"")->s, "\\\""));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\t")->s, "\\t"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\b")->s, "\\b"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\r")->s, "\\r"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\f")->s, "\\f"));
|
|
|
|
/* You're allowed to escape / according to json.org. */
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "/")->s, "/"));
|
|
|
|
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF"));
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx"));
|
|
|
|
|
|
|
|
/* Unknown escapes should be escaped. */
|
|
|
|
assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x"));
|
|
|
|
tal_free(ctx);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Test that we can segment and parse a stream of json objects correctly. */
|
|
|
|
static void test_json_stream(void)
|
|
|
|
{
|
|
|
|
bool valid;
|
|
|
|
char *input, *talstr;
|
|
|
|
jsmntok_t *toks;
|
|
|
|
|
|
|
|
/* Multiple full messages in a single buffer (happens when buffer
|
|
|
|
* boundary coincides with message boundary, or read returned after
|
|
|
|
* timeout. */
|
|
|
|
input = "{\"x\":\"x\"}{\"y\":\"y\"}";
|
|
|
|
talstr = tal_strndup(NULL, input, strlen(input));
|
2018-12-08 01:37:56 +01:00
|
|
|
toks = json_parse_input(talstr, talstr, strlen(talstr), &valid);
|
2018-10-19 03:17:48 +02:00
|
|
|
assert(toks);
|
|
|
|
assert(tal_count(toks) == 4);
|
|
|
|
assert(toks[0].start == 0 && toks[0].end == 9);
|
|
|
|
assert(valid);
|
|
|
|
tal_free(talstr);
|
|
|
|
|
|
|
|
/* Multiple messages, and the last one is partial, far more likely than
|
|
|
|
* accidentally getting the boundaries to match. */
|
|
|
|
input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z";
|
|
|
|
talstr = tal_strndup(NULL, input, strlen(input));
|
2018-12-08 01:37:56 +01:00
|
|
|
toks = json_parse_input(talstr, talstr, strlen(talstr), &valid);
|
2018-10-19 03:17:48 +02:00
|
|
|
assert(toks);
|
|
|
|
assert(tal_count(toks) == 4);
|
|
|
|
assert(toks[0].start == 0 && toks[0].end == 9);
|
|
|
|
assert(valid);
|
|
|
|
tal_free(talstr);
|
|
|
|
}
|
|
|
|
|
|
|
|
int main(void)
|
|
|
|
{
|
|
|
|
setup_locale();
|
|
|
|
|
|
|
|
test_json_filter();
|
|
|
|
test_json_escape();
|
|
|
|
test_json_partial();
|
|
|
|
test_json_stream();
|
|
|
|
assert(!taken_any());
|
|
|
|
take_cleanup();
|
|
|
|
}
|