lightningd: --dev-disconnect support.

We use a file descriptor, so when we consume an entry, we move past it
(and everyone shares a file offset, so this works).

The file contains packet names prefixed by - (treat fd as closed when
we try to write this packet), + (write the packet then ensure the file
descriptor fails), or @ ("lose" the packet then ensure the file
descriptor fails).

The sync and async peer-write functions hook this in automatically.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>



Header from folded patch 'test-run-cryptomsg__fix_compilation.patch':

test/run-cryptomsg: fix compilation.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/Makefile

@@ -44,6 +44,7 @@ LIGHTNINGD_LIB_SRC :=				\
 	lightningd/debug.c			\
 	lightningd/daemon_conn.c		\
 	lightningd/derive_basepoints.c		\
+	lightningd/dev_disconnect.c		\
 	lightningd/funding_tx.c			\
 	lightningd/gossip_msg.c			\
 	lightningd/htlc_tx.c			\

lightningd/crypto_sync.c

@@ -3,19 +3,38 @@
 #include <inttypes.h>
 #include <lightningd/crypto_sync.h>
 #include <lightningd/cryptomsg.h>
+#include <lightningd/dev_disconnect.h>
 #include <lightningd/status.h>
 #include <utils.h>
+#include <wire/wire.h>
 #include <wire/wire_sync.h>
 
 bool sync_crypto_write(struct crypto_state *cs, int fd, const void *msg)
 {
 	u8 *enc = cryptomsg_encrypt_msg(msg, cs, msg);
 	bool ret;
+	bool post_sabotage = false;
 
 	status_trace("Writing crypto with sn=%"PRIu64" msg=%s",
 		     cs->sn-2, tal_hex(trc, msg));
+
+	switch (dev_disconnect(fromwire_peektype(msg))) {
+	case DEV_DISCONNECT_BEFORE:
+		dev_sabotage_fd(fd);
+		return false;
+	case DEV_DISCONNECT_DROPPKT:
+		enc = tal_free(enc); /* FALL THRU */
+	case DEV_DISCONNECT_AFTER:
+		post_sabotage = true;
+		break;
+	default:
+		break;
+	}
 	ret = write_all(fd, enc, tal_len(enc));
 	tal_free(enc);
+
+	if (post_sabotage)
+		dev_sabotage_fd(fd);
 	return ret;
 }
 

lightningd/cryptomsg.c

@@ -7,6 +7,7 @@
 #include <ccan/short_types/short_types.h>
 #include <ccan/take/take.h>
 #include <lightningd/cryptomsg.h>
+#include <lightningd/dev_disconnect.h>
 #include <lightningd/status.h>
 #include <sodium/crypto_aead_chacha20poly1305.h>
 #include <utils.h>
@@ -316,12 +317,21 @@ u8 *cryptomsg_encrypt_msg(const tal_t *ctx,
 	return out;
 }
 
+static struct io_plan *peer_write_postclose(struct io_conn *conn,
+					    struct peer_crypto_state *pcs)
+{
+	pcs->out = tal_free(pcs->out);
+	dev_sabotage_fd(io_conn_fd(conn));
+	return pcs->next_out(conn, pcs->peer);
+}
+
 struct io_plan *peer_write_message(struct io_conn *conn,
 				   struct peer_crypto_state *pcs,
 				   const u8 *msg,
 				   struct io_plan *(*next)(struct io_conn *,
 							   struct peer *))
 {
+	struct io_plan *(*post)(struct io_conn *, struct peer_crypto_state *);
 	assert(!pcs->out);
 
 	pcs->out = cryptomsg_encrypt_msg(conn, &pcs->cs, msg);
@@ -329,11 +339,24 @@ struct io_plan *peer_write_message(struct io_conn *conn,
 		tal_free(msg);
 	pcs->next_out = next;
 
+	post = peer_write_done;
+
+	switch (dev_disconnect(fromwire_peektype(msg))) {
+	case DEV_DISCONNECT_BEFORE:
+		return io_close(conn);
+	case DEV_DISCONNECT_DROPPKT:
+		pcs->out = NULL; /* FALL THRU */
+	case DEV_DISCONNECT_AFTER:
+		post = peer_write_postclose;
+		break;
+	default:
+		break;
+	}
+
 	/* BOLT #8:
 	 *   * Send `lc || c` over the network buffer.
 	 */
-	return io_write(conn, pcs->out, tal_count(pcs->out),
-			peer_write_done, pcs);
+	return io_write(conn, pcs->out, tal_count(pcs->out), post, pcs);
 }
 
 void init_peer_crypto_state(struct peer *peer, struct peer_crypto_state *pcs)

lightningd/debug.c

@@ -1,6 +1,8 @@
 #include <ccan/str/str.h>
 #include <lightningd/debug.h>
+#include <lightningd/dev_disconnect.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/types.h>
 #include <unistd.h>
 
@@ -9,6 +11,13 @@ void subdaemon_debug(int argc, char *argv[])
 	int i;
 	bool printed = false;
 
+	for (i = 1; i < argc; i++) {
+		if (strstarts(argv[i], "--dev-disconnect=")) {
+			dev_disconnect_init(atoi(argv[i]
+						 + strlen("--dev-disconnect=")));
+		}
+	}
+
 	/* From debugger, tell gdb "return". */
 	for (i = 1; i < argc; i++) {
 		while (streq(argv[i], "--debugger")) {

lightningd/dev_disconnect.c

@@ -0,0 +1,55 @@
+#include <assert.h>
+#include <ccan/err/err.h>
+#include <ccan/str/str.h>
+#include <lightningd/dev_disconnect.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <wire/gen_peer_wire.h>
+
+/* We move the fd IFF we do a disconnect. */
+static int dev_disconnect_fd = -1;
+static char dev_disconnect_line[200];
+
+void dev_disconnect_init(int fd)
+{
+	int r;
+
+	r = read(fd, dev_disconnect_line, sizeof(dev_disconnect_line)-1);
+	if (r < 0)
+		err(1, "Reading dev_disconnect file");
+	lseek(fd, -r, SEEK_CUR);
+
+	/* Get first line */
+	dev_disconnect_line[r] = '\n';
+	*strchr(dev_disconnect_line, '\n') = '\0';
+
+	/* So we can move forward if we do use the line. */
+	dev_disconnect_fd = fd;
+}
+
+char dev_disconnect(int pkt_type)
+{
+	if (!streq(wire_type_name(pkt_type), dev_disconnect_line+1))
+		return DEV_DISCONNECT_NORMAL;
+
+	assert(dev_disconnect_fd != -1);
+	lseek(dev_disconnect_fd, strlen(dev_disconnect_line)+1, SEEK_CUR);
+
+	return dev_disconnect_line[0];
+}
+
+void dev_sabotage_fd(int fd)
+{
+	int fds[2];
+
+	if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0)
+		errx(1, "dev_sabotage_fd: creating socketpair");
+
+	/* Close one. */
+	close(fds[0]);
+	/* Move other over to the fd we want to sabotage. */
+	dup2(fds[1], fd);
+	close(fds[1]);
+}

lightningd/dev_disconnect.h

@@ -0,0 +1,20 @@
+#ifndef LIGHTNING_LIGHTNINGD_DEV_DISCONNECT_H
+#define LIGHTNING_LIGHTNINGD_DEV_DISCONNECT_H
+#include "config.h"
+#include <stdbool.h>
+
+#define DEV_DISCONNECT_BEFORE '-'
+#define DEV_DISCONNECT_AFTER '+'
+#define DEV_DISCONNECT_DROPPKT '@'
+#define DEV_DISCONNECT_NORMAL 0
+
+/* Force a close fd before or after a certain packet type */
+char dev_disconnect(int pkt_type);
+
+/* Make next write on fd fail as if they'd disconnected. */
+void dev_sabotage_fd(int fd);
+
+/* For debug code to set in daemon. */
+void dev_disconnect_init(int fd);
+
+#endif /* LIGHTNING_LIGHTNINGD_DEV_DISCONNECT_H */

lightningd/lightningd.c

@@ -111,6 +111,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	ld->dev_debug_subdaemon = NULL;
 	list_head_init(&ld->utxos);
 	htlc_end_map_init(&ld->htlc_ends);
+	ld->dev_disconnect_fd = -1;
 	ld->dstate.log_book = new_log_book(&ld->dstate, 20*1024*1024,LOG_INFORM);
 	ld->log = ld->dstate.base_log = new_log(&ld->dstate,
 						ld->dstate.log_book,
@@ -233,6 +234,9 @@ int main(int argc, char *argv[])
 			 opt_show_uintval, &ld->broadcast_interval,
 			 "Time between gossip broadcasts in milliseconds (default: 30000)");
 
+	opt_register_arg("--dev-disconnect=<filename>", opt_subd_dev_disconnect,
+			 NULL, ld, "File containing disconnection points");
+
 	/* FIXME: move to option initialization once we drop the
 	 * legacy daemon */
 	ld->broadcast_interval = 30000;

lightningd/lightningd.h

@@ -50,6 +50,9 @@ struct lightningd {
 	/* If we want to debug a subdaemon. */
 	const char *dev_debug_subdaemon;
 
+	/* If we have a --dev-disconnect file */
+	int dev_disconnect_fd;
+
 	/* UTXOs we have available to spend. */
 	struct list_head utxos;
 

lightningd/subd.c

@@ -3,6 +3,7 @@
 #include <ccan/noerr/noerr.h>
 #include <ccan/take/take.h>
 #include <ccan/tal/path/path.h>
+#include <ccan/tal/str/str.h>
 #include <daemon/log.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -120,7 +121,7 @@ static struct subd_req *get_req(struct subd *sd, int reply_type)
 
 /* We use sockets, not pipes, because fds are bidir. */
 static int subd(const char *dir, const char *name, bool debug,
-		     int *msgfd, va_list ap)
+		int *msgfd, int dev_disconnect_fd, va_list ap)
 {
 	int childmsg[2], execfail[2];
 	pid_t childpid;
@@ -143,7 +144,7 @@ static int subd(const char *dir, const char *name, bool debug,
 	if (childpid == 0) {
 		int fdnum = 3, i;
 		long max;
-		const char *debug_arg = NULL;
+		const char *debug_arg[2] = { NULL, NULL };
 
 		close(childmsg[0]);
 		close(execfail[0]);
@@ -154,6 +155,13 @@ static int subd(const char *dir, const char *name, bool debug,
 				goto child_errno_fail;
 		}
 
+		// Move dev_disconnect_fd out the way.
+		if (dev_disconnect_fd != -1) {
+			if (!move_fd(dev_disconnect_fd, 101))
+				goto child_errno_fail;
+			dev_disconnect_fd = 101;
+		}
+
 		/* Dup any extra fds up first. */
 		while ((fd = va_arg(ap, int *)) != NULL) {
 			/* If this were stdin, dup2 closed! */
@@ -166,11 +174,14 @@ static int subd(const char *dir, const char *name, bool debug,
 		/* Make (fairly!) sure all other fds are closed. */
 		max = sysconf(_SC_OPEN_MAX);
 		for (i = fdnum; i < max; i++)
-			close(i);
+			if (i != dev_disconnect_fd)
+				close(i);
 
+		if (dev_disconnect_fd != -1)
+			debug_arg[0] = tal_fmt(NULL, "--dev-disconnect=%i", dev_disconnect_fd);
 		if (debug)
-			debug_arg = "--debugger";
-		execl(path_join(NULL, dir, name), name, debug_arg, NULL);
+			debug_arg[debug_arg[0] ? 1 : 0] = "--debugger";
+		execl(path_join(NULL, dir, name), name, debug_arg[0], debug_arg[1], NULL);
 
 	child_errno_fail:
 		err = errno;
@@ -381,7 +392,8 @@ struct subd *new_subd(const tal_t *ctx,
 	debug = ld->dev_debug_subdaemon
 		&& strends(name, ld->dev_debug_subdaemon);
 	va_start(ap, finished);
-	sd->pid = subd(ld->daemon_dir, name, debug, &msg_fd, ap);
+	sd->pid = subd(ld->daemon_dir, name, debug, &msg_fd,
+		       ld->dev_disconnect_fd, ap);
 	va_end(ap);
 	if (sd->pid == (pid_t)-1) {
 		log_unusual(ld->log, "subd %s failed: %s",
@@ -467,3 +479,12 @@ char *opt_subd_debug(const char *optarg, struct lightningd *ld)
 	ld->dev_debug_subdaemon = optarg;
 	return NULL;
 }
+
+char *opt_subd_dev_disconnect(const char *optarg, struct lightningd *ld)
+{
+	ld->dev_disconnect_fd = open(optarg, O_RDONLY);
+	if (ld->dev_disconnect_fd < 0)
+		return tal_fmt(ld, "Could not open --dev-disconnect=%s: %s",
+			       optarg, strerror(errno));
+	return NULL;
+}

lightningd/subd.h

@@ -122,5 +122,5 @@ void subd_req_(const tal_t *ctx,
 void subd_shutdown(struct subd *subd, unsigned int seconds);
 
 char *opt_subd_debug(const char *optarg, struct lightningd *ld);
-
+char *opt_subd_dev_disconnect(const char *optarg, struct lightningd *ld);
 #endif /* LIGHTNING_LIGHTNINGD_SUBD_H */

lightningd/test/run-cryptomsg.c

@@ -1,6 +1,7 @@
 #include <assert.h>
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
+#include <lightningd/dev_disconnect.h>
 #include <lightningd/status.h>
 #include <stdio.h>
 #include <wire/peer_wire.h>
@@ -38,6 +39,16 @@ static void do_write(const void *buf, size_t len)
 #define status_trace(fmt, ...) \
 	printf(fmt "\n", __VA_ARGS__)
 
+void dev_sabotage_fd(int fd)
+{
+	abort();
+}
+
+char dev_disconnect(int pkt_type)
+{
+	return DEV_DISCONNECT_NORMAL;
+}
+
 /* We test what look like unknown messages. */
 #define unknown_msg_discardable(x) 0