ccan: update and add more.

We need the following additional modules for the daemon:
	io, time, timer, pipecmd

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Makefile

@@ -71,6 +71,8 @@ CCAN_OBJS :=					\
 	ccan-crypto-shachain.o			\
 	ccan-err.o				\
 	ccan-ilog.o				\
+	ccan-io-io.o				\
+	ccan-io-poll.o				\
 	ccan-isaac.o				\
 	ccan-isaac64.o				\
 	ccan-list.o				\
@@ -79,6 +81,7 @@ CCAN_OBJS :=					\
 	ccan-opt-parse.o			\
 	ccan-opt-usage.o			\
 	ccan-opt.o				\
+	ccan-pipecmd.o				\
 	ccan-read_write_all.o			\
 	ccan-str-hex.o				\
 	ccan-str.o				\
@@ -86,7 +89,8 @@ CCAN_OBJS :=					\
 	ccan-tal-grab_file.o			\
 	ccan-tal-str.o				\
 	ccan-tal.o				\
-	ccan-time.o
+	ccan-time.o				\
+	ccan-timer.o
 
 # For tests
 CCAN_EXTRA_OBJS :=				\
@@ -113,6 +117,9 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/htable/htable.h			\
 	$(CCANDIR)/ccan/htable/htable_type.h		\
 	$(CCANDIR)/ccan/ilog/ilog.h			\
+	$(CCANDIR)/ccan/io/backend.h			\
+	$(CCANDIR)/ccan/io/io.h				\
+	$(CCANDIR)/ccan/io/io_plan.h			\
 	$(CCANDIR)/ccan/isaac/isaac.h			\
 	$(CCANDIR)/ccan/isaac/isaac64.h			\
 	$(CCANDIR)/ccan/likely/likely.h			\
@@ -122,6 +129,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/opt/opt.h			\
 	$(CCANDIR)/ccan/opt/private.h			\
 	$(CCANDIR)/ccan/order/order.h			\
+	$(CCANDIR)/ccan/pipecmd/pipecmd.h		\
 	$(CCANDIR)/ccan/ptrint/ptrint.h			\
 	$(CCANDIR)/ccan/read_write_all/read_write_all.h	\
 	$(CCANDIR)/ccan/short_types/short_types.h	\
@@ -140,6 +148,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/tal/talloc/talloc.h		\
 	$(CCANDIR)/ccan/tcon/tcon.h			\
 	$(CCANDIR)/ccan/time/time.h			\
+	$(CCANDIR)/ccan/timer/timer.h			\
 	$(CCANDIR)/ccan/typesafe_cb/typesafe_cb.h
 
 TEST_CLI_HEADERS := test-cli/gather_updates.h \
@@ -369,3 +378,11 @@ ccan-isaac64.o: $(CCANDIR)/ccan/isaac/isaac64.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-time.o: $(CCANDIR)/ccan/time/time.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-timer.o: $(CCANDIR)/ccan/timer/timer.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-io-io.o: $(CCANDIR)/ccan/io/io.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-io-poll.o: $(CCANDIR)/ccan/io/poll.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-pipecmd.o: $(CCANDIR)/ccan/pipecmd/pipecmd.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2084-gb87f63c
+CCAN version: init-2136-g64e9e71

ccan/ccan/io/LICENSE

@@ -0,0 +1 @@
+../../licenses/LGPL-2.1

ccan/ccan/io/SCENARIOS

@@ -0,0 +1,88 @@
+Simple:
+	step1(conn): read(conn), then step2
+	step2(conn): write(conn), then close
+
+Pass-through:
+	step1(conn): read(conn), then step2
+	step2(conn): write(otherconn), then step1
+
+Pass-through-and-connect:
+	step1(conn): read(conn), then step2
+	step2(conn): connect(otherconn), then step3
+	step3(conn): write(otherconn), then step1
+
+Chatroom:
+	step1(conn): read(conn), then step2
+	step2(conn): for c in allcons: write(c).  goto step1
+
+Simple:
+
+void event(struct io_event *done)
+{
+	char *buf = done->priv;
+	struct io_event *e;
+
+	e = queue_read(done, done->conn, buf, 100);
+	e = queue_write(e, done->conn, buf, 100);
+	queue_close(e, done->conn);
+}
+
+Pass-through:
+struct passthru {
+	char buf[100];
+	struct conn *rconn, *wconn;
+};
+
+void event(struct io_event *done)
+{
+	struct passthru *p = done->priv;
+	struct io_event *e;
+
+	e = queue_read(done, p->rconn, p->buf, 100);
+	e = queue_write(e, p->wconn, buf, 100);
+	queue_event(e, event);
+}
+
+Chatroom:
+struct list_head clients;
+
+struct buffer {
+	char buf[100];
+	unsigned int ref;
+};
+
+struct client {
+	struct list_node list;
+	struct connection *conn;
+	struct buffer *rbuf, *wbuf;
+};
+
+void broadcast(struct io_event *done)
+{
+	struct client *i, *c = done->conn->priv;
+	struct io_event *e;
+
+	list_for_each(&clients, i, list) {
+		e = queue_write(done, i->conn, c->buf->buf, 100);
+		e->priv = c->buf;
+		c->buf->ref++;
+		queue_event(e, drop_ref);
+	}
+
+
+
+void event(struct io_event *done)
+{
+	struct client *c = done->conn->priv;
+	struct io_event *e;
+
+	assert(c->conn == done->conn);
+	c->buf = malloc(sizeof(*c->buf));
+	c->buf->ref = 0;
+	e = queue_read(done, c->conn, c->buf->buf, 100);
+	e = queue_event(e, broadcast);
+}
+
+
+	step1(conn): read(conn), then step2
+	step2(conn): for c in allcons: write(c).  goto step1

ccan/ccan/io/_info

@@ -0,0 +1,146 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * io - simple library for asynchronous io handling.
+ *
+ * io provides a mechanism to write I/O servers with multiple
+ * connections.  Each callback indicates what I/O they plan next
+ * (eg. read, write).  It is also possible to write custom I/O
+ * plans.
+ *
+ * Example:
+ * // Given "tr A-Z a-z" outputs tr a-z a-z
+ * #include <ccan/io/io.h>
+ * #include <ccan/err/err.h>
+ * #include <assert.h>
+ * #include <stdlib.h>
+ * #include <signal.h>
+ * #include <sys/types.h>
+ * #include <sys/wait.h>
+ * #include <string.h>
+ *
+ * struct buffer {
+ * 	bool finished;
+ * 	size_t start, end, rlen, wlen;
+ * 	char buf[4096];
+ * };
+ *
+ * static void finish(struct io_conn *c, struct buffer *b)
+ * {
+ * 	// Mark us finished.
+ * 	b->finished = true;
+ * 	// Wake writer just in case it's asleep.
+ * 	io_wake(b);
+ * }
+ *
+ * static struct io_plan *read_in(struct io_conn *c, struct buffer *b)
+ * {
+ * 	// Add what we just read.
+ * 	b->end += b->rlen;
+ * 	assert(b->end <= sizeof(b->buf));
+ *
+ * 	// If we just read something, wake writer.
+ * 	if (b->rlen != 0)
+ * 		io_wake(b);
+ *
+ * 	// If buffer is empty, return to start.
+ * 	if (b->start == b->end)
+ * 		b->start = b->end = 0;
+ *
+ * 	// No room?  Wait for writer
+ * 	if (b->end == sizeof(b->buf))
+ * 		return io_wait(c, b, read_in, b);
+ *
+ * 	return io_read_partial(c, b->buf + b->end, sizeof(b->buf) - b->end,
+ *			       &b->rlen, read_in, b);
+ * }
+ *
+ * static struct io_plan *write_out(struct io_conn *c, struct buffer *b)
+ * {
+ * 	// Remove what we just wrote.
+ * 	b->start += b->wlen;
+ * 	assert(b->start <= sizeof(b->buf));
+ *
+ * 	// If we wrote something, wake writer.
+ * 	if (b->wlen != 0)
+ * 		io_wake(b);
+ *
+ * 	// Nothing to write?  Wait for reader.
+ * 	if (b->end == b->start) {
+ * 		if (b->finished)
+ * 			return io_close(c);
+ * 		return io_wait(c, b, write_out, b);
+ * 	}
+ *
+ * 	return io_write_partial(c, b->buf + b->start, b->end - b->start,
+ *				&b->wlen, write_out, b);
+ * }
+ *
+ * // Feed a program our stdin, gather its stdout, print that at end.
+ * int main(int argc, char *argv[])
+ * {
+ * 	int tochild[2], fromchild[2];
+ * 	struct buffer to, from;
+ * 	int status;
+ * 	struct io_conn *reader;
+ *
+ * 	if (argc == 1)
+ * 		errx(1, "Usage: runner <cmdline>...");
+ *
+ * 	if (pipe(tochild) != 0 || pipe(fromchild) != 0)
+ * 		err(1, "Creating pipes");
+ *
+ * 	if (!fork()) {
+ * 		// Child runs command.
+ * 		close(tochild[1]);
+ * 		close(fromchild[0]);
+ *
+ * 		dup2(tochild[0], STDIN_FILENO);
+ * 		dup2(fromchild[1], STDOUT_FILENO);
+ * 		execvp(argv[1], argv + 1);
+ * 		exit(127);
+ * 	}
+ *
+ * 	close(tochild[0]);
+ * 	close(fromchild[1]);
+ * 	signal(SIGPIPE, SIG_IGN);
+ *
+ * 	// Read from stdin, write to child.
+ * 	memset(&to, 0, sizeof(to));
+ * 	reader = io_new_conn(NULL, STDIN_FILENO, read_in, &to);
+ * 	io_set_finish(reader, finish, &to);
+ * 	io_new_conn(NULL, tochild[1], write_out, &to);
+ *
+ * 	// Read from child, write to stdout.
+ * 	reader = io_new_conn(NULL, fromchild[0], read_in, &from);
+ * 	io_set_finish(reader, finish, &from);
+ * 	io_new_conn(NULL, STDOUT_FILENO, write_out, &from);
+ *
+ * 	io_loop(NULL, NULL);
+ * 	wait(&status);
+ *
+ * 	return WIFEXITED(status) ? WEXITSTATUS(status) : 2;
+ * }
+ *
+ * License: LGPL (v2.1 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/container_of\n");
+		printf("ccan/list\n");
+		printf("ccan/tal\n");
+		printf("ccan/time\n");
+		printf("ccan/timer\n");
+		printf("ccan/typesafe_cb\n");
+		return 0;
+	}
+
+	return 1;
+}

ccan/ccan/io/backend.h

@@ -0,0 +1,94 @@
+/* Licensed under LGPLv2.1+ - see LICENSE file for details */
+#ifndef CCAN_IO_BACKEND_H
+#define CCAN_IO_BACKEND_H
+#include <stdbool.h>
+#include <poll.h>
+#include "io_plan.h"
+#include <ccan/list/list.h>
+
+struct fd {
+	int fd;
+	bool listener;
+	size_t backend_info;
+};
+
+/* Listeners create connections. */
+struct io_listener {
+	struct fd fd;
+
+	const tal_t *ctx;
+
+	/* These are for connections we create. */
+	struct io_plan *(*init)(struct io_conn *conn, void *arg);
+	void *arg;
+};
+
+enum io_plan_status {
+	/* As before calling next function. */
+	IO_UNSET,
+	/* Normal. */
+	IO_POLLING,
+	/* Waiting for io_wake */
+	IO_WAITING,
+	/* Always do this. */
+	IO_ALWAYS,
+	/* Closing (both plans will be the same). */
+	IO_CLOSING
+};
+
+/**
+ * struct io_plan - one half of I/O to do
+ * @status: the status of this plan.
+ * @io: function to call when fd becomes read/writable, returns 0 to be
+ *      called again, 1 if it's finished, and -1 on error (fd will be closed)
+ * @next: the next function which is called if io returns 1.
+ * @next_arg: the argument to @next
+ * @u1, @u2: scratch space for @io.
+ */
+struct io_plan {
+	enum io_plan_status status;
+
+	int (*io)(int fd, struct io_plan_arg *arg);
+
+	struct io_plan *(*next)(struct io_conn *, void *next_arg);
+	void *next_arg;
+
+	struct io_plan_arg arg;
+};
+
+/* One connection per client. */
+struct io_conn {
+	struct fd fd;
+	bool debug;
+	/* For duplex to save. */
+	bool debug_saved;
+
+	/* always and closing lists. */
+	struct list_node always, closing;
+
+	void (*finish)(struct io_conn *, void *arg);
+	void *finish_arg;
+
+	struct io_plan plan[2];
+};
+
+extern void *io_loop_return;
+
+bool add_listener(struct io_listener *l);
+bool add_conn(struct io_conn *c);
+bool add_duplex(struct io_conn *c);
+void del_listener(struct io_listener *l);
+void backend_new_closing(struct io_conn *conn);
+void backend_new_always(struct io_conn *conn);
+void backend_new_plan(struct io_conn *conn);
+void remove_from_always(struct io_conn *conn);
+void backend_plan_done(struct io_conn *conn);
+
+void backend_wake(const void *wait);
+void backend_del_conn(struct io_conn *conn);
+
+void io_ready(struct io_conn *conn, int pollflags);
+void io_do_always(struct io_conn *conn);
+void io_do_wakeup(struct io_conn *conn, enum io_direction dir);
+void *do_io_loop(struct io_conn **ready);
+#endif /* CCAN_IO_BACKEND_H */

ccan/ccan/io/benchmarks/Makefile

@@ -0,0 +1,29 @@
+ALL:=run-loop run-different-speed run-length-prefix
+CCANDIR:=../../..
+CFLAGS:=-Wall -I$(CCANDIR) -O3 -flto
+LDFLAGS:=-O3 -flto
+LDLIBS:=-lrt
+
+OBJS:=time.o poll.o io.o err.o timer.o list.o
+
+default: $(ALL)
+
+run-loop: run-loop.o $(OBJS)
+run-different-speed: run-different-speed.o $(OBJS)
+run-length-prefix: run-length-prefix.o $(OBJS)
+
+time.o: $(CCANDIR)/ccan/time/time.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+timer.o: $(CCANDIR)/ccan/timer/timer.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+list.o: $(CCANDIR)/ccan/list/list.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+poll.o: $(CCANDIR)/ccan/io/poll.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+io.o: $(CCANDIR)/ccan/io/io.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+err.o: $(CCANDIR)/ccan/err/err.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+clean:
+	$(RM) -f *.o $(ALL)

ccan/ccan/io/benchmarks/run-different-speed.c

@@ -0,0 +1,176 @@
+/* Simulate a server with connections of different speeds.  We count
+ * how many connections complete in 10 seconds. */
+#include <ccan/io/io.h>
+#include <ccan/time/time.h>
+#include <ccan/err/err.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdio.h>
+#include <signal.h>
+
+#define REQUEST_SIZE 1024
+#define REPLY_SIZE 10240
+#define NUM_CONNS 500 /* per child */
+#define NUM_CHILDREN 2
+
+static unsigned int completed;
+
+struct client {
+	char request_buffer[REQUEST_SIZE];
+	char reply_buffer[REPLY_SIZE];
+};
+
+static struct io_plan write_reply(struct io_conn *conn, struct client *client);
+static struct io_plan read_request(struct io_conn *conn, struct client *client)
+{
+	return io_read(client->request_buffer, REQUEST_SIZE,
+		       write_reply, client);
+}
+
+/* once we're done, loop again. */
+static struct io_plan write_complete(struct io_conn *conn, struct client *client)
+{
+	completed++;
+	return read_request(conn, client);
+}
+
+static struct io_plan write_reply(struct io_conn *conn, struct client *client)
+{
+	return io_write(client->reply_buffer, REPLY_SIZE,
+			write_complete, client);
+}
+
+/* This runs in the child. */
+static void create_clients(struct sockaddr_un *addr, int waitfd)
+{
+	struct client data;
+	int i, sock[NUM_CONNS], speed[NUM_CONNS], done[NUM_CONNS], count = 0;
+
+	for (i = 0; i < NUM_CONNS; i++) {
+		/* Set speed. */
+		speed[i] = (1 << (random() % 10));
+		sock[i] = socket(AF_UNIX, SOCK_STREAM, 0);
+		if (sock[i] < 0)
+			err(1, "creating socket");
+		if (connect(sock[i], (void *)addr, sizeof(*addr)) != 0)
+			err(1, "connecting socket");
+		/* Make nonblocking. */
+		fcntl(sock[i], F_SETFD, fcntl(sock[i], F_GETFD)|O_NONBLOCK);
+		done[i] = 0;
+	}
+
+	read(waitfd, &i, 1);
+
+	for (;;) {
+		for (i = 0; i < NUM_CONNS; i++) {
+			int ret, bytes = speed[i];
+			if (done[i] < REQUEST_SIZE) {
+				if (REQUEST_SIZE - done[i] < bytes)
+					bytes = REQUEST_SIZE - done[i];
+				ret = write(sock[i], data.request_buffer,
+					    bytes);
+				if (ret > 0)
+					done[i] += ret;
+				else if (ret < 0 && errno != EAGAIN)
+					goto fail;
+			} else {
+				if (REQUEST_SIZE + REPLY_SIZE - done[i] < bytes)
+					bytes = REQUEST_SIZE + REPLY_SIZE
+						- done[i];
+				ret = read(sock[i], data.reply_buffer,
+					    bytes);
+				if (ret > 0) {
+					done[i] += ret;
+					if (done[i] == REQUEST_SIZE + REPLY_SIZE) {
+						count++;
+						done[i] = 0;
+					}
+				} else if (ret < 0 && errno != EAGAIN)
+					goto fail;
+			}
+		}
+	}
+fail:
+	printf("Child did %u\n", count);
+	exit(0);
+}
+
+static int timeout[2];
+static void sigalarm(int sig)
+{
+	write(timeout[1], "1", 1);
+}
+
+static struct io_plan do_timeout(struct io_conn *conn, char *buf)
+{
+	return io_break(buf, io_idle());
+}
+
+int main(int argc, char *argv[])
+{
+	struct client client;
+	unsigned int i, j;
+	struct sockaddr_un addr;
+	struct timespec start, end;
+	int fd, wake[2];
+	char buf;
+
+	addr.sun_family = AF_UNIX;
+	sprintf(addr.sun_path, "/tmp/run-different-speed.sock.%u", getpid());
+
+	if (pipe(wake) != 0 || pipe(timeout) != 0)
+		err(1, "Creating pipes");
+
+	fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd < 0)
+		err(1, "Creating socket");
+
+	if (bind(fd, (void *)&addr, sizeof(addr)) != 0)
+		err(1, "Binding to %s", addr.sun_path);
+
+	if (listen(fd, NUM_CONNS) != 0)
+		err(1, "Listening on %s", addr.sun_path);
+
+	for (i = 0; i < NUM_CHILDREN; i++) {
+		switch (fork()) {
+		case -1:
+			err(1, "forking");
+		case 0:
+			close(wake[1]);
+			create_clients(&addr, wake[0]);
+			break;
+		}
+		for (j = 0; j < NUM_CONNS; j++) {
+			int ret = accept(fd, NULL, 0);
+			if (ret < 0)
+				err(1, "Accepting fd");
+			/* For efficiency, we share client structure */
+			io_new_conn(ret,
+				    io_read(client.request_buffer, REQUEST_SIZE,
+					    write_reply, &client));
+		}
+	}
+
+	io_new_conn(timeout[0], io_read(&buf, 1, do_timeout, &buf));
+
+	close(wake[0]);
+	for (i = 0; i < NUM_CHILDREN; i++)
+		write(wake[1], "1", 1);
+
+	signal(SIGALRM, sigalarm);
+	alarm(10);
+	start = time_now();
+	io_loop();
+	end = time_now();
+	close(fd);
+
+	printf("%u connections complete (%u ns per conn)\n",
+	       completed,
+	       (int)time_to_nsec(time_divide(time_sub(end, start), completed)));
+	return 0;
+}

ccan/ccan/io/benchmarks/run-length-prefix.c

@@ -0,0 +1,181 @@
+/* Simulate a server with connections of different speeds.  We count
+ * how many connections complete in 10 seconds. */
+#include <ccan/io/io.h>
+#include <ccan/time/time.h>
+#include <ccan/err/err.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdio.h>
+#include <signal.h>
+#include <assert.h>
+
+#define REQUEST_MAX 131072
+#define NUM_CONNS 500 /* per child */
+#define NUM_CHILDREN 2
+
+static unsigned int completed;
+
+struct client {
+	unsigned int len;
+	char *request_buffer;
+};
+
+static struct io_plan write_reply(struct io_conn *conn, struct client *client);
+static struct io_plan read_body(struct io_conn *conn, struct client *client)
+{
+	assert(client->len <= REQUEST_MAX);
+	return io_read(client->request_buffer, client->len,
+		       write_reply, client);
+}
+
+static struct io_plan io_read_header(struct client *client)
+{
+	return io_read(&client->len, sizeof(client->len), read_body, client);
+}
+
+/* once we're done, loop again. */
+static struct io_plan write_complete(struct io_conn *conn, struct client *client)
+{
+	completed++;
+	return io_read_header(client);
+}
+
+static struct io_plan write_reply(struct io_conn *conn, struct client *client)
+{
+	return io_write(&client->len, sizeof(client->len),
+			write_complete, client);
+}
+
+/* This runs in the child. */
+static void create_clients(struct sockaddr_un *addr, int waitfd)
+{
+	struct client data;
+	int i, sock[NUM_CONNS], len[NUM_CONNS], done[NUM_CONNS],
+		result[NUM_CONNS], count = 0;
+
+	for (i = 0; i < NUM_CONNS; i++) {
+		len[i] = (random() % REQUEST_MAX) + 1;
+		sock[i] = socket(AF_UNIX, SOCK_STREAM, 0);
+		if (sock[i] < 0)
+			err(1, "creating socket");
+		if (connect(sock[i], (void *)addr, sizeof(*addr)) != 0)
+			err(1, "connecting socket");
+		/* Make nonblocking. */
+		fcntl(sock[i], F_SETFD, fcntl(sock[i], F_GETFD)|O_NONBLOCK);
+		done[i] = 0;
+	}
+
+	read(waitfd, &i, 1);
+
+	for (;;) {
+		for (i = 0; i < NUM_CONNS; i++) {
+			int ret, totlen = len[i] + sizeof(len[i]);
+			if (done[i] < sizeof(len[i]) + len[i]) {
+				data.len = len[i];
+				ret = write(sock[i], (void *)&data + done[i],
+					    totlen - done[i]);
+				if (ret > 0)
+					done[i] += ret;
+				else if (ret < 0 && errno != EAGAIN)
+					goto fail;
+			} else {
+				int off = done[i] - totlen;
+				ret = read(sock[i], (void *)&result[i] + off,
+					   sizeof(result[i]) - off);
+				if (ret > 0) {
+					done[i] += ret;
+					if (done[i] == totlen
+					    + sizeof(result[i])) {
+						assert(result[i] == len[i]);
+						count++;
+						done[i] = 0;
+					}
+				} else if (ret < 0 && errno != EAGAIN)
+					goto fail;
+			}
+		}
+	}
+fail:
+	printf("Child did %u\n", count);
+	exit(0);
+}
+
+static int timeout[2];
+static void sigalarm(int sig)
+{
+	write(timeout[1], "1", 1);
+}
+
+static struct io_plan do_timeout(struct io_conn *conn, char *buf)
+{
+	return io_break(buf, io_idle());
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j;
+	struct sockaddr_un addr;
+	struct timespec start, end;
+	char buffer[REQUEST_MAX];
+	int fd, wake[2];
+	char buf;
+
+	addr.sun_family = AF_UNIX;
+	sprintf(addr.sun_path, "/tmp/run-different-speed.sock.%u", getpid());
+
+	if (pipe(wake) != 0 || pipe(timeout) != 0)
+		err(1, "Creating pipes");
+
+	fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd < 0)
+		err(1, "Creating socket");
+
+	if (bind(fd, (void *)&addr, sizeof(addr)) != 0)
+		err(1, "Binding to %s", addr.sun_path);
+
+	if (listen(fd, NUM_CONNS) != 0)
+		err(1, "Listening on %s", addr.sun_path);
+
+	for (i = 0; i < NUM_CHILDREN; i++) {
+		switch (fork()) {
+		case -1:
+			err(1, "forking");
+		case 0:
+			close(wake[1]);
+			create_clients(&addr, wake[0]);
+			break;
+		}
+		for (j = 0; j < NUM_CONNS; j++) {
+			struct client *client = malloc(sizeof(*client));
+			int ret = accept(fd, NULL, 0);
+			if (ret < 0)
+				err(1, "Accepting fd");
+			/* For efficiency, we share buffer */
+			client->request_buffer = buffer;
+			io_new_conn(ret, io_read_header(client));
+		}
+	}
+
+	io_new_conn(timeout[0], io_read(&buf, 1, do_timeout, &buf));
+
+	close(wake[0]);
+	for (i = 0; i < NUM_CHILDREN; i++)
+		write(wake[1], "1", 1);
+
+	signal(SIGALRM, sigalarm);
+	alarm(10);
+	start = time_now();
+	io_loop();
+	end = time_now();
+	close(fd);
+
+	printf("%u connections complete (%u ns per conn)\n",
+	       completed,
+	       (int)time_to_nsec(time_divide(time_sub(end, start), completed)));
+	return 0;
+}

ccan/ccan/io/benchmarks/run-loop.c

@@ -0,0 +1,112 @@
+#include <ccan/io/io.h>
+#include <ccan/time/time.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <err.h>
+#include <signal.h>
+
+#define NUM 500
+#define NUM_ITERS 10000
+
+struct buffer {
+	int iters;
+	struct io_conn *reader, *writer;
+	char buf[32];
+};
+
+static struct io_plan poke_reader(struct io_conn *conn, struct buffer *buf);
+
+static struct io_plan poke_writer(struct io_conn *conn, struct buffer *buf)
+{
+	assert(conn == buf->reader);
+
+	if (buf->iters == NUM_ITERS)
+		return io_close();
+
+	/* You write. */
+	io_wake(buf->writer,
+		io_write(&buf->buf, sizeof(buf->buf), poke_reader, buf));
+
+	/* I'll wait until you wake me. */
+	return io_idle();
+}
+
+static struct io_plan poke_reader(struct io_conn *conn, struct buffer *buf)
+{
+	assert(conn == buf->writer);
+	/* You read. */
+	io_wake(buf->reader,
+		io_read(&buf->buf, sizeof(buf->buf), poke_writer, buf));
+
+	if (++buf->iters == NUM_ITERS)
+		return io_close();
+
+	/* I'll wait until you tell me to write. */
+	return io_idle();
+}
+
+int main(void)
+{
+	unsigned int i;
+	int fds[2], last_read, last_write;
+	struct timespec start, end;
+	struct buffer buf[NUM];
+
+	if (pipe(fds) != 0)
+		err(1, "pipe");
+	last_read = fds[0];
+	last_write = fds[1];
+
+	for (i = 1; i < NUM; i++) {
+		buf[i].iters = 0;
+		if (pipe(fds) < 0)
+			err(1, "pipe");
+		memset(buf[i].buf, i, sizeof(buf[i].buf));
+		sprintf(buf[i].buf, "%i-%i", i, i);
+
+		buf[i].reader = io_new_conn(last_read, io_idle());
+		if (!buf[i].reader)
+			err(1, "Creating reader %i", i);
+		buf[i].writer = io_new_conn(fds[1],
+					    io_write(&buf[i].buf,
+						     sizeof(buf[i].buf),
+						     poke_reader, &buf[i]));
+		if (!buf[i].writer)
+			err(1, "Creating writer %i", i);
+		last_read = fds[0];
+	}
+
+	/* Last one completes the cirle. */
+	i = 0;
+	buf[i].iters = 0;
+	sprintf(buf[i].buf, "%i-%i", i, i);
+	buf[i].reader = io_new_conn(last_read, io_idle());
+	if (!buf[i].reader)
+		err(1, "Creating reader %i", i);
+	buf[i].writer = io_new_conn(last_write, io_write(&buf[i].buf,
+							 sizeof(buf[i].buf),
+							 poke_reader, &buf[i]));
+	if (!buf[i].writer)
+		err(1, "Creating writer %i", i);
+
+	/* They should eventually exit */
+	start = time_now();
+	if (io_loop() != NULL)
+		errx(1, "io_loop?");
+	end = time_now();
+
+	for (i = 0; i < NUM; i++) {
+		char b[sizeof(buf[0].buf)];
+		memset(b, i, sizeof(b));
+		sprintf(b, "%i-%i", i, i);
+		if (memcmp(b, buf[(i + NUM_ITERS) % NUM].buf, sizeof(b)) != 0)
+			errx(1, "Buffer for %i was '%s' not '%s'",
+			     i, buf[(i + NUM_ITERS) % NUM].buf, b);
+	}
+
+	printf("run-many: %u %u iterations: %llu usec\n",
+	       NUM, NUM_ITERS, (long long)time_to_usec(time_sub(end, start)));
+	return 0;
+}

ccan/ccan/io/io.c

@@ -0,0 +1,546 @@
+/* Licensed under LGPLv2.1+ - see LICENSE file for details */
+#include "io.h"
+#include "backend.h"
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <ccan/container_of/container_of.h>
+
+void *io_loop_return;
+
+struct io_listener *io_new_listener_(const tal_t *ctx, int fd,
+				     struct io_plan *(*init)(struct io_conn *,
+							     void *),
+				     void *arg)
+{
+	struct io_listener *l = tal(ctx, struct io_listener);
+	if (!l)
+		return NULL;
+
+	l->fd.listener = true;
+	l->fd.fd = fd;
+	l->init = init;
+	l->arg = arg;
+	l->ctx = ctx;
+	if (!add_listener(l))
+		return tal_free(l);
+	return l;
+}
+
+void io_close_listener(struct io_listener *l)
+{
+	close(l->fd.fd);
+	del_listener(l);
+	tal_free(l);
+}
+
+static struct io_plan *io_never_called(struct io_conn *conn, void *arg)
+{
+	abort();
+}
+
+static void next_plan(struct io_conn *conn, struct io_plan *plan)
+{
+	struct io_plan *(*next)(struct io_conn *, void *arg);
+
+	next = plan->next;
+
+	plan->status = IO_UNSET;
+	plan->io = NULL;
+	plan->next = io_never_called;
+
+	plan = next(conn, plan->next_arg);
+
+	/* It should have set a plan inside this conn (or duplex) */
+	assert(plan == &conn->plan[IO_IN]
+	       || plan == &conn->plan[IO_OUT]
+	       || plan == &conn->plan[2]);
+	assert(conn->plan[IO_IN].status != IO_UNSET
+	       || conn->plan[IO_OUT].status != IO_UNSET);
+
+	backend_new_plan(conn);
+}
+
+static void set_blocking(int fd, bool block)
+{
+	int flags = fcntl(fd, F_GETFL);
+
+	if (block)
+		flags &= ~O_NONBLOCK;
+	else
+		flags |= O_NONBLOCK;
+
+	fcntl(fd, F_SETFL, flags);
+}
+
+struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
+			     struct io_plan *(*init)(struct io_conn *, void *),
+			     void *arg)
+{
+	struct io_conn *conn = tal(ctx, struct io_conn);
+
+	if (!conn)
+		return NULL;
+
+	conn->fd.listener = false;
+	conn->fd.fd = fd;
+	conn->finish = NULL;
+	conn->finish_arg = NULL;
+	list_node_init(&conn->always);
+	list_node_init(&conn->closing);
+	conn->debug = false;
+
+	if (!add_conn(conn))
+		return tal_free(conn);
+
+	/* Keep our I/O async. */
+	set_blocking(fd, false);
+
+	/* We start with out doing nothing, and in doing our init. */
+	conn->plan[IO_OUT].status = IO_UNSET;
+
+	conn->plan[IO_IN].next = init;
+	conn->plan[IO_IN].next_arg = arg;
+	next_plan(conn, &conn->plan[IO_IN]);
+
+	return conn;
+}
+
+void io_set_finish_(struct io_conn *conn,
+		    void (*finish)(struct io_conn *, void *),
+		    void *arg)
+{
+	conn->finish = finish;
+	conn->finish_arg = arg;
+}
+
+struct io_plan_arg *io_plan_arg(struct io_conn *conn, enum io_direction dir)
+{
+	assert(conn->plan[dir].status == IO_UNSET);
+
+	conn->plan[dir].status = IO_POLLING;
+	return &conn->plan[dir].arg;
+}
+
+static struct io_plan *set_always(struct io_conn *conn,
+				  enum io_direction dir,
+				  struct io_plan *(*next)(struct io_conn *,
+							  void *),
+				  void *arg)
+{
+	struct io_plan *plan = &conn->plan[dir];
+
+	plan->status = IO_ALWAYS;
+	backend_new_always(conn);
+	return io_set_plan(conn, dir, NULL, next, arg);
+}
+
+static struct io_plan *io_always_dir(struct io_conn *conn,
+				     enum io_direction dir,
+				     struct io_plan *(*next)(struct io_conn *,
+							     void *),
+				     void *arg)
+{
+	return set_always(conn, dir, next, arg);
+}
+
+struct io_plan *io_always_(struct io_conn *conn,
+			   struct io_plan *(*next)(struct io_conn *, void *),
+			   void *arg)
+{
+	return io_always_dir(conn, IO_IN, next, arg);
+}
+
+struct io_plan *io_out_always_(struct io_conn *conn,
+			       struct io_plan *(*next)(struct io_conn *,
+						       void *),
+			       void *arg)
+{
+	return io_always_dir(conn, IO_OUT, next, arg);
+}
+
+static int do_write(int fd, struct io_plan_arg *arg)
+{
+	ssize_t ret = write(fd, arg->u1.cp, arg->u2.s);
+	if (ret < 0)
+		return -1;
+
+	arg->u1.cp += ret;
+	arg->u2.s -= ret;
+	return arg->u2.s == 0;
+}
+
+/* Queue some data to be written. */
+struct io_plan *io_write_(struct io_conn *conn, const void *data, size_t len,
+			  struct io_plan *(*next)(struct io_conn *, void *),
+			  void *next_arg)
+{
+	struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT);
+
+	if (len == 0)
+		return set_always(conn, IO_OUT, next, next_arg);
+
+	arg->u1.const_vp = data;
+	arg->u2.s = len;
+
+	return io_set_plan(conn, IO_OUT, do_write, next, next_arg);
+}
+
+static int do_read(int fd, struct io_plan_arg *arg)
+{
+	ssize_t ret = read(fd, arg->u1.cp, arg->u2.s);
+	if (ret <= 0)
+		return -1;
+
+	arg->u1.cp += ret;
+	arg->u2.s -= ret;
+	return arg->u2.s == 0;
+}
+
+/* Queue a request to read into a buffer. */
+struct io_plan *io_read_(struct io_conn *conn,
+			 void *data, size_t len,
+			 struct io_plan *(*next)(struct io_conn *, void *),
+			 void *next_arg)
+{
+	struct io_plan_arg *arg = io_plan_arg(conn, IO_IN);
+
+	if (len == 0)
+		return set_always(conn, IO_IN, next, next_arg);
+
+	arg->u1.cp = data;
+	arg->u2.s = len;
+
+	return io_set_plan(conn, IO_IN, do_read, next, next_arg);
+}
+
+static int do_read_partial(int fd, struct io_plan_arg *arg)
+{
+	ssize_t ret = read(fd, arg->u1.cp, *(size_t *)arg->u2.vp);
+	if (ret <= 0)
+		return -1;
+
+	*(size_t *)arg->u2.vp = ret;
+	return 1;
+}
+
+/* Queue a partial request to read into a buffer. */
+struct io_plan *io_read_partial_(struct io_conn *conn,
+				 void *data, size_t maxlen, size_t *len,
+				 struct io_plan *(*next)(struct io_conn *,
+							 void *),
+				 void *next_arg)
+{
+	struct io_plan_arg *arg = io_plan_arg(conn, IO_IN);
+
+	if (maxlen == 0)
+		return set_always(conn, IO_IN, next, next_arg);
+
+	arg->u1.cp = data;
+	/* We store the max len in here temporarily. */
+	*len = maxlen;
+	arg->u2.vp = len;
+
+	return io_set_plan(conn, IO_IN, do_read_partial, next, next_arg);
+}
+
+static int do_write_partial(int fd, struct io_plan_arg *arg)
+{
+	ssize_t ret = write(fd, arg->u1.cp, *(size_t *)arg->u2.vp);
+	if (ret < 0)
+		return -1;
+
+	*(size_t *)arg->u2.vp = ret;
+	return 1;
+}
+
+/* Queue a partial write request. */
+struct io_plan *io_write_partial_(struct io_conn *conn,
+				  const void *data, size_t maxlen, size_t *len,
+				  struct io_plan *(*next)(struct io_conn *,
+							  void*),
+				  void *next_arg)
+{
+	struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT);
+
+	if (maxlen == 0)
+		return set_always(conn, IO_OUT, next, next_arg);
+
+	arg->u1.const_vp = data;
+	/* We store the max len in here temporarily. */
+	*len = maxlen;
+	arg->u2.vp = len;
+
+	return io_set_plan(conn, IO_OUT, do_write_partial, next, next_arg);
+}
+
+static int do_connect(int fd, struct io_plan_arg *arg)
+{
+	int err, ret;
+	socklen_t len = sizeof(err);
+
+	/* Has async connect finished? */
+	ret = getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len);
+	if (ret < 0)
+		return -1;
+
+	if (err == 0) {
+		return 1;
+	} else if (err == EINPROGRESS)
+		return 0;
+
+	errno = err;
+	return -1;
+}
+
+struct io_plan *io_connect_(struct io_conn *conn, const struct addrinfo *addr,
+			    struct io_plan *(*next)(struct io_conn *, void *),
+			    void *next_arg)
+{
+	int fd = io_conn_fd(conn);
+
+	/* We don't actually need the arg, but we need it polling. */
+	io_plan_arg(conn, IO_OUT);
+
+	/* Note that io_new_conn() will make fd O_NONBLOCK */
+
+	/* Immediate connect can happen. */
+	if (connect(fd, addr->ai_addr, addr->ai_addrlen) == 0)
+		return set_always(conn, IO_OUT, next, next_arg);
+
+	if (errno != EINPROGRESS)
+		return io_close(conn);
+
+	return io_set_plan(conn, IO_OUT, do_connect, next, next_arg);
+}
+
+static struct io_plan *io_wait_dir(struct io_conn *conn,
+				   const void *wait,
+				   enum io_direction dir,
+				   struct io_plan *(*next)(struct io_conn *,
+							   void *),
+				   void *next_arg)
+{
+	struct io_plan_arg *arg = io_plan_arg(conn, dir);
+	arg->u1.const_vp = wait;
+
+	conn->plan[dir].status = IO_WAITING;
+
+	return io_set_plan(conn, dir, NULL, next, next_arg);
+}
+
+struct io_plan *io_wait_(struct io_conn *conn,
+			 const void *wait,
+			 struct io_plan *(*next)(struct io_conn *, void *),
+			 void *next_arg)
+{
+	return io_wait_dir(conn, wait, IO_IN, next, next_arg);
+}
+
+struct io_plan *io_out_wait_(struct io_conn *conn,
+			     const void *wait,
+			     struct io_plan *(*next)(struct io_conn *, void *),
+			     void *next_arg)
+{
+	return io_wait_dir(conn, wait, IO_OUT, next, next_arg);
+}
+
+void io_wake(const void *wait)
+{
+	backend_wake(wait);
+}
+
+static int do_plan(struct io_conn *conn, struct io_plan *plan)
+{
+	/* Someone else might have called io_close() on us. */
+	if (plan->status == IO_CLOSING)
+		return -1;
+
+	/* We shouldn't have polled for this event if this wasn't true! */
+	assert(plan->status == IO_POLLING);
+
+	switch (plan->io(conn->fd.fd, &plan->arg)) {
+	case -1:
+		io_close(conn);
+		return -1;
+	case 0:
+		return 0;
+	case 1:
+		next_plan(conn, plan);
+		return 1;
+	default:
+		/* IO should only return -1, 0 or 1 */
+		abort();
+	}
+}
+
+void io_ready(struct io_conn *conn, int pollflags)
+{
+	if (pollflags & POLLIN)
+		do_plan(conn, &conn->plan[IO_IN]);
+
+	if (pollflags & POLLOUT)
+		do_plan(conn, &conn->plan[IO_OUT]);
+}
+
+void io_do_always(struct io_conn *conn)
+{
+	if (conn->plan[IO_IN].status == IO_ALWAYS)
+		next_plan(conn, &conn->plan[IO_IN]);
+
+	if (conn->plan[IO_OUT].status == IO_ALWAYS)
+		next_plan(conn, &conn->plan[IO_OUT]);
+}
+
+void io_do_wakeup(struct io_conn *conn, enum io_direction dir)
+{
+	struct io_plan *plan = &conn->plan[dir];
+
+	assert(plan->status == IO_WAITING);
+
+	set_always(conn, dir, plan->next, plan->next_arg);
+}
+
+/* Close the connection, we're done. */
+struct io_plan *io_close(struct io_conn *conn)
+{
+	/* Already closing?  Don't close twice. */
+	if (conn->plan[IO_IN].status == IO_CLOSING)
+		return &conn->plan[IO_IN];
+
+	conn->plan[IO_IN].status = conn->plan[IO_OUT].status = IO_CLOSING;
+	conn->plan[IO_IN].arg.u1.s = errno;
+	backend_new_closing(conn);
+
+	return io_set_plan(conn, IO_IN, NULL, NULL, NULL);
+}
+
+struct io_plan *io_close_cb(struct io_conn *conn, void *next_arg)
+{
+	return io_close(conn);
+}
+
+/* Exit the loop, returning this (non-NULL) arg. */
+void io_break(const void *ret)
+{
+	assert(ret);
+	io_loop_return = (void *)ret;
+}
+
+struct io_plan *io_never(struct io_conn *conn, void *unused)
+{
+	return io_always(conn, io_never_called, NULL);
+}
+
+int io_conn_fd(const struct io_conn *conn)
+{
+	return conn->fd.fd;
+}
+
+void io_duplex_prepare(struct io_conn *conn)
+{
+	assert(conn->plan[IO_IN].status == IO_UNSET);
+	assert(conn->plan[IO_OUT].status == IO_UNSET);
+
+	/* We can't sync debug until we've set both: io_wait() and io_always
+	 * can't handle it. */
+	conn->debug_saved = conn->debug;
+	io_set_debug(conn, false);
+}
+
+struct io_plan *io_duplex_(struct io_plan *in_plan, struct io_plan *out_plan)
+{
+	struct io_conn *conn;
+
+	/* in_plan must be conn->plan[IO_IN], out_plan must be [IO_OUT] */
+	assert(out_plan == in_plan + 1);
+
+	/* Restore debug. */
+	conn = container_of(in_plan, struct io_conn, plan[IO_IN]);
+	io_set_debug(conn, conn->debug_saved);
+
+	/* Now set the plans again, to invoke sync debug. */
+	io_set_plan(conn, IO_OUT,
+		    out_plan->io, out_plan->next, out_plan->next_arg);
+	io_set_plan(conn, IO_IN,
+		    in_plan->io, in_plan->next, in_plan->next_arg);
+
+	return out_plan + 1;
+}
+
+struct io_plan *io_halfclose(struct io_conn *conn)
+{
+	/* Already closing?  Don't close twice. */
+	if (conn->plan[IO_IN].status == IO_CLOSING)
+		return &conn->plan[IO_IN];
+
+	/* Both unset?  OK. */
+	if (conn->plan[IO_IN].status == IO_UNSET
+	    && conn->plan[IO_OUT].status == IO_UNSET)
+		return io_close(conn);
+
+	/* We leave this unset then. */
+	if (conn->plan[IO_IN].status == IO_UNSET)
+		return &conn->plan[IO_IN];
+	else
+		return &conn->plan[IO_OUT];
+}
+
+struct io_plan *io_set_plan(struct io_conn *conn, enum io_direction dir,
+			    int (*io)(int fd, struct io_plan_arg *arg),
+			    struct io_plan *(*next)(struct io_conn *, void *),
+			    void *next_arg)
+{
+	struct io_plan *plan = &conn->plan[dir];
+
+	plan->io = io;
+	plan->next = next;
+	plan->next_arg = next_arg;
+	assert(plan->status == IO_CLOSING || next != NULL);
+
+	if (!conn->debug)
+		return plan;
+
+	if (io_loop_return) {
+		io_debug_complete(conn);
+		return plan;
+	}
+
+	switch (plan->status) {
+	case IO_POLLING:
+		while (do_plan(conn, plan) == 0);
+		break;
+	/* Shouldn't happen, since you said you did plan! */
+	case IO_UNSET:
+		abort();
+	case IO_ALWAYS:
+		/* If other one is ALWAYS, leave in list! */
+		if (conn->plan[!dir].status != IO_ALWAYS)
+			remove_from_always(conn);
+		next_plan(conn, plan);
+		break;
+	case IO_WAITING:
+	case IO_CLOSING:
+		io_debug_complete(conn);
+	}
+
+	return plan;
+}
+
+void io_set_debug(struct io_conn *conn, bool debug)
+{
+	conn->debug = debug;
+
+	/* Debugging means fds must block. */
+	set_blocking(io_conn_fd(conn), debug);
+}
+
+void io_debug_complete(struct io_conn *conn)
+{
+}

ccan/ccan/io/io.h

@@ -0,0 +1,695 @@
+/* Licensed under LGPLv2.1+ - see LICENSE file for details */
+#ifndef CCAN_IO_H
+#define CCAN_IO_H
+#include <ccan/tal/tal.h>
+#include <ccan/typesafe_cb/typesafe_cb.h>
+#include <stdbool.h>
+#include <unistd.h>
+
+struct timers;
+struct timer;
+struct list_head;
+
+/**
+ * struct io_plan - a plan for input or output.
+ *
+ * Each io_conn has zero to two of these active at any time.
+ */
+struct io_plan;
+
+/**
+ * struct io_conn - a connection associated with an fd.
+ */
+struct io_conn;
+
+/**
+ * io_new_conn - create a new connection.
+ * @ctx: the context to tal from (or NULL)
+ * @fd: the file descriptor.
+ * @init: the function to call for a new connection
+ * @arg: the argument to @init.
+ *
+ * This creates a connection which owns @fd, it then calls
+ * @init to initialize the connection, which sets up an io_plan.
+ *
+ * Returns NULL on error (and sets errno).
+ *
+ * Example:
+ * // Dumb init function to print string and tell conn to close.
+ * static struct io_plan *conn_init(struct io_conn *conn, const char *msg)
+ * {
+ *	printf("Created conn %p: %s", conn, msg);
+ *	return io_close(conn);
+ * }
+ *
+ * static void create_self_closing_pipe(void)
+ * {
+ *	int fd[2];
+ *	struct io_conn *conn;
+ *
+ *	pipe(fd);
+ *	conn = io_new_conn(NULL, fd[0], conn_init, (const char *)"hi!");
+ *	if (!conn)
+ *		exit(1);
+ * }
+ */
+#define io_new_conn(ctx, fd, init, arg)					\
+	io_new_conn_((ctx), (fd),					\
+		     typesafe_cb_preargs(struct io_plan *, void *,	\
+					 (init), (arg),			\
+					 struct io_conn *conn),		\
+		     (void *)(arg))
+
+struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
+			     struct io_plan *(*init)(struct io_conn *, void *),
+			     void *arg);
+
+/**
+ * io_set_finish - set finish function on a connection.
+ * @conn: the connection.
+ * @finish: the function to call when it's closed or fails.
+ * @arg: the argument to @finish.
+ *
+ * @finish will be called when an I/O operation fails, or you call
+ * io_close() on the connection.  errno will be set to the value
+ * after the failed I/O, or at the call to io_close().  The fd
+ * will be closed before @finish is called.
+ *
+ * Example:
+ * static void finish(struct io_conn *conn, const char *msg)
+ * {
+ *	// errno is not 0 after success, so this is a bit useless.
+ *	printf("Conn %p closed with errno %i (%s)\n", conn, errno, msg);
+ * }
+ *
+ * // Dumb init function to print string and tell conn to close.
+ * static struct io_plan *conn_init(struct io_conn *conn, const char *msg)
+ * {
+ *	io_set_finish(conn, finish, msg);
+ *	return io_close(conn);
+ * }
+ */
+#define io_set_finish(conn, finish, arg)				\
+	io_set_finish_((conn),						\
+		       typesafe_cb_preargs(void, void *,		\
+					   (finish), (arg),		\
+					   struct io_conn *),		\
+		       (void *)(arg))
+void io_set_finish_(struct io_conn *conn,
+		    void (*finish)(struct io_conn *, void *),
+		    void *arg);
+
+
+/**
+ * io_new_listener - create a new accepting listener.
+ * @ctx: the context to tal from (or NULL)
+ * @fd: the file descriptor.
+ * @init: the function to call for a new connection
+ * @arg: the argument to @init.
+ *
+ * When @fd becomes readable, we accept(), create a new connection,
+ * (tal'ocated off @ctx) and pass that to init().
+ *
+ * Returns NULL on error (and sets errno).
+ *
+ * Example:
+ * #include <sys/types.h>
+ * #include <sys/socket.h>
+ * #include <netdb.h>
+ *
+ * ...
+ *
+ * // Set up a listening socket, return it.
+ * static struct io_listener *do_listen(const char *port)
+ * {
+ *	struct addrinfo *addrinfo, hints;
+ *	int fd, on = 1;
+ *
+ *	memset(&hints, 0, sizeof(hints));
+ *	hints.ai_family = AF_UNSPEC;
+ *	hints.ai_socktype = SOCK_STREAM;
+ *	hints.ai_flags = AI_PASSIVE;
+ *	hints.ai_protocol = 0;
+ *
+ *	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+ *		return NULL;
+ *
+ *	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+ *		    addrinfo->ai_protocol);
+ *	if (fd < 0)
+ *		return NULL;
+ *
+ *	freeaddrinfo(addrinfo);
+ *	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+ *	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+ *		close(fd);
+ *		return NULL;
+ *	}
+ *	if (listen(fd, 1) != 0) {
+ *		close(fd);
+ *		return NULL;
+ *	}
+ *	return io_new_listener(NULL, fd, conn_init, (const char *)"listened!");
+ * }
+ */
+#define io_new_listener(ctx, fd, init, arg)				\
+	io_new_listener_((ctx), (fd),					\
+			 typesafe_cb_preargs(struct io_plan *, void *,	\
+					     (init), (arg),		\
+					     struct io_conn *conn),	\
+			 (void *)(arg))
+struct io_listener *io_new_listener_(const tal_t *ctx, int fd,
+				     struct io_plan *(*init)(struct io_conn *,
+							     void *),
+				     void *arg);
+
+/**
+ * io_close_listener - delete a listener.
+ * @listener: the listener returned from io_new_listener.
+ *
+ * This closes the fd and frees @listener.
+ *
+ * Example:
+ * ...
+ *	struct io_listener *l = do_listen("8111");
+ *	if (l) {
+ *		io_loop(NULL, NULL);
+ *		io_close_listener(l);
+ *	}
+ */
+void io_close_listener(struct io_listener *listener);
+
+/**
+ * io_write - output plan to write data.
+ * @conn: the connection that plan is for.
+ * @data: the data buffer.
+ * @len: the length to write.
+ * @next: function to call output is done.
+ * @arg: @next argument
+ *
+ * This updates the output plan, to write out a data buffer.  Once it's all
+ * written, the @next function will be called: on an error, the finish
+ * function is called instead.
+ *
+ * Note that the I/O may actually be done immediately.
+ *
+ * Example:
+ * static struct io_plan *write_to_conn(struct io_conn *conn, const char *msg)
+ * {
+ *	// Write message, then close.
+ *	return io_write(conn, msg, strlen(msg), io_close_cb, NULL);
+ * }
+ */
+#define io_write(conn, data, len, next, arg)				\
+	io_write_((conn), (data), (len),				\
+		  typesafe_cb_preargs(struct io_plan *, void *,		\
+				      (next), (arg), struct io_conn *),	\
+		  (arg))
+struct io_plan *io_write_(struct io_conn *conn,
+			  const void *data, size_t len,
+			  struct io_plan *(*next)(struct io_conn *, void *),
+			  void *arg);
+
+/**
+ * io_read - input plan to read data.
+ * @conn: the connection that plan is for.
+ * @data: the data buffer.
+ * @len: the length to read.
+ * @next: function to call once input is done.
+ * @arg: @next argument
+ *
+ * This creates a plan to read data into a buffer.  Once it's all
+ * read, the @next function will be called: on an error, the finish
+ * function is called instead.
+ *
+ * Note that the I/O may actually be done immediately.
+ *
+ * Example:
+ * static struct io_plan *read_from_conn(struct io_conn *conn, char *buf)
+ * {
+ *	// Read message, then close.
+ *	return io_read(conn, buf, 12, io_close_cb, NULL);
+ * }
+ */
+#define io_read(conn, data, len, next, arg)				\
+	io_read_((conn), (data), (len),					\
+		 typesafe_cb_preargs(struct io_plan *, void *,		\
+				     (next), (arg), struct io_conn *),	\
+		 (arg))
+struct io_plan *io_read_(struct io_conn *conn,
+			 void *data, size_t len,
+			 struct io_plan *(*next)(struct io_conn *, void *),
+			 void *arg);
+
+
+/**
+ * io_read_partial - input plan to read some data.
+ * @conn: the connection that plan is for.
+ * @data: the data buffer.
+ * @maxlen: the maximum length to read
+ * @lenp: set to the length actually read.
+ * @next: function to call once input is done.
+ * @arg: @next argument
+ *
+ * This creates a plan to read data into a buffer.  Once any data is
+ * read, @len is updated and the @next function will be called: on an
+ * error, the finish function is called instead.
+ *
+ * Note that the I/O may actually be done immediately.
+ *
+ * Example:
+ * struct buf {
+ *	size_t len;
+ *	char buf[12];
+ * };
+ *
+ * static struct io_plan *dump(struct io_conn *conn, struct buf *b)
+ * {
+ *	printf("Partial read: '%*s'\n", (int)b->len, b->buf);
+ *	free(b);
+ *	return io_close(conn);
+ * }
+ *
+ * static struct io_plan *read_part(struct io_conn *conn, struct buf *b)
+ * {
+ *	// Read message, then dump and close.
+ *	return io_read_partial(conn, b->buf, sizeof(b->buf), &b->len, dump, b);
+ * }
+ */
+#define io_read_partial(conn, data, maxlen, lenp, next, arg)		\
+	io_read_partial_((conn), (data), (maxlen), (lenp),		\
+			 typesafe_cb_preargs(struct io_plan *, void *,	\
+					     (next), (arg),		\
+					     struct io_conn *),		\
+			 (arg))
+struct io_plan *io_read_partial_(struct io_conn *conn,
+				 void *data, size_t maxlen, size_t *lenp,
+				 struct io_plan *(*next)(struct io_conn *,
+							 void *),
+				 void *arg);
+
+/**
+ * io_write_partial - output plan to write some data.
+ * @conn: the connection that plan is for.
+ * @data: the data buffer.
+ * @maxlen: the maximum length to write
+ * @lenp: set to the length actually written.
+ * @next: function to call once output is done.
+ * @arg: @next argument
+ *
+ * This creates a plan to write data from a buffer.   Once any data is
+ * written, @len is updated and the @next function will be called: on an
+ * error, the finish function is called instead.
+ *
+ * Note that the I/O may actually be done immediately.
+ *
+ * Example:
+ * struct buf {
+ *	size_t len;
+ *	char buf[12];
+ * };
+ *
+ * static struct io_plan *show_partial(struct io_conn *conn, struct buf *b)
+ * {
+ *	printf("Only wrote: '%*s'\n", (int)b->len, b->buf);
+ *	free(b);
+ *	return io_close(conn);
+ * }
+ *
+ * static struct io_plan *write_part(struct io_conn *conn, struct buf *b)
+ * {
+ *	// Write message, then dump and close.
+ *	strcpy(b->buf, "Hello world");
+ *	return io_write_partial(conn, b->buf, strlen(b->buf),
+ *				&b->len, show_partial, b);
+ * }
+ */
+#define io_write_partial(conn, data, maxlen, lenp, next, arg)		\
+	io_write_partial_((conn), (data), (maxlen), (lenp),		\
+			  typesafe_cb_preargs(struct io_plan *, void *,	\
+					      (next), (arg),		\
+					      struct io_conn *),	\
+			  (arg))
+struct io_plan *io_write_partial_(struct io_conn *conn,
+				  const void *data, size_t maxlen, size_t *lenp,
+				  struct io_plan *(*next)(struct io_conn *,
+							  void*),
+				  void *arg);
+
+/**
+ * io_always - plan to immediately call next callback
+ * @conn: the connection that plan is for.
+ * @next: function to call.
+ * @arg: @next argument
+ *
+ * Sometimes it's neater to plan a callback rather than call it directly;
+ * for example, if you only need to read data for one path and not another.
+ *
+ * Example:
+ * static struct io_plan *init_conn_with_nothing(struct io_conn *conn,
+ *						 void *unused)
+ * {
+ *	// Silly example: close on next time around loop.
+ *	return io_always(conn, io_close_cb, NULL);
+ * }
+ */
+#define io_always(conn, next, arg)					\
+	io_always_((conn), typesafe_cb_preargs(struct io_plan *, void *, \
+					       (next), (arg),		\
+					       struct io_conn *),	\
+		   (arg))
+
+struct io_plan *io_always_(struct io_conn *conn,
+			   struct io_plan *(*next)(struct io_conn *, void *),
+			   void *arg);
+
+/**
+ * io_out_always - output plan to immediately call next callback
+ * @conn: the connection that plan is for.
+ * @next: function to call.
+ * @arg: @next argument
+ *
+ * This is a variant of io_always() which uses the output plan; it only
+ * matters if you are using io_duplex, and thus have two plans running at
+ * once.
+ */
+#define io_out_always(conn, next, arg)					\
+	io_out_always_((conn), typesafe_cb_preargs(struct io_plan *, void *, \
+						   (next), (arg),	\
+						   struct io_conn *),	\
+		       (arg))
+
+struct io_plan *io_out_always_(struct io_conn *conn,
+			       struct io_plan *(*next)(struct io_conn *,
+						       void *),
+			       void *arg);
+
+/**
+ * io_connect - create an asynchronous connection to a listening socket.
+ * @conn: the connection that plan is for.
+ * @addr: where to connect.
+ * @init: function to call once it's connected
+ * @arg: @init argument
+ *
+ * This initiates a connection, and creates a plan for
+ * (asynchronously) completing it.  Once complete, the @init function
+ * will be called.
+ *
+ * Example:
+ * #include <sys/types.h>
+ * #include <sys/socket.h>
+ * #include <netdb.h>
+ *
+ * // Write, then close socket.
+ * static struct io_plan *init_connect(struct io_conn *conn,
+ *				       struct addrinfo *addrinfo)
+ * {
+ *	return io_connect(conn, addrinfo, io_close_cb, NULL);
+ * }
+ *
+ * ...
+ *
+ *	int fd;
+ *	struct addrinfo *addrinfo;
+ *
+ *	fd = socket(AF_INET, SOCK_STREAM, 0);
+ *	getaddrinfo("localhost", "8111", NULL, &addrinfo);
+ *	io_new_conn(NULL, fd, init_connect, addrinfo);
+ */
+struct addrinfo;
+#define io_connect(conn, addr, next, arg)				\
+	io_connect_((conn), (addr),					\
+		    typesafe_cb_preargs(struct io_plan *, void *,	\
+					(next), (arg),			\
+					struct io_conn *),		\
+		    (arg))
+
+struct io_plan *io_connect_(struct io_conn *conn, const struct addrinfo *addr,
+			    struct io_plan *(*next)(struct io_conn *, void *),
+			    void *arg);
+
+/**
+ * io_duplex - set plans for both input and output.
+ * @conn: the connection that plan is for.
+ * @in: the input plan
+ * @out: the output plan
+ *
+ * Most plans are either for input or output; io_duplex creates a plan
+ * which does both.  This is often used in the init function to create
+ * two independent streams, though it can be used once on any connection.
+ *
+ * Note that if either plan closes the connection, it will be closed.
+ *
+ * Example:
+ * struct buf {
+ *	char in[100];
+ *	char out[100];
+ * };
+ *
+ * static struct io_plan *read_and_write(struct io_conn *conn, struct buf *b)
+ * {
+ *	return io_duplex(conn,
+ *			 io_read(conn, b->in, sizeof(b->in), io_close_cb, b),
+ *			 io_write(conn, b->out, sizeof(b->out), io_close_cb,b));
+ * }
+ */
+#define io_duplex(conn, in_plan, out_plan) \
+	(io_duplex_prepare(conn), io_duplex_(in_plan, out_plan))
+
+struct io_plan *io_duplex_(struct io_plan *in_plan, struct io_plan *out_plan);
+void io_duplex_prepare(struct io_conn *conn);
+
+/**
+ * io_halfclose - close half of an io_duplex connection.
+ * @conn: the connection that plan is for.
+ *
+ * It's common to want to close a duplex connection after both input and
+ * output plans have completed.  If either calls io_close() the connection
+ * closes immediately.  Instead, io_halfclose() needs to be called twice.
+ *
+ * Example:
+ * struct buf {
+ *	char in[100];
+ *	char out[100];
+ * };
+ *
+ * static struct io_plan *finish(struct io_conn *conn, struct buf *b)
+ * {
+ *	return io_halfclose(conn);
+ * }
+ *
+ * static struct io_plan *read_and_write(struct io_conn *conn, struct buf *b)
+ * {
+ *	return io_duplex(conn,
+ *			 io_read(conn, b->in, sizeof(b->in), finish, b),
+ *			 io_write(conn, b->out, sizeof(b->out), finish, b));
+ * }
+ */
+struct io_plan *io_halfclose(struct io_conn *conn);
+
+/**
+ * io_wait - leave a plan idle until something wakes us.
+ * @conn: the connection that plan is for.
+ * @waitaddr: the address to wait on.
+ * @next: function to call after waiting.
+ * @arg: @next argument
+ *
+ * This leaves the input or output idle: io_wake(@waitaddr) will be
+ * called later to restart the connection.
+ *
+ * Example:
+ * // Silly example to wait then close.
+ * static struct io_plan *wait(struct io_conn *conn, void *b)
+ * {
+ *	return io_wait(conn, b, io_close_cb, NULL);
+ * }
+ */
+#define io_wait(conn, waitaddr, next, arg)				\
+	io_wait_((conn), (waitaddr),					\
+		 typesafe_cb_preargs(struct io_plan *, void *,		\
+				     (next), (arg),			\
+				     struct io_conn *),			\
+		 (arg))
+
+struct io_plan *io_wait_(struct io_conn *conn,
+			 const void *wait,
+			 struct io_plan *(*next)(struct io_conn *, void *),
+			 void *arg);
+
+
+/**
+ * io_out_wait - leave the output plan idle until something wakes us.
+ * @conn: the connection that plan is for.
+ * @waitaddr: the address to wait on.
+ * @next: function to call after waiting.
+ * @arg: @next argument
+ *
+ * io_wait() makes the input plan idle: if you're not using io_duplex it
+ * doesn't matter which plan is waiting.  Otherwise, you may need to use
+ * io_out_wait() instead, to specify explicitly that the output plan is
+ * waiting.
+ */
+#define io_out_wait(conn, waitaddr, next, arg)				\
+	io_out_wait_((conn), (waitaddr),				\
+		     typesafe_cb_preargs(struct io_plan *, void *,	\
+					 (next), (arg),			\
+					 struct io_conn *),		\
+		     (arg))
+
+struct io_plan *io_out_wait_(struct io_conn *conn,
+			     const void *wait,
+			     struct io_plan *(*next)(struct io_conn *, void *),
+			     void *arg);
+
+/**
+ * io_wake - wake up any connections waiting on @wait
+ * @waitaddr: the address to trigger.
+ *
+ * All io_conns who have returned io_wait() on @waitaddr will move on
+ * to their next callback.
+ *
+ * Example:
+ * static struct io_plan *wake_it(struct io_conn *conn, void *b)
+ * {
+ *	io_wake(b);
+ *	return io_close(conn);
+ * }
+ */
+void io_wake(const void *wait);
+
+/**
+ * io_break - return from io_loop()
+ * @ret: non-NULL value to return from io_loop().
+ *
+ * This breaks out of the io_loop.  As soon as the current function
+ * returns, any io_close()'d connections will have their finish
+ * callbacks called, then io_loop() with return with @ret.
+ *
+ * If io_loop() is called again, then @plan will be carried out.
+ *
+ * Example:
+ *	static struct io_plan *fail_on_timeout(struct io_conn *conn, char *msg)
+ *	{
+ *		io_break(msg);
+ *		return io_close(conn);
+ *	}
+ */
+void io_break(const void *ret);
+
+/**
+ * io_never - assert if callback is called.
+ * @conn: the connection that plan is for.
+ * @unused: an unused parameter to make this suitable for use as a callback.
+ *
+ * Sometimes you want to make it clear that a callback should never happen
+ * (eg. for io_break).  This will assert() if called.
+ *
+ * Example:
+ * static struct io_plan *break_out(struct io_conn *conn, void *unused)
+ * {
+ *	io_break(conn);
+ *	// We won't ever return from io_break
+ *	return io_never(conn, NULL);
+ * }
+ */
+struct io_plan *io_never(struct io_conn *conn, void *unused);
+
+/* FIXME: io_recvfrom/io_sendto */
+
+/**
+ * io_close - plan to close a connection.
+ * @conn: the connection to close.
+ *
+ * On return to io_loop, the connection will be closed.  It doesn't have
+ * to be the current connection and it doesn't need to be idle.  No more
+ * IO or callbacks will occur.
+ *
+ * You can close a connection twice without harmful effects.
+ *
+ * Example:
+ * static struct io_plan *close_on_timeout(struct io_conn *conn, const char *msg)
+ * {
+ *	printf("closing: %s\n", msg);
+ *	return io_close(conn);
+ * }
+ */
+struct io_plan *io_close(struct io_conn *conn);
+
+/**
+ * io_close_cb - helper callback to close a connection.
+ * @conn: the connection.
+ *
+ * This schedules a connection to be closed; designed to be used as
+ * a callback function.
+ *
+ * Example:
+ *	#define close_on_timeout io_close_cb
+ */
+struct io_plan *io_close_cb(struct io_conn *, void *unused);
+
+/**
+ * io_loop - process fds until all closed on io_break.
+ * @timers - timers which are waiting to go off (or NULL for none)
+ * @expired - an expired timer (can be NULL if @timers is)
+ *
+ * This is the core loop; it exits with the io_break() arg, or NULL if
+ * all connections and listeners are closed, or with @expired set to an
+ * expired timer (if @timers isn't NULL).
+ *
+ * Example:
+ *	io_loop(NULL, NULL);
+ */
+void *io_loop(struct timers *timers, struct timer **expired);
+
+/**
+ * io_conn_fd - get the fd from a connection.
+ * @conn: the connection.
+ *
+ * Sometimes useful, eg for getsockname().
+ */
+int io_conn_fd(const struct io_conn *conn);
+
+/**
+ * io_time_override - override the normal call for time.
+ * @nowfn: the function to call.
+ *
+ * io usually uses time_now() internally, but this forces it
+ * to use your function (eg. for debugging).  Returns the old
+ * one.
+ */
+struct timeabs (*io_time_override(struct timeabs (*now)(void)))(void);
+
+/**
+ * io_set_debug - set synchronous mode on a connection.
+ * @conn: the connection.
+ * @debug: whether to enable or disable debug.
+ *
+ * Once @debug is true on a connection, all I/O is done synchronously
+ * as soon as it is set, until it is unset or @conn is closed.  This
+ * makes it easy to debug what's happening with a connection, but note
+ * that other connections are starved while this is being done.
+ *
+ * See also: io_debug_complete()
+ *
+ * Example:
+ * // Dumb init function to set debug and tell conn to close.
+ * static struct io_plan *conn_init(struct io_conn *conn, const char *msg)
+ * {
+ *	io_set_debug(conn, true);
+ *	return io_close(conn);
+ * }
+ */
+void io_set_debug(struct io_conn *conn, bool debug);
+
+/**
+ * io_debug_complete - empty function called when conn is closing/waiting.
+ * @conn: the connection.
+ *
+ * This is for putting a breakpoint onto, when debugging.  It is called
+ * when a conn with io_set_debug() true can no longer be synchronous:
+ * 1) It is io_close()'d
+ * 2) It enters io_wait() (sychronous debug will resume after io_wake())
+ * 3) io_break() is called (sychronous debug will resume after io_loop())
+ */
+void io_debug_complete(struct io_conn *conn);
+#endif /* CCAN_IO_H */

ccan/ccan/io/io_plan.h

@@ -0,0 +1,78 @@
+/* Licensed under LGPLv2.1+ - see LICENSE file for details */
+#ifndef CCAN_IO_PLAN_H
+#define CCAN_IO_PLAN_H
+struct io_conn;
+
+/**
+ * union io_plan_union - type for struct io_plan read/write fns.
+ */
+union io_plan_union {
+	char *cp;
+	void *vp;
+	const void *const_vp;
+	size_t s;
+	char c[sizeof(size_t)];
+};
+
+/**
+ * struct io_plan_arg - scratch space for struct io_plan read/write fns.
+ */
+struct io_plan_arg {
+	union io_plan_union u1, u2;
+};
+
+enum io_direction {
+	IO_IN,
+	IO_OUT
+};
+
+/**
+ * io_plan_arg - get a conn's io_plan_arg for a given direction.
+ * @conn: the connection.
+ * @dir: IO_IN or IO_OUT.
+ *
+ * This is how an io helper gets scratch space to store into; you must call
+ * io_set_plan() when you've initialized it.
+ *
+ * Example:
+ * #include <ccan/io/io_plan.h>
+ *
+ * // Simple helper to read a single char.
+ * static int do_readchar(int fd, struct io_plan_arg *arg)
+ * {
+ *	return read(fd, arg->u1.cp, 1) <= 0 ? -1 : 1;
+ * }
+ *
+ * static struct io_plan *io_read_char_(struct io_conn *conn, char *in,
+ *				 struct io_plan *(*next)(struct io_conn*,void*),
+ *				 void *next_arg)
+ * {
+ *	struct io_plan_arg *arg = io_plan_arg(conn, IO_IN);
+ *
+ *	// Store information we need in the plan unions u1 and u2.
+ *	arg->u1.cp = in;
+ *
+ *	return io_set_plan(conn, IO_IN, do_readchar, next, next_arg);
+ * }
+ */
+struct io_plan_arg *io_plan_arg(struct io_conn *conn, enum io_direction dir);
+
+/**
+ * io_set_plan - set a conn's io_plan.
+ * @conn: the connection.
+ * @dir: IO_IN or IO_OUT.
+ * @io: the IO function to call when the fd is ready.
+ * @next: the next callback when @io returns 1.
+ * @next_arg: the argument to @next.
+ *
+ * If @conn has debug set, the io function will be called immediately,
+ * so it's important that this be the last thing in your function!
+ *
+ * See also:
+ *	io_get_plan_arg()
+ */
+struct io_plan *io_set_plan(struct io_conn *conn, enum io_direction dir,
+			    int (*io)(int fd, struct io_plan_arg *arg),
+			    struct io_plan *(*next)(struct io_conn *, void *),
+			    void *next_arg);
+#endif /* CCAN_IO_PLAN_H */

ccan/ccan/io/poll.c

@@ -0,0 +1,318 @@
+/* Licensed under LGPLv2.1+ - see LICENSE file for details */
+#include "io.h"
+#include "backend.h"
+#include <assert.h>
+#include <poll.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <limits.h>
+#include <errno.h>
+#include <ccan/time/time.h>
+#include <ccan/timer/timer.h>
+
+static size_t num_fds = 0, max_fds = 0, num_waiting = 0;
+static struct pollfd *pollfds = NULL;
+static struct fd **fds = NULL;
+static LIST_HEAD(closing);
+static LIST_HEAD(always);
+static struct timeabs (*nowfn)(void) = time_now;
+
+struct timeabs (*io_time_override(struct timeabs (*now)(void)))(void)
+{
+	struct timeabs (*old)(void) = nowfn;
+	nowfn = now;
+	return old;
+}
+
+static bool add_fd(struct fd *fd, short events)
+{
+	if (!max_fds) {
+		assert(num_fds == 0);
+		pollfds = tal_arr(NULL, struct pollfd, 8);
+		if (!pollfds)
+			return false;
+		fds = tal_arr(pollfds, struct fd *, 8);
+		if (!fds)
+			return false;
+		max_fds = 8;
+	}
+
+	if (num_fds + 1 > max_fds) {
+		size_t num = max_fds * 2;
+
+		if (!tal_resize(&pollfds, num))
+			return false;
+		if (!tal_resize(&fds, num))
+			return false;
+		max_fds = num;
+	}
+
+	pollfds[num_fds].events = events;
+	/* In case it's idle. */
+	if (!events)
+		pollfds[num_fds].fd = -fd->fd;
+	else
+		pollfds[num_fds].fd = fd->fd;
+	pollfds[num_fds].revents = 0; /* In case we're iterating now */
+	fds[num_fds] = fd;
+	fd->backend_info = num_fds;
+	num_fds++;
+	if (events)
+		num_waiting++;
+
+	return true;
+}
+
+static void del_fd(struct fd *fd)
+{
+	size_t n = fd->backend_info;
+
+	assert(n != -1);
+	assert(n < num_fds);
+	if (pollfds[n].events)
+		num_waiting--;
+	if (n != num_fds - 1) {
+		/* Move last one over us. */
+		pollfds[n] = pollfds[num_fds-1];
+		fds[n] = fds[num_fds-1];
+		assert(fds[n]->backend_info == num_fds-1);
+		fds[n]->backend_info = n;
+	} else if (num_fds == 1) {
+		/* Free everything when no more fds. */
+		pollfds = tal_free(pollfds);
+		fds = NULL;
+		max_fds = 0;
+	}
+	num_fds--;
+	fd->backend_info = -1;
+
+	/* Closing a local socket doesn't wake poll() because other end
+	 * has them open.  See 2.6.  When should I use shutdown()?
+	 * in http://www.faqs.org/faqs/unix-faq/socket/ */
+	shutdown(fd->fd, SHUT_RDWR);
+
+	close(fd->fd);
+}
+
+bool add_listener(struct io_listener *l)
+{
+	if (!add_fd(&l->fd, POLLIN))
+		return false;
+	return true;
+}
+
+void remove_from_always(struct io_conn *conn)
+{
+	list_del_init(&conn->always);
+}
+
+void backend_new_closing(struct io_conn *conn)
+{
+	/* In case it's on always list, remove it. */
+	list_del_init(&conn->always);
+	list_add_tail(&closing, &conn->closing);
+}
+
+void backend_new_always(struct io_conn *conn)
+{
+	/* In case it's already in always list. */
+	list_del(&conn->always);
+	list_add_tail(&always, &conn->always);
+}
+
+void backend_new_plan(struct io_conn *conn)
+{
+	struct pollfd *pfd = &pollfds[conn->fd.backend_info];
+
+	if (pfd->events)
+		num_waiting--;
+
+	pfd->events = 0;
+	if (conn->plan[IO_IN].status == IO_POLLING)
+		pfd->events |= POLLIN;
+	if (conn->plan[IO_OUT].status == IO_POLLING)
+		pfd->events |= POLLOUT;
+
+	if (pfd->events) {
+		num_waiting++;
+		pfd->fd = conn->fd.fd;
+	} else {
+		pfd->fd = -conn->fd.fd;
+	}
+}
+
+void backend_wake(const void *wait)
+{
+	unsigned int i;
+
+	for (i = 0; i < num_fds; i++) {
+		struct io_conn *c;
+
+		/* Ignore listeners */
+		if (fds[i]->listener)
+			continue;
+
+		c = (void *)fds[i];
+		if (c->plan[IO_IN].status == IO_WAITING
+		    && c->plan[IO_IN].arg.u1.const_vp == wait)
+			io_do_wakeup(c, IO_IN);
+
+		if (c->plan[IO_OUT].status == IO_WAITING
+		    && c->plan[IO_OUT].arg.u1.const_vp == wait)
+			io_do_wakeup(c, IO_OUT);
+	}
+}
+
+bool add_conn(struct io_conn *c)
+{
+	return add_fd(&c->fd, 0);
+}
+
+static void del_conn(struct io_conn *conn)
+{
+	del_fd(&conn->fd);
+	if (conn->finish) {
+		/* Saved by io_close */
+		errno = conn->plan[IO_IN].arg.u1.s;
+		conn->finish(conn, conn->finish_arg);
+	}
+	tal_free(conn);
+}
+
+void del_listener(struct io_listener *l)
+{
+	del_fd(&l->fd);
+}
+
+static void accept_conn(struct io_listener *l)
+{
+	int fd = accept(l->fd.fd, NULL, NULL);
+
+	/* FIXME: What to do here? */
+	if (fd < 0)
+		return;
+
+	io_new_conn(l->ctx, fd, l->init, l->arg);
+}
+
+/* It's OK to miss some, as long as we make progress. */
+static bool close_conns(void)
+{
+	bool ret = false;
+	struct io_conn *conn;
+
+	while ((conn = list_pop(&closing, struct io_conn, closing)) != NULL) {
+		assert(conn->plan[IO_IN].status == IO_CLOSING);
+		assert(conn->plan[IO_OUT].status == IO_CLOSING);
+
+		del_conn(conn);
+		ret = true;
+	}
+	return ret;
+}
+
+static bool handle_always(void)
+{
+	bool ret = false;
+	struct io_conn *conn;
+
+	while ((conn = list_pop(&always, struct io_conn, always)) != NULL) {
+		assert(conn->plan[IO_IN].status == IO_ALWAYS
+		       || conn->plan[IO_OUT].status == IO_ALWAYS);
+
+		/* Re-initialize, for next time. */
+		list_node_init(&conn->always);
+		io_do_always(conn);
+		ret = true;
+	}
+	return ret;
+}
+
+/* This is the main loop. */
+void *io_loop(struct timers *timers, struct timer **expired)
+{
+	void *ret;
+
+	/* if timers is NULL, expired must be.  If not, not. */
+	assert(!timers == !expired);
+
+	/* Make sure this is NULL if we exit for some other reason. */
+	if (expired)
+		*expired = NULL;
+
+	while (!io_loop_return) {
+		int i, r, ms_timeout = -1;
+
+		if (close_conns()) {
+			/* Could have started/finished more. */
+			continue;
+		}
+
+		if (handle_always()) {
+			/* Could have started/finished more. */
+			continue;
+		}
+
+		/* Everything closed? */
+		if (num_fds == 0)
+			break;
+
+		/* You can't tell them all to go to sleep! */
+		assert(num_waiting);
+
+		if (timers) {
+			struct timeabs now, first;
+
+			now = nowfn();
+
+			/* Call functions for expired timers. */
+			*expired = timers_expire(timers, now);
+			if (*expired)
+				break;
+
+			/* Now figure out how long to wait for the next one. */
+			if (timer_earliest(timers, &first)) {
+				uint64_t next;
+				next = time_to_msec(time_between(first, now));
+				if (next < INT_MAX)
+					ms_timeout = next;
+				else
+					ms_timeout = INT_MAX;
+			}
+		}
+
+		r = poll(pollfds, num_fds, ms_timeout);
+		if (r < 0)
+			break;
+
+		for (i = 0; i < num_fds && !io_loop_return; i++) {
+			struct io_conn *c = (void *)fds[i];
+			int events = pollfds[i].revents;
+
+			if (r == 0)
+				break;
+
+			if (fds[i]->listener) {
+				if (events & POLLIN) {
+					accept_conn((void *)c);
+					r--;
+				}
+			} else if (events & (POLLIN|POLLOUT)) {
+				r--;
+				io_ready(c, events);
+			} else if (events & (POLLHUP|POLLNVAL|POLLERR)) {
+				r--;
+				errno = EBADF;
+				io_close(c);
+			}
+		}
+	}
+
+	close_conns();
+
+	ret = io_loop_return;
+	io_loop_return = NULL;
+
+	return ret;
+}

ccan/ccan/io/test/run-01-start-finish-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-01-start-finish.c"

ccan/ccan/io/test/run-01-start-finish.c

@@ -0,0 +1,105 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64001"
+#else
+#define PORT "65001"
+#endif
+static int expected_fd;
+
+static void finish_ok(struct io_conn *conn, int *state)
+{
+	ok1(*state == 1);
+	ok1(io_conn_fd(conn) == expected_fd);
+	(*state)++;
+	io_break(state + 1);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, int *state)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(*state == 0);
+	(*state)++;
+	expected_fd = io_conn_fd(conn);
+	io_set_finish(conn, finish_ok, state);
+
+	return io_close(conn);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	int state = 0;
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd;
+
+	/* This is how many tests you plan to run */
+	plan_tests(10);
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, &state);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		close(fd);
+		freeaddrinfo(addrinfo);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == &state + 1);
+	ok1(state == 2);
+	io_close_listener(l);
+	ok1(wait(&state));
+	ok1(WIFEXITED(state));
+	ok1(WEXITSTATUS(state) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-02-read-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-02-read.c"

ccan/ccan/io/test/run-02-read.c

@@ -0,0 +1,119 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64002"
+#else
+#define PORT "65002"
+#endif
+
+struct data {
+	int state;
+	char buf[4];
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	io_set_finish(conn, finish_ok, d);
+	return io_read(conn, d->buf, sizeof(d->buf), io_close_cb, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(10);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		int i;
+
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		free(d);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 2);
+	ok1(memcmp(d->buf, "hellothere", sizeof(d->buf)) == 0);
+	free(d);
+	io_close_listener(l);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-03-readpartial-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-03-readpartial.c"

ccan/ccan/io/test/run-03-readpartial.c

@@ -0,0 +1,149 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64003"
+#else
+#define PORT "65003"
+#endif
+
+struct data {
+	int state;
+	size_t bytes;
+	char buf[4];
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	io_set_finish(conn, finish_ok, d);
+
+	return io_read_partial(conn, d->buf, sizeof(d->buf), &d->bytes,
+			       io_close_cb, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+static void write_to_socket(const char *str, const struct addrinfo *addrinfo)
+{
+	int fd, i;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		exit(1);
+	if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+		exit(2);
+	signal(SIGPIPE, SIG_IGN);
+	for (i = 0; i < strlen(str); i++) {
+		if (write(fd, str + i, 1) != 1)
+			break;
+	}
+	close(fd);
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(22);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		write_to_socket("hellothere", addrinfo);
+		freeaddrinfo(addrinfo);
+		free(d);
+		exit(0);
+	}
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 2);
+	ok1(d->bytes > 0);
+	ok1(d->bytes <= sizeof(d->buf));
+	ok1(memcmp(d->buf, "hellothere", d->bytes) == 0);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		write_to_socket("hi", addrinfo);
+		freeaddrinfo(addrinfo);
+		free(d);
+		exit(0);
+	}
+	d->state = 0;
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 2);
+	ok1(d->bytes > 0);
+	ok1(d->bytes <= strlen("hi"));
+	ok1(memcmp(d->buf, "hi", d->bytes) == 0);
+
+	freeaddrinfo(addrinfo);
+	free(d);
+	io_close_listener(l);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-04-writepartial-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-04-writepartial.c"

ccan/ccan/io/test/run-04-writepartial.c

@@ -0,0 +1,133 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64004"
+#else
+#define PORT "65004"
+#endif
+
+struct data {
+	int state;
+	size_t bytes;
+	char *buf;
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+	io_set_finish(conn, finish_ok, d);
+
+	return io_write_partial(conn, d->buf, d->bytes, &d->bytes,
+				io_close_cb, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+static void read_from_socket(const char *str, const struct addrinfo *addrinfo)
+{
+	int fd;
+	char buf[100];
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		exit(1);
+	if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+		exit(2);
+	if (read(fd, buf, strlen(str)) != strlen(str))
+		exit(3);
+	if (memcmp(buf, str, strlen(str)) != 0)
+		exit(4);
+	close(fd);
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(11);
+	d->state = 0;
+	d->bytes = 1024*1024;
+	d->buf = malloc(d->bytes);
+	memset(d->buf, 'a', d->bytes);
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		read_from_socket("aaaaaa", addrinfo);
+		freeaddrinfo(addrinfo);
+		free(d->buf);
+		free(d);
+		exit(0);
+	}
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 2);
+	ok1(d->bytes > 0);
+	ok1(d->bytes <= 1024*1024);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	freeaddrinfo(addrinfo);
+	free(d->buf);
+	free(d);
+	io_close_listener(l);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-05-write-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-05-write.c"

ccan/ccan/io/test/run-05-write.c

@@ -0,0 +1,132 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64005"
+#else
+#define PORT "65005"
+#endif
+
+struct data {
+	int state;
+	size_t bytes;
+	char *buf;
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+	io_set_finish(conn, finish_ok, d);
+	return io_write(conn, d->buf, d->bytes, io_close_cb, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+static void read_from_socket(size_t bytes, const struct addrinfo *addrinfo)
+{
+	int fd, done, r;
+	char buf[100];
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		exit(1);
+	if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+		exit(2);
+
+	for (done = 0; done < bytes; done += r) {
+		r = read(fd, buf, sizeof(buf));
+		if (r < 0)
+			exit(3);
+		done += r;
+	}
+	close(fd);
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(9);
+	d->state = 0;
+	d->bytes = 1024*1024;
+	d->buf = malloc(d->bytes);
+	memset(d->buf, 'a', d->bytes);
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		read_from_socket(d->bytes, addrinfo);
+		freeaddrinfo(addrinfo);
+		free(d->buf);
+		free(d);
+		exit(0);
+	}
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 2);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	freeaddrinfo(addrinfo);
+	free(d->buf);
+	free(d);
+	io_close_listener(l);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-06-idle.c

@@ -0,0 +1,160 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64006"
+#else
+#define PORT "65006"
+#endif
+
+static struct io_conn *idler;
+
+struct data {
+	int state;
+	char buf[4];
+};
+
+static struct io_plan *read_done(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 2 || d->state == 3);
+	d->state++;
+	return io_close(conn);
+}
+
+static void finish_waker(struct io_conn *conn, struct data *d)
+{
+	io_wake(d);
+	ok1(d->state == 1);
+	d->state++;
+}
+
+static void finish_idle(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 3);
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *never(struct io_conn *conn, void *arg)
+{
+	abort();
+}
+
+static struct io_plan *read_buf(struct io_conn *conn, struct data *d)
+{
+	return io_read(conn, d->buf, sizeof(d->buf), read_done, d);
+}
+
+static struct io_plan *init_waker(struct io_conn *conn, void *unused)
+{
+	/* This is /dev/null, so will never succeed. */
+	return io_read(conn, unused, 1, never, NULL);
+}
+
+static struct io_plan *init_idle(struct io_conn *conn, struct data *d)
+{
+	int fd2;
+
+	ok1(d->state == 0);
+	d->state++;
+	idler = conn;
+	io_set_finish(conn, finish_idle, d);
+
+	/* This will wake us up, as read will fail. */
+	fd2 = open("/dev/null", O_RDONLY);
+	ok1(fd2 >= 0);
+	io_set_finish(io_new_conn(NULL, fd2, init_waker, d), finish_waker, d);
+
+	return io_wait(conn, d, read_buf, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(13);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_idle, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		int i;
+
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		free(d);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 4);
+	ok1(memcmp(d->buf, "hellothere", sizeof(d->buf)) == 0);
+	free(d);
+	io_close_listener(l);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-07-break-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-07-break.c"

ccan/ccan/io/test/run-07-break.c

@@ -0,0 +1,130 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64007"
+#else
+#define PORT "65007"
+#endif
+
+struct data {
+	int state;
+	char buf[4];
+};
+
+static struct io_plan *read_done(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	return io_close(conn);
+}
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 2);
+	d->state++;
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	io_set_finish(conn, finish_ok, d);
+
+	io_break(d);
+	return io_read(conn, d->buf, sizeof(d->buf), read_done, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(13);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		int i;
+
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		free(d);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 1);
+	io_close_listener(l);
+
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(d->state == 3);
+	ok1(memcmp(d->buf, "hellothere", sizeof(d->buf)) == 0);
+	free(d);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-08-hangup-on-idle.c

@@ -0,0 +1,60 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+static int fds2[2];
+
+static struct io_plan *read_in(struct io_conn *conn, char *buf)
+{
+	return io_read(conn, buf, 16, io_close_cb, NULL);
+}
+
+static struct io_plan *setup_waiter(struct io_conn *conn, char *buf)
+{
+	return io_wait(conn, buf, read_in, buf);
+}
+
+static struct io_plan *wake_and_close(struct io_conn *conn, char *buf)
+{
+	io_wake(buf);
+	return io_close(conn);
+}
+
+static struct io_plan *setup_waker(struct io_conn *conn, char *buf)
+{
+	return io_read(conn, buf, 1, wake_and_close, buf);
+}
+
+int main(void)
+{
+	int fds[2];
+	char buf[16];
+
+	plan_tests(4);
+
+	ok1(pipe(fds) == 0);
+
+	io_new_conn(NULL, fds[0], setup_waiter, buf);
+	ok1(pipe(fds2) == 0);
+	io_new_conn(NULL, fds2[0], setup_waker, buf);
+
+	if (fork() == 0) {
+		write(fds[1], "hello there world", 16);
+		close(fds[1]);
+
+		/* Now wake it. */
+		sleep(1);
+		write(fds2[1], "", 1);
+		exit(0);
+	}
+
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(memcmp(buf, "hello there world", 16) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-08-read-after-hangup.c

@@ -0,0 +1,49 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <signal.h>
+
+static char inbuf[8];
+
+static struct io_plan *wake_it(struct io_conn *conn, struct io_conn *reader)
+{
+	io_wake(inbuf);
+	return io_close(conn);
+}
+
+static struct io_plan *read_buf(struct io_conn *conn, void *unused)
+{
+	return io_read(conn, inbuf, 8, io_close_cb, NULL);
+}
+
+static struct io_plan *init_writer(struct io_conn *conn, struct io_conn *wakeme)
+{
+	return io_write(conn, "EASYTEST", 8, wake_it, wakeme);
+}
+
+static struct io_plan *init_waiter(struct io_conn *conn, void *unused)
+{
+	return io_wait(conn, inbuf, read_buf, NULL);
+}
+
+int main(void)
+{
+	int fds[2];
+	struct io_conn *conn;
+
+	plan_tests(3);
+
+	ok1(pipe(fds) == 0);
+	conn = io_new_conn(NULL, fds[0], init_waiter, NULL);
+	io_new_conn(conn, fds[1], init_writer, conn);
+
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(memcmp(inbuf, "EASYTEST", sizeof(inbuf)) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-09-connect-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-09-connect.c"

ccan/ccan/io/test/run-09-connect.c

@@ -0,0 +1,117 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64009"
+#else
+#define PORT "65009"
+#endif
+
+static struct io_listener *l;
+static struct data *d2;
+
+struct data {
+	int state;
+	char buf[10];
+};
+
+static struct io_plan *closer(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	return io_close(conn);
+}
+
+static struct io_plan *connected(struct io_conn *conn, struct data *d2)
+{
+	ok1(d2->state == 0);
+	d2->state++;
+	return io_read(conn, d2->buf, sizeof(d2->buf), closer, d2);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+	io_close_listener(l);
+
+	return io_write(conn, d->buf, sizeof(d->buf), closer, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+static struct io_plan *setup_connect(struct io_conn *conn,
+				     struct addrinfo *addrinfo)
+{
+	d2 = malloc(sizeof(*d2));
+	d2->state = 0;
+	return io_connect(conn, addrinfo, connected, d2);
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	int fd;
+
+	/* This is how many tests you plan to run */
+	plan_tests(8);
+	d->state = 0;
+	memset(d->buf, 'a', sizeof(d->buf));
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	ok1(io_new_conn(NULL, fd, setup_connect, addrinfo));
+
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(d->state == 2);
+	ok1(d2->state == 2);
+
+	freeaddrinfo(addrinfo);
+	free(d);
+	free(d2);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-10-many.c

@@ -0,0 +1,115 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#define NUM 100
+#define NUM_ITERS 1000
+
+struct buffer {
+	int iters;
+	struct io_conn *reader, *writer;
+	char buf[32];
+};
+
+static struct io_plan *poke_reader(struct io_conn *conn, struct buffer *buf);
+static struct io_plan *poke_writer(struct io_conn *conn, struct buffer *buf);
+
+static struct io_plan *read_buf(struct io_conn *conn, struct buffer *buf)
+{
+	return io_read(conn, &buf->buf, sizeof(buf->buf), poke_writer, buf);
+}
+
+static struct io_plan *poke_writer(struct io_conn *conn, struct buffer *buf)
+{
+	assert(conn == buf->reader);
+
+	if (buf->iters == NUM_ITERS)
+		return io_close(conn);
+
+	/* You write. */
+	io_wake(&buf->writer);
+
+	/* I'll wait until you wake me. */
+	return io_wait(conn, &buf->reader, read_buf, buf);
+}
+
+static struct io_plan *write_buf(struct io_conn *conn, struct buffer *buf)
+{
+	return io_write(conn, &buf->buf, sizeof(buf->buf), poke_reader, buf);
+}
+
+static struct io_plan *poke_reader(struct io_conn *conn, struct buffer *buf)
+{
+	assert(conn == buf->writer);
+	/* You read. */
+	io_wake(&buf->reader);
+
+	if (++buf->iters == NUM_ITERS)
+		return io_close(conn);
+
+	/* I'll wait until you tell me to write. */
+	return io_wait(conn, &buf->writer, write_buf, buf);
+}
+
+static struct io_plan *setup_reader(struct io_conn *conn, struct buffer *buf)
+{
+	return io_wait(conn, &buf->reader, read_buf, buf);
+}
+
+static struct buffer buf[NUM];
+
+int main(void)
+{
+	unsigned int i;
+	int fds[2], last_read, last_write;
+
+	plan_tests(5 + NUM);
+
+	ok1(pipe(fds) == 0);
+	last_read = fds[0];
+	last_write = fds[1];
+
+	for (i = 1; i < NUM; i++) {
+		if (pipe(fds) < 0)
+			break;
+		memset(buf[i].buf, i, sizeof(buf[i].buf));
+		sprintf(buf[i].buf, "%i-%i", i, i);
+
+		/* Wait for writer to tell us to read. */
+		buf[i].reader = io_new_conn(NULL, last_read,
+					    setup_reader, &buf[i]);
+		if (!buf[i].reader)
+			break;
+		buf[i].writer = io_new_conn(NULL, fds[1], write_buf, &buf[i]);
+		if (!buf[i].writer)
+			break;
+		last_read = fds[0];
+	}
+	if (!ok1(i == NUM))
+		exit(exit_status());
+
+	/* Last one completes the cirle. */
+	i = 0;
+	sprintf(buf[i].buf, "%i-%i", i, i);
+	buf[i].reader = io_new_conn(NULL, last_read, setup_reader, &buf[i]);
+	ok1(buf[i].reader);
+	buf[i].writer = io_new_conn(NULL, last_write, write_buf, &buf[i]);
+	ok1(buf[i].writer);
+
+	/* They should eventually exit */
+	ok1(io_loop(NULL, NULL) == NULL);
+
+	for (i = 0; i < NUM; i++) {
+		char b[sizeof(buf[0].buf)];
+		memset(b, i, sizeof(b));
+		sprintf(b, "%i-%i", i, i);
+		ok1(memcmp(b, buf[(i + NUM_ITERS) % NUM].buf, sizeof(b)) == 0);
+	}
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-12-bidir-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-12-bidir.c"

ccan/ccan/io/test/run-12-bidir.c

@@ -0,0 +1,144 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64012"
+#else
+#define PORT "65012"
+#endif
+
+struct data {
+	struct io_listener *l;
+	int state;
+	char buf[4];
+	char wbuf[32];
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+}
+
+static struct io_plan *r_done(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	if (d->state == 3)
+		return io_close(conn);
+	return io_wait(conn, NULL, io_never, NULL);
+}
+
+static struct io_plan *w_done(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	if (d->state == 3)
+		return io_close(conn);
+	return io_out_wait(conn, NULL, io_never, NULL);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	io_close_listener(d->l);
+
+	memset(d->wbuf, 7, sizeof(d->wbuf));
+	io_set_finish(conn, finish_ok, d);
+
+	return io_duplex(conn,
+			 io_read(conn, d->buf, sizeof(d->buf), r_done, d),
+			 io_write(conn, d->wbuf, sizeof(d->wbuf), w_done, d));
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(9);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	d->l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(d->l);
+	fflush(stdout);
+	if (!fork()) {
+		int i;
+		char buf[32];
+
+		io_close_listener(d->l);
+		free(d);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		for (i = 0; i < 32; i++) {
+			if (read(fd, buf+i, 1) != 1)
+				break;
+		}
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(d->state == 4);
+	ok1(memcmp(d->buf, "hellothere", sizeof(d->buf)) == 0);
+	free(d);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-13-all-idle.c

@@ -0,0 +1,36 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <signal.h>
+
+static struct io_plan *setup_waiter(struct io_conn *conn, int *status)
+{
+	return io_wait(conn, status, io_close_cb, NULL);
+}
+
+int main(void)
+{
+	int status;
+
+	plan_tests(3);
+
+	if (fork() == 0) {
+		int fds[2];
+
+		ok1(pipe(fds) == 0);
+		io_new_conn(NULL, fds[0], setup_waiter, &status);
+		io_loop(NULL, NULL);
+		exit(1);
+	}
+
+	ok1(wait(&status) != -1);
+	ok1(WIFSIGNALED(status));
+	ok1(WTERMSIG(status) == SIGABRT);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-14-duplex-both-read-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-14-duplex-both-read.c"

ccan/ccan/io/test/run-14-duplex-both-read.c

@@ -0,0 +1,146 @@
+/* Check a bug where we have just completed a read, then set up a duplex
+ * which tries to do a read. */
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64014"
+#else
+#define PORT "65014"
+#endif
+
+struct data {
+	struct io_listener *l;
+	int state;
+	char buf[4];
+	char wbuf[32];
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+}
+
+static struct io_plan *end(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	/* Close on top of halfclose should work. */
+	if (d->state == 4)
+		return io_close(conn);
+	else
+		return io_halfclose(conn);
+}
+
+static struct io_plan *make_duplex(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	/* Have duplex read the rest of the buffer. */
+	return io_duplex(conn,
+			 io_read(conn, d->buf+1, sizeof(d->buf)-1, end, d),
+			 io_write(conn, d->wbuf, sizeof(d->wbuf), end, d));
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	io_close_listener(d->l);
+
+	memset(d->wbuf, 7, sizeof(d->wbuf));
+	io_set_finish(conn, finish_ok, d);
+	return io_read(conn, d->buf, 1, make_duplex, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(9);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	d->l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(d->l);
+	fflush(stdout);
+	if (!fork()) {
+		int i;
+		char buf[32];
+
+		io_close_listener(d->l);
+		free(d);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		for (i = 0; i < 32; i++) {
+			if (read(fd, buf+i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(d->state == 5);
+	ok1(memcmp(d->buf, "hellothere", sizeof(d->buf)) == 0);
+	free(d);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-15-timeout.c

@@ -0,0 +1,190 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <ccan/time/time.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64015"
+#else
+#define PORT "65015"
+#endif
+
+struct data {
+	struct timers timers;
+	int state;
+	struct io_conn *conn;
+	struct timer timer;
+	int timeout_usec;
+	char buf[4];
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *no_timeout(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	return io_close(conn);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	d->conn = conn;
+	io_set_finish(conn, finish_ok, d);
+
+	timer_add(&d->timers, &d->timer,
+		  timeabs_add(time_now(), time_from_usec(d->timeout_usec)));
+
+	return io_read(conn, d->buf, sizeof(d->buf), no_timeout, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	struct timer *expired;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(21);
+	d->state = 0;
+	d->timeout_usec = 100000;
+	timers_init(&d->timers, time_now());
+	timer_init(&d->timer);
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+
+	if (!fork()) {
+		int i;
+
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		usleep(500000);
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		timers_cleanup(&d->timers);
+		free(d);
+		exit(i);
+	}
+	ok1(io_loop(&d->timers, &expired) == NULL);
+
+	/* One element, d->timer. */
+	ok1(expired == &d->timer);
+	ok1(!timers_expire(&d->timers, time_now()));
+	ok1(d->state == 1);
+
+	io_close(d->conn);
+
+	/* Finished will be called, d will be returned */
+	ok1(io_loop(&d->timers, &expired) == d);
+	ok1(expired == NULL);
+	ok1(d->state == 2);
+
+	/* It should have died. */
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) < sizeof(d->buf));
+
+	/* This one shouldn't time out. */
+	d->state = 0;
+	d->timeout_usec = 500000;
+	fflush(stdout);
+
+	if (!fork()) {
+		int i;
+
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		usleep(100000);
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		timers_cleanup(&d->timers);
+		free(d);
+		exit(i);
+	}
+	ok1(io_loop(&d->timers, &expired) == d);
+	ok1(d->state == 3);
+	ok1(expired == NULL);
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) >= sizeof(d->buf));
+
+	io_close_listener(l);
+	freeaddrinfo(addrinfo);
+	timers_cleanup(&d->timers);
+	free(d);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-16-duplex-test-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-16-duplex-test.c"

ccan/ccan/io/test/run-16-duplex-test.c

@@ -0,0 +1,137 @@
+/* Tests when the last connection is a duplex, and poll.c moves it over
+ * deleted fd. */
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64016"
+#else
+#define PORT "65016"
+#endif
+
+struct data {
+	struct io_listener *l;
+	int state;
+	char buf[4];
+	char wbuf[32];
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+}
+
+static struct io_plan *io_done(struct io_conn *conn, struct data *d)
+{
+	d->state++;
+	return io_halfclose(conn);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+
+	memset(d->wbuf, 7, sizeof(d->wbuf));
+
+	io_set_finish(conn, finish_ok, d);
+
+	io_close_listener(d->l);
+
+	return io_duplex(conn,
+			 io_read(conn, d->buf, sizeof(d->buf), io_done, d),
+			 io_write(conn, d->wbuf, sizeof(d->wbuf), io_done, d));
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(9);
+	d->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	d->l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(d->l);
+	fflush(stdout);
+	if (!fork()) {
+		int i;
+		char buf[32];
+
+		io_close_listener(d->l);
+		free(d);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+		for (i = 0; i < 32; i++) {
+			if (read(fd, buf+i, 1) != 1)
+				break;
+		}
+		for (i = 0; i < strlen("hellothere"); i++) {
+			if (write(fd, "hellothere" + i, 1) != 1)
+				break;
+		}
+		close(fd);
+		freeaddrinfo(addrinfo);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == NULL);
+	ok1(d->state == 4);
+	ok1(memcmp(d->buf, "hellothere", sizeof(d->buf)) == 0);
+	free(d);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-17-homemade-io-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-17-homemade-io.c"

ccan/ccan/io/test/run-17-homemade-io.c

@@ -0,0 +1,185 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64017"
+#else
+#define PORT "65017"
+#endif
+
+struct packet {
+	int state;
+	size_t len;
+	void *contents;
+};
+
+static void finish_ok(struct io_conn *conn, struct packet *pkt)
+{
+	ok1(pkt->state == 3);
+	pkt->state++;
+	io_break(pkt);
+}
+
+static int do_read_packet(int fd, struct io_plan_arg *arg)
+{
+	struct packet *pkt = arg->u1.vp;
+	char *dest;
+	ssize_t ret;
+	size_t off, totlen;
+
+	/* Reading len? */
+	if (arg->u2.s < sizeof(size_t)) {
+		ok1(pkt->state == 1);
+		pkt->state++;
+		dest = (char *)&pkt->len;
+		off = arg->u2.s;
+		totlen = sizeof(pkt->len);
+	} else {
+		ok1(pkt->state == 2);
+		pkt->state++;
+		if (pkt->len == 0)
+			return 1;
+		if (!pkt->contents && !(pkt->contents = malloc(pkt->len)))
+			goto fail;
+		else {
+			dest = pkt->contents;
+			off = arg->u2.s - sizeof(pkt->len);
+			totlen = pkt->len;
+		}
+	}
+
+	ret = read(fd, dest + off, totlen - off);
+	if (ret <= 0)
+		goto fail;
+
+	arg->u2.s += ret;
+
+	/* Finished? */
+	return arg->u2.s >= sizeof(pkt->len)
+		&& arg->u2.s == pkt->len + sizeof(pkt->len);
+
+fail:
+	free(pkt->contents);
+	return -1;
+}
+
+static struct io_plan *io_read_packet(struct io_conn *conn,
+				      struct packet *pkt,
+				      struct io_plan *(*cb)(struct io_conn *,
+							    void *),
+				      void *cb_arg)
+{
+	struct io_plan_arg *arg = io_plan_arg(conn, IO_IN);
+
+	pkt->contents = NULL;
+	arg->u1.vp = pkt;
+	arg->u2.s = 0;
+
+	return io_set_plan(conn, IO_IN, do_read_packet, cb, cb_arg);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct packet *pkt)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(pkt->state == 0);
+	pkt->state++;
+
+	io_set_finish(conn, finish_ok, pkt);
+	return io_read_packet(conn, pkt, io_close_cb, pkt);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	struct packet *pkt = malloc(sizeof(*pkt));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(13);
+	pkt->state = 0;
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, pkt);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		struct {
+			size_t len;
+			char data[8];
+		} data;
+
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		signal(SIGPIPE, SIG_IGN);
+
+		data.len = sizeof(data.data);
+		memcpy(data.data, "hithere!", sizeof(data.data));
+		if (write(fd, &data, sizeof(data)) != sizeof(data))
+			exit(3);
+
+		close(fd);
+		freeaddrinfo(addrinfo);
+		free(pkt);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == pkt);
+	ok1(pkt->state == 4);
+	ok1(pkt->len == 8);
+	ok1(memcmp(pkt->contents, "hithere!", 8) == 0);
+	free(pkt->contents);
+	free(pkt);
+	io_close_listener(l);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-18-errno-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-18-errno.c"

ccan/ccan/io/test/run-18-errno.c

@@ -0,0 +1,126 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64018"
+#else
+#define PORT "65018"
+#endif
+
+static void finish_100(struct io_conn *conn, int *state)
+{
+	ok1(errno == 100);
+	ok1(*state == 1);
+	(*state)++;
+}
+
+static void finish_EBADF(struct io_conn *conn, int *state)
+{
+	ok1(errno == EBADF);
+	ok1(*state == 3);
+	(*state)++;
+	io_break(state + 1);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, int *state)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	if (*state == 0) {
+		(*state)++;
+		errno = 100;
+		io_set_finish(conn, finish_100, state);
+		return io_close(conn);
+	} else {
+		ok1(*state == 2);
+		(*state)++;
+		close(io_conn_fd(conn));
+		errno = 0;
+		io_set_finish(conn, finish_EBADF, state);
+
+		return io_read(conn, state, 1, io_close_cb, NULL);
+	}
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+int main(void)
+{
+	int state = 0;
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd;
+
+	/* This is how many tests you plan to run */
+	plan_tests(12);
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, &state);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(1);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(2);
+		close(fd);
+		fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+			    addrinfo->ai_protocol);
+		if (fd < 0)
+			exit(3);
+		if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+			exit(4);
+		close(fd);
+		freeaddrinfo(addrinfo);
+		exit(0);
+	}
+	freeaddrinfo(addrinfo);
+	ok1(io_loop(NULL, NULL) == &state + 1);
+	ok1(state == 4);
+	io_close_listener(l);
+	ok1(wait(&state));
+	ok1(WIFEXITED(state));
+	ok1(WEXITSTATUS(state) == 0);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-19-always-debug.c

@@ -0,0 +1,2 @@
+#define DEBUG_CONN
+#include "run-19-always.c"

ccan/ccan/io/test/run-19-always.c

@@ -0,0 +1,139 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#ifdef DEBUG_CONN
+#define PORT "64019"
+#else
+#define PORT "65019"
+#endif
+
+struct data {
+	int state;
+	size_t bytes;
+	char *buf;
+};
+
+static void finish_ok(struct io_conn *conn, struct data *d)
+{
+	ok1(d->state == 1);
+	d->state++;
+	io_break(d);
+}
+
+static struct io_plan *write_buf(struct io_conn *conn, struct data *d)
+{
+	return io_write(conn, d->buf, d->bytes, io_close_cb, d);
+}
+
+static struct io_plan *init_conn(struct io_conn *conn, struct data *d)
+{
+#ifdef DEBUG_CONN
+	io_set_debug(conn, true);
+#endif
+	ok1(d->state == 0);
+	d->state++;
+	io_set_finish(conn, finish_ok, d);
+
+	/* Empty read should run immediately... */
+	return io_read(conn, NULL, 0, write_buf, d);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+static void read_from_socket(size_t bytes, const struct addrinfo *addrinfo)
+{
+	int fd, done, r;
+	char buf[100];
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		exit(1);
+	if (connect(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0)
+		exit(2);
+
+	for (done = 0; done < bytes; done += r) {
+		r = read(fd, buf, sizeof(buf));
+		if (r < 0)
+			exit(3);
+		done += r;
+	}
+	close(fd);
+}
+
+int main(void)
+{
+	struct data *d = malloc(sizeof(*d));
+	struct addrinfo *addrinfo;
+	struct io_listener *l;
+	int fd, status;
+
+	/* This is how many tests you plan to run */
+	plan_tests(9);
+	d->state = 0;
+	d->bytes = 1024*1024;
+	d->buf = malloc(d->bytes);
+	memset(d->buf, 'a', d->bytes);
+	fd = make_listen_fd(PORT, &addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, d);
+	ok1(l);
+	fflush(stdout);
+	if (!fork()) {
+		io_close_listener(l);
+		read_from_socket(d->bytes, addrinfo);
+		freeaddrinfo(addrinfo);
+		free(d->buf);
+		free(d);
+		exit(0);
+	}
+	ok1(io_loop(NULL, NULL) == d);
+	ok1(d->state == 2);
+
+	ok1(wait(&status));
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	freeaddrinfo(addrinfo);
+	free(d->buf);
+	free(d);
+	io_close_listener(l);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/io/test/run-20-io_time_override.c

@@ -0,0 +1,92 @@
+#include <ccan/io/io.h>
+/* Include the C files directly. */
+#include <ccan/io/poll.c>
+#include <ccan/io/io.c>
+#include <ccan/tap/tap.h>
+#include <sys/wait.h>
+#include <stdio.h>
+
+#define PORT "65020"
+
+static struct io_plan *init_conn(struct io_conn *conn, void *unused)
+{
+	return io_close(conn);
+}
+
+static int make_listen_fd(const char *port, struct addrinfo **info)
+{
+	int fd, on = 1;
+	struct addrinfo *addrinfo, hints;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+	hints.ai_protocol = 0;
+
+	if (getaddrinfo(NULL, port, &hints, &addrinfo) != 0)
+		return -1;
+
+	fd = socket(addrinfo->ai_family, addrinfo->ai_socktype,
+		    addrinfo->ai_protocol);
+	if (fd < 0)
+		return -1;
+
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+	if (bind(fd, addrinfo->ai_addr, addrinfo->ai_addrlen) != 0) {
+		close(fd);
+		return -1;
+	}
+	if (listen(fd, 1) != 0) {
+		close(fd);
+		return -1;
+	}
+	*info = addrinfo;
+	return fd;
+}
+
+static struct timeabs fake_time;
+
+static struct timeabs get_fake_time(void)
+{
+	return fake_time;
+}
+
+int main(void)
+{
+	struct io_listener *l;
+	int fd;
+	struct timers timers;
+	struct timer timer, *expired;
+	struct addrinfo *addrinfo;
+
+	/* This is how many tests you plan to run */
+	plan_tests(7);
+
+	fake_time = time_now();
+
+	timers_init(&timers, fake_time);
+	timer_init(&timer);
+	timer_add(&timers, &timer,
+		  timeabs_add(fake_time, time_from_sec(1000)));
+
+	fd = make_listen_fd(PORT, &addrinfo);
+	freeaddrinfo(addrinfo);
+	ok1(fd >= 0);
+	l = io_new_listener(NULL, fd, init_conn, NULL);
+	ok1(l);
+
+	fake_time.ts.tv_sec += 1000;
+	ok1(io_time_override(get_fake_time) == time_now);
+	ok1(io_loop(&timers, &expired) == NULL);
+
+	ok1(expired == &timer);
+	ok1(!timers_expire(&timers, fake_time));
+	ok1(io_time_override(time_now) == get_fake_time);
+	io_close_listener(l);
+
+	timers_cleanup(&timers);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/isaac/_info

@@ -2,8 +2,9 @@
  * isaac - A fast, high-quality pseudo-random number generator.
  *
  * ISAAC (Indirect, Shift, Accumulate, Add, and Count) is the most advanced of
- *  a series of pseudo-random number generators designed by Robert J. Jenkins
- *  Jr. in 1996: http://www.burtleburtle.net/bob/rand/isaac.html
+ * a series of pseudo-random number generators designed by Robert J. Jenkins
+ * Jr. in 1996: http://www.burtleburtle.net/bob/rand/isaac.html
+ *
  * To quote:
  *   No efficient method is known for deducing their internal states.
  *   ISAAC requires an amortized 18.75 instructions to produce a 32-bit value.
@@ -11,35 +12,27 @@
  *   The expected cycle length is 2**8295 values.
  *   ...
  *   ISAAC-64 generates a different sequence than ISAAC, but it uses the same
- *    principles.
+ *   principles.
  *   It uses 64-bit arithmetic.
  *   It generates a 64-bit result every 19 instructions.
  *   All cycles are at least 2**72 values, and the average cycle length is
  *    2**16583.
+ *
  * An additional, important comment from Bob Jenkins in 2006:
+ *
  *   Seeding a random number generator is essentially the same problem as
- *    encrypting the seed with a block cipher.
+ *   encrypting the seed with a block cipher.
  *   ISAAC should be initialized with the encryption of the seed by some
- *    secure cipher.
+ *   secure cipher.
  *   I've provided a seeding routine in my implementations, which nobody has
- *    broken so far, but I have less faith in that initialization routine than
- *    I have in ISAAC.
+ *   broken so far, but I have less faith in that initialization routine than
+ *   I have in ISAAC.
  *
  * A number of attacks on ISAAC have been published.
+ *
  * [Pudo01] can recover the entire internal state and has expected running time
- *  less than the square root of the number of states, or 2**4121 (4.67E+1240).
- * [Auma06] reveals a large set of weak states, consisting of those for which
- *  the first value is repeated one or more times elsewhere in the state
- *  vector.
- * These induce a bias in the output relative to the repeated value.
- * The seed values used as input below are scrambled before being used, so any
- *  duplicates in them do not imply duplicates in the resulting internal state,
- *  however the chances of some duplicate existing elsewhere in a random state
- *  are just over 255/2**32, or merely 1 in 16 million.
- * Such states are, of course, much rarer in ISAAC-64.
- * It is not clear if an attacker can tell from just the output if ISAAC is in
- *  a weak state, or deduce the full internal state in any case except that
- *  where all or almost all of the entries in the state vector are identical.
+ * less than the square root of the number of states, or 2**4121 (4.67E+1240).
+ *
  *   @MISC{Pudo01,
  *     author="Marina Pudovkina",
  *     title="A Known Plaintext Attack on the {ISAAC} Keystream Generator",
@@ -47,6 +40,11 @@
  *     year=2001,
  *     note="\url{http://eprint.iacr.org/2001/049}",
  *   }
+ *
+ * [Auma06] reveals a large set of weak states, consisting of those for which
+ * the first value is repeated one or more times elsewhere in the state
+ * vector.
+ *
  *   @MISC{Auma06,
  *     author="Jean-Philippe Aumasson",
  *     title="On the Pseudo-Random Generator {ISAAC}",
@@ -55,17 +53,32 @@
  *     note="\url{http://eprint.iacr.org/2006/438}",
  *   }
  *
+ * These induce a bias in the output relative to the repeated value.
+ *
+ * The seed values used as input below are scrambled before being used, so any
+ * duplicates in them do not imply duplicates in the resulting internal state,
+ * however the chances of some duplicate existing elsewhere in a random state
+ * are just over 255/2**32, or merely 1 in 16 million.
+ *
+ * Such states are, of course, much rarer in ISAAC-64.
+ *
+ * It is not clear if an attacker can tell from just the output if ISAAC is in
+ * a weak state, or deduce the full internal state in any case except that
+ * where all or almost all of the entries in the state vector are identical.
+ *
  * Even if one does not trust the security of this PRNG (and, without a good
- *  source of entropy to seed it, one should not), ISAAC is an excellent source
- *  of high-quality random numbers for Monte Carlo simulations, etc.
+ * source of entropy to seed it, one should not), ISAAC is an excellent source
+ * of high-quality random numbers for Monte Carlo simulations, etc.
+ *
  * It is the fastest 32-bit generator among all of those that pass the
- *  statistical tests in the recent survey
- *  http://www.iro.umontreal.ca/~simardr/testu01/tu01.html, with the exception
- *  of Marsa-LFIB4, and it is quite competitive on 64-bit archtectures.
+ * statistical tests in the recent survey
+ * http://www.iro.umontreal.ca/~simardr/testu01/tu01.html, with the exception
+ * of Marsa-LFIB4, and it is quite competitive on 64-bit archtectures.
+ *
  * Unlike Marsa-LFIB4 (and all other LFib generators), there are no linear
- *  dependencies between successive values, and unlike many generators found in
- *  libc implementations, there are no small periods in the least significant
- *  bits, or seeds which lead to very small periods in general.
+ * dependencies between successive values, and unlike many generators found in
+ * libc implementations, there are no small periods in the least significant
+ * bits, or seeds which lead to very small periods in general.
  *
  * Example:
  *  #include <stdio.h>

ccan/ccan/mem/bench/Makefile

@@ -0,0 +1,17 @@
+CCANDIR=../../..
+CFLAGS=-Wall -Werror -O3 -I$(CCANDIR)
+#CFLAGS=-Wall -Werror -g -I$(CCANDIR)
+
+all: speed
+
+CCAN_OBJS:=ccan-mem.o ccan-time.o
+
+speed: speed.o $(CCAN_OBJS)
+
+clean:
+	rm -f speed *.o
+
+ccan-time.o: $(CCANDIR)/ccan/time/time.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-mem.o: $(CCANDIR)/ccan/mem/mem.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/ccan/mem/bench/speed.c

@@ -0,0 +1,49 @@
+/* Test speed of memiszero */
+#include <ccan/time/time.h>
+#include <ccan/mem/mem.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+#define MAX_TEST 65536
+
+int main(int argc, char *argv[])
+{
+	size_t n, i, max = argv[1] ? atol(argv[1]) : 100000000, runs;
+	char *arr;
+	size_t total = 0;
+
+	arr = calloc(1, max + MAX_TEST + 1);
+
+	runs = max;
+	/* First test even sizes case. */
+	for (n = 1; n <= MAX_TEST; n *= 2) {
+		struct timeabs start = time_now();
+		struct timerel each;
+
+		for (i = 0; i < runs; i++)
+			total += memeqzero(arr + i, n);
+		each = time_divide(time_between(time_now(), start), runs);
+		assert(each.ts.tv_sec == 0);
+		printf("%zu: %uns\n", n, (unsigned int)each.ts.tv_nsec);
+
+		/* Reduce runs over time, as bigger take longer. */
+		runs = runs * 2 / 3;
+	}
+
+	runs = max;
+	for (n = 1; n <= MAX_TEST; n *= 2) {
+		struct timeabs start = time_now();
+		struct timerel each;
+
+		for (i = 0; i < runs; i++)
+			total += memeqzero(arr + i, n+1);
+		each = time_divide(time_between(time_now(), start), runs);
+		assert(each.ts.tv_sec == 0);
+		printf("%zu: %uns\n", n+1, (unsigned int)each.ts.tv_nsec);
+		runs = runs * 2 / 3;
+	}
+
+	printf("total = %zu\n", total);
+	return 0;
+}

ccan/ccan/mem/mem.c

@@ -88,3 +88,22 @@ void memswap(void *a, void *b, size_t n)
 		n -= m;
 	}
 }
+
+bool memeqzero(const void *data, size_t length)
+{
+	const unsigned char *p = data;
+	size_t len;
+
+	/* Check first 16 bytes manually */
+	for (len = 0; len < 16; len++) {
+		if (!length)
+			return true;
+		if (*p)
+			return false;
+		p++;
+		length--;
+	}
+
+	/* Now we know that's zero, memcmp with self. */
+	return memcmp(data, p, length) == 0;
+}

ccan/ccan/mem/mem.h

@@ -149,6 +149,19 @@ static inline bool memeqstr(const void *data, size_t length, const char *string)
 	return memeq(data, length, string, strlen(string));
 }
 
+/**
+ * memeqzero - Is a byte array all zeroes?
+ * @data: byte array
+ * @length: length of @data in bytes
+ *
+ * Example:
+ *	if (memeqzero(somebytes, bytes_len)) {
+ *		printf("somebytes == 0!\n");
+ *	}
+ */
+PURE_FUNCTION
+bool memeqzero(const void *data, size_t length);
+
 /**
  * memstarts_str - Does this byte array start with a string prefix?
  * @a: byte array

ccan/ccan/mem/test/api.c

@@ -18,7 +18,7 @@ int main(void)
 	char tmp1[SWAPSIZE], tmp2[SWAPSIZE];
 
 	/* This is how many tests you plan to run */
-	plan_tests(62);
+	plan_tests(65);
 
 	ok1(memmem(haystack1, sizeof(haystack1), needle1, 2) == haystack1);
 	ok1(memmem(haystack1, sizeof(haystack1), needle1, 3) == NULL);
@@ -113,6 +113,10 @@ int main(void)
 	ok1(memcmp(tmp1, haystack2, sizeof(haystack2)) == 0);
 	ok1(memcmp(tmp2, haystack1, sizeof(haystack1)) == 0);
 
+	ok1(memeqzero(NULL, 0));
+	ok1(memeqzero(scan2, 3));
+	ok1(!memeqzero(scan2, 4));
+
 	/* This exits depending on whether all tests passed */
 	return exit_status();
 }

ccan/ccan/noerr/noerr.c

@@ -2,6 +2,7 @@
 #include "noerr.h"
 #include <unistd.h>
 #include <errno.h>
+#include <stdlib.h>
 
 int close_noerr(int fd)
 {
@@ -41,3 +42,10 @@ int unlink_noerr(const char *pathname)
 	errno = saved_errno;
 	return ret;
 }
+
+void free_noerr(void *p)
+{
+	int saved_errno = errno;
+	free(p);
+	errno = saved_errno;
+}

ccan/ccan/noerr/noerr.h

@@ -30,4 +30,12 @@ int fclose_noerr(FILE *fp);
  */
 int unlink_noerr(const char *pathname);
 
+/**
+ * free_noerr - free memory without stomping errno.
+ * @p: the pointer to free.
+ *
+ * errno is saved and restored across the call to free: the standard leaves
+ * that undefined.
+ */
+void free_noerr(void *p);
 #endif /* NOERR_H */

ccan/ccan/noerr/test/run.c

@@ -13,7 +13,7 @@ int main(int argc, char *argv[])
 	int fd;
 	FILE *fp;
 
-	plan_tests(15);
+	plan_tests(16);
 	/* Should fail to unlink. */
 	ok1(unlink(name) != 0);
 	ok1(errno == ENOENT);
@@ -59,5 +59,9 @@ int main(int argc, char *argv[])
 	ok1(errno == 100);
 	unlink(name);
 
+	errno = 101;
+	free_noerr(malloc(7));
+	ok1(errno == 101);
+
 	return exit_status();
 }

ccan/ccan/opt/helpers.c

@@ -14,10 +14,9 @@
 /* Upper bound to sprintf this simple type?  Each 3 bits < 1 digit. */
 #define CHAR_SIZE(type) (((sizeof(type)*CHAR_BIT + 2) / 3) + 1)
 
-/* FIXME: asprintf module? */
 static char *arg_bad(const char *fmt, const char *arg)
 {
-	char *str = malloc(strlen(fmt) + strlen(arg));
+	char *str = opt_alloc.alloc(strlen(fmt) + strlen(arg));
 	sprintf(str, fmt, arg);
 	return str;
 }

ccan/ccan/opt/test/run-set_alloc.c

@@ -66,8 +66,9 @@ static void freefn(void *ptr)
 int main(int argc, char *argv[])
 {
 	const char *myname = argv[0];
+	unsigned int val;
 
-	plan_tests(220);
+	plan_tests(222);
 
 	opt_set_alloc(allocfn, reallocfn, freefn);
 
@@ -341,6 +342,12 @@ int main(int argc, char *argv[])
 	ok1(strcmp(argv[4], "-a") == 0);
 	ok1(!argv[5]);
 
+	/* Finally, test the helpers don't use malloc. */
+	reset_options();
+	opt_register_arg("-a", opt_set_uintval, opt_show_uintval, &val, "a");
+	ok1(!parse_args(&argc, &argv, "-a", "notanumber", NULL));
+	ok1(strstr(err_output, ": -a: 'notanumber' is not a number"));
+
 	/* We should have tested each one at least once! */
 	ok1(realloc_count);
 	ok1(alloc_count);

ccan/ccan/order/order.h

@@ -31,6 +31,9 @@ struct _total_order {
 		_ctx ctx;				\
 	} _name
 
+#define total_order_cmp(_order, _a, _b)					\
+	((_order).cb((_a), (_b), (_order).ctx))
+
 #define _DECL_ONAME(_oname, _itype)					\
 	extern int _order_##_oname(const void *, const void *, void *);	\
 	extern int order_##_oname(const _itype *, const _itype *, void *); \

ccan/ccan/order/test/fancy_cmp.h

@@ -8,7 +8,7 @@ struct cmp_info {
 
 struct item {
 	unsigned value;
-	char *str;
+	const char *str;
 };
 
 static inline int fancy_cmp(const struct item *a, const struct item *b,

ccan/ccan/order/test/run-fancy.c

@@ -0,0 +1,68 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <ccan/order/order.h>
+
+#include <ccan/tap/tap.h>
+
+#include "fancy_cmp.h"
+
+int main(int argc, char *argv[])
+{
+	struct item item1 = {
+		.value = 0,
+		.str = "aaa",
+	};
+	struct item item2 = {
+		.value = 0,
+		.str = "abb",
+	};
+	struct item item3 = {
+		.value = 0x1000,
+		.str = "baa",
+	};
+	struct cmp_info ctx1 = {
+		.xcode = 0,
+		.offset = 0,
+	};
+	struct cmp_info ctx2 = {
+		.xcode = 0x1000,
+		.offset = 1,
+	};
+	total_order(order1, struct item, struct cmp_info *) = {
+		fancy_cmp, &ctx1,
+	};
+	total_order(order2, struct item, struct cmp_info *) = {
+		fancy_cmp, &ctx2,
+	};
+
+	plan_tests(18);
+
+	ok1(total_order_cmp(order1, &item1, &item1) == 0);
+	ok1(total_order_cmp(order1, &item2, &item2) == 0);
+	ok1(total_order_cmp(order1, &item3, &item3) == 0);
+
+	ok1(total_order_cmp(order1, &item1, &item2) == -1);
+	ok1(total_order_cmp(order1, &item2, &item3) == -1);
+	ok1(total_order_cmp(order1, &item1, &item3) == -1);
+
+	ok1(total_order_cmp(order1, &item2, &item1) == 1);
+	ok1(total_order_cmp(order1, &item3, &item2) == 1);
+	ok1(total_order_cmp(order1, &item3, &item1) == 1);
+
+
+	ok1(total_order_cmp(order2, &item1, &item1) == 0);
+	ok1(total_order_cmp(order2, &item2, &item2) == 0);
+	ok1(total_order_cmp(order2, &item3, &item3) == 0);
+
+	ok1(total_order_cmp(order2, &item1, &item2) == 1);
+	ok1(total_order_cmp(order2, &item2, &item3) == 1);
+	ok1(total_order_cmp(order2, &item1, &item3) == 1);
+
+	ok1(total_order_cmp(order2, &item2, &item1) == -1);
+	ok1(total_order_cmp(order2, &item3, &item2) == -1);
+	ok1(total_order_cmp(order2, &item3, &item1) == -1);
+	
+	exit(0);
+}

ccan/ccan/pipecmd/LICENSE

@@ -0,0 +1 @@
+../../licenses/CC0

ccan/ccan/pipecmd/_info

@@ -0,0 +1,58 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * pipecmd - code to fork and run a command in a pipe.
+ *
+ * This code is a classic example of how to run a command in a child, while
+ * handling the case where the exec fails.
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ * // Outputs HELLO WORLD
+ * #include <ccan/pipecmd/pipecmd.h>
+ * #include <ccan/err/err.h>
+ * #include <sys/types.h>
+ * #include <sys/wait.h>
+ * #include <unistd.h>
+ * #include <ctype.h>
+ *
+ * // Runs ourselves with an argument, upcases output.
+ * int main(int argc, char **argv)
+ * {
+ *	pid_t child;
+ *	int outputfd, i, status;
+ *	char input[12];
+ *
+ *	if (argc == 2) {
+ *		write(STDOUT_FILENO, "hello world\n", 12);
+ *		exit(0);
+ *	}
+ *	child = pipecmd(&outputfd, NULL, NULL, argv[0], "ignoredarg", NULL);
+ *	if (child < 0)
+ *		err(1, "Creating child");
+ *	if (read(outputfd, input, sizeof(input)) != sizeof(input))
+ *		err(1, "Reading input");
+ *	if (waitpid(child, &status, 0) != child)
+ *		err(1, "Waiting for child");
+ *	for (i = 0; i < sizeof(input); i++)
+ *		printf("%c", toupper(input[i]));
+ *	exit(0);
+ * }
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/noerr\n");
+		return 0;
+	}
+
+	return 1;
+}

ccan/ccan/pipecmd/pipecmd.c

@@ -0,0 +1,176 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#include <ccan/pipecmd/pipecmd.h>
+#include <ccan/noerr/noerr.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+static char **gather_args(const char *arg0, va_list ap)
+{
+	size_t n = 1;
+	char **arr = calloc(sizeof(char *), n + 1);
+
+	if (!arr)
+		return NULL;
+	arr[0] = (char *)arg0;
+
+	while ((arr[n++] = va_arg(ap, char *)) != NULL) {
+		arr = realloc(arr, sizeof(char *) * (n + 1));
+		if (!arr)
+			return NULL;
+	}
+	return arr;
+}
+
+pid_t pipecmdv(int *fd_fromchild, int *fd_tochild, int *fd_errfromchild,
+	       const char *cmd, va_list ap)
+{
+	char **arr = gather_args(cmd, ap);
+	pid_t ret;
+
+	if (!arr) {
+		errno = ENOMEM;
+		return -1;
+	}
+	ret = pipecmdarr(fd_fromchild, fd_tochild, fd_errfromchild, arr);
+	free_noerr(arr);
+	return ret;
+}
+
+pid_t pipecmdarr(int *fd_fromchild, int *fd_tochild, int *fd_errfromchild,
+		 char *const *arr)
+{
+	int tochild[2], fromchild[2], errfromchild[2], execfail[2];
+	pid_t childpid;
+	int err;
+
+	if (fd_tochild) {
+		if (pipe(tochild) != 0)
+			goto fail;
+	} else {
+		tochild[0] = open("/dev/null", O_RDONLY);
+		if (tochild[0] < 0)
+			goto fail;
+	}
+	if (fd_fromchild) {
+		if (pipe(fromchild) != 0)
+			goto close_tochild_fail;
+	} else {
+		fromchild[1] = open("/dev/null", O_WRONLY);
+		if (fromchild[1] < 0)
+			goto close_tochild_fail;
+	}
+	if (fd_errfromchild) {
+		if (fd_errfromchild == fd_fromchild) {
+			errfromchild[0] = fromchild[0];
+			errfromchild[1] = fromchild[1];
+		} else {
+			if (pipe(errfromchild) != 0)
+				goto close_fromchild_fail;
+		}
+	} else {
+		errfromchild[1] = open("/dev/null", O_WRONLY);
+		if (errfromchild[1] < 0)
+			goto close_fromchild_fail;
+	}
+		
+	if (pipe(execfail) != 0)
+		goto close_errfromchild_fail;
+
+	if (fcntl(execfail[1], F_SETFD, fcntl(execfail[1], F_GETFD)
+		  | FD_CLOEXEC) < 0)
+		goto close_execfail_fail;
+
+	childpid = fork();
+	if (childpid < 0)
+		goto close_execfail_fail;
+
+	if (childpid == 0) {
+		if (fd_tochild)
+			close(tochild[1]);
+		if (fd_fromchild)
+			close(fromchild[0]);
+		if (fd_errfromchild && fd_errfromchild != fd_fromchild)
+			close(errfromchild[0]);
+
+		close(execfail[0]);
+
+		// Child runs command.
+		if (tochild[0] != STDIN_FILENO) {
+			if (dup2(tochild[0], STDIN_FILENO) == -1)
+				goto child_errno_fail;
+			close(tochild[0]);
+		}
+		if (fromchild[1] != STDOUT_FILENO) {
+			if (dup2(fromchild[1], STDOUT_FILENO) == -1)
+				goto child_errno_fail;
+			close(fromchild[1]);
+		}
+		if (fd_errfromchild && fd_errfromchild == fd_fromchild) {
+			if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+				goto child_errno_fail;
+		} else if (errfromchild[1] != STDERR_FILENO) {
+			if (dup2(errfromchild[1], STDERR_FILENO) == -1)
+				goto child_errno_fail;
+			close(errfromchild[1]);
+		}
+		execvp(arr[0], arr);
+
+	child_errno_fail:
+		err = errno;
+		write(execfail[1], &err, sizeof(err));
+		exit(127);
+	}
+
+	close(tochild[0]);
+	close(fromchild[1]);
+	close(errfromchild[1]);
+	close(execfail[1]);
+	/* Child will close this without writing on successful exec. */
+	if (read(execfail[0], &err, sizeof(err)) == sizeof(err)) {
+		close(execfail[0]);
+		waitpid(childpid, NULL, 0);
+		errno = err;
+		return -1;
+	}
+	close(execfail[0]);
+	if (fd_tochild)
+		*fd_tochild = tochild[1];
+	if (fd_fromchild)
+		*fd_fromchild = fromchild[0];
+	if (fd_errfromchild)
+		*fd_errfromchild = errfromchild[0];
+	return childpid;
+
+close_execfail_fail:
+	close_noerr(execfail[0]);
+	close_noerr(execfail[1]);
+close_errfromchild_fail:
+	if (fd_errfromchild)
+		close_noerr(errfromchild[0]);
+	close_noerr(errfromchild[1]);
+close_fromchild_fail:
+	if (fd_fromchild)
+		close_noerr(fromchild[0]);
+	close_noerr(fromchild[1]);
+close_tochild_fail:
+	close_noerr(tochild[0]);
+	if (fd_tochild)
+		close_noerr(tochild[1]);
+fail:
+	return -1;
+}
+
+pid_t pipecmd(int *fd_fromchild, int *fd_tochild, int *fd_errfromchild,
+	      const char *cmd, ...)
+{
+	pid_t childpid;
+
+	va_list ap;
+	va_start(ap, cmd);
+	childpid = pipecmdv(fd_fromchild, fd_tochild, fd_errfromchild, cmd, ap);
+	va_end(ap);
+
+	return childpid;
+}

ccan/ccan/pipecmd/pipecmd.h

@@ -0,0 +1,44 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#ifndef CCAN_PIPECMD_H
+#define CCAN_PIPECMD_H
+#include "config.h"
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+
+/**
+ * pipecmd - run a command, optionally connect pipes.
+ * @infd: input fd to write to child (if non-NULL)
+ * @outfd: output fd to read from child (if non-NULL)
+ * @errfd: error-output fd to read from child (if non-NULL)
+ * @cmd...: NULL-terminate list of command and arguments.
+ *
+ * If @infd is NULL, the child's input is (read-only) /dev/null.
+ * If @outfd is NULL, the child's output is (write-only) /dev/null.
+ * If @errfd is NULL, the child's stderr is (write-only) /dev/null.
+ *
+ * If @errfd == @outfd (and non-NULL) they will be shared.
+ *
+ * The return value is the pid of the child, or -1.
+ */
+pid_t pipecmd(int *infd, int *outfd, int *errfd, const char *cmd, ...);
+
+/**
+ * pipecmdv - run a command, optionally connect pipes (stdarg version)
+ * @infd: input fd to write to child (if non-NULL)
+ * @outfd: output fd to read from child (if non-NULL)
+ * @errfd: error-output fd to read from child (if non-NULL)
+ * @cmd: command to run.
+ * @ap: argument list for arguments.
+ */
+pid_t pipecmdv(int *infd, int *outfd, int *errfd, const char *cmd, va_list ap);
+
+/**
+ * pipecmdarr - run a command, optionally connect pipes (char arry version)
+ * @infd: input fd to write to child (if non-NULL)
+ * @outfd: output fd to read from child (if non-NULL)
+ * @errfd: error-output fd to read from child (if non-NULL)
+ * @arr: NULL-terminated array for arguments (first is program to run).
+ */
+pid_t pipecmdarr(int *infd, int *outfd, int *errfd, char *const *arr);
+#endif /* CCAN_PIPECMD_H */

ccan/ccan/pipecmd/test/run-fdleak.c

@@ -0,0 +1,44 @@
+#include <ccan/pipecmd/pipecmd.h>
+/* Include the C files directly. */
+#include <ccan/pipecmd/pipecmd.c>
+#include <ccan/tap/tap.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+int main(int argc, char *argv[])
+{
+	pid_t child;
+	int outfd, status;
+	char buf[5] = "test";
+
+	/* We call ourselves, to test pipe. */
+	if (argc == 2) {
+		if (write(STDOUT_FILENO, buf, sizeof(buf)) != sizeof(buf))
+				exit(1);
+		exit(0);
+	}
+
+	/* This is how many tests you plan to run */
+	plan_tests(13);
+	child = pipecmd(&outfd, NULL, NULL, argv[0], "out", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(read(outfd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(memcmp(buf, "test", sizeof(buf)) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* No leaks! */
+	ok1(close(outfd) == 0);
+	ok1(close(outfd) == -1 && errno == EBADF);
+	ok1(close(++outfd) == -1 && errno == EBADF);
+	ok1(close(++outfd) == -1 && errno == EBADF);
+	ok1(close(++outfd) == -1 && errno == EBADF);
+	ok1(close(++outfd) == -1 && errno == EBADF);
+	ok1(close(++outfd) == -1 && errno == EBADF);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/pipecmd/test/run.c

@@ -0,0 +1,157 @@
+#include <ccan/pipecmd/pipecmd.h>
+/* Include the C files directly. */
+#include <ccan/pipecmd/pipecmd.c>
+#include <ccan/tap/tap.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+int main(int argc, char *argv[])
+{
+	pid_t child;
+	int infd, outfd, errfd, status;
+	char buf[5] = "test";
+
+	/* We call ourselves, to test pipe. */
+	if (argc == 2) {
+		if (strcmp(argv[1], "out") == 0) {
+			if (write(STDOUT_FILENO, buf, sizeof(buf)) != sizeof(buf))
+				exit(1);
+		} else if (strcmp(argv[1], "in") == 0) {
+			if (read(STDIN_FILENO, buf, sizeof(buf)) != sizeof(buf))
+				exit(1);
+			if (memcmp(buf, "test", sizeof(buf)) != 0)
+				exit(1);
+		} else if (strcmp(argv[1], "inout") == 0) {
+			if (read(STDIN_FILENO, buf, sizeof(buf)) != sizeof(buf))
+				exit(1);
+			buf[0]++;
+			if (write(STDOUT_FILENO, buf, sizeof(buf)) != sizeof(buf))
+				exit(1);
+		} else if (strcmp(argv[1], "err") == 0) {
+			if (write(STDERR_FILENO, buf, sizeof(buf)) != sizeof(buf))
+				exit(1);
+		} else
+			abort();
+		exit(0);
+	}
+
+	/* We assume no fd leaks, so close them now. */
+	close(3);
+	close(4);
+	close(5);
+	close(6);
+	close(7);
+	close(8);
+	close(9);
+	close(10);
+	
+	/* This is how many tests you plan to run */
+	plan_tests(67);
+	child = pipecmd(&outfd, &infd, &errfd, argv[0], "inout", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(write(infd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(read(outfd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(read(errfd, buf, sizeof(buf)) == 0);
+	ok1(close(infd) == 0);
+	ok1(close(outfd) == 0);
+	ok1(close(errfd) == 0);
+	buf[0]--;
+	ok1(memcmp(buf, "test", sizeof(buf)) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	child = pipecmd(NULL, &infd, NULL, argv[0], "in", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(write(infd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(close(infd) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	child = pipecmd(&outfd, NULL, NULL, argv[0], "out", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(read(outfd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(close(outfd) == 0);
+	ok1(memcmp(buf, "test", sizeof(buf)) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* Errfd only should be fine. */
+	child = pipecmd(NULL, NULL, &errfd, argv[0], "err", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(read(errfd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(close(errfd) == 0);
+	ok1(memcmp(buf, "test", sizeof(buf)) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	/* errfd == outfd should work with both. */
+	child = pipecmd(&errfd, NULL, &errfd, argv[0], "err", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(read(errfd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(close(errfd) == 0);
+	ok1(memcmp(buf, "test", sizeof(buf)) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	child = pipecmd(&outfd, NULL, &outfd, argv[0], "out", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(read(outfd, buf, sizeof(buf)) == sizeof(buf));
+	ok1(close(outfd) == 0);
+	ok1(memcmp(buf, "test", sizeof(buf)) == 0);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	// Writing to /dev/null should be fine.
+	child = pipecmd(NULL, NULL, NULL, argv[0], "out", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	// Reading should fail.
+	child = pipecmd(NULL, NULL, NULL, argv[0], "in", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 1);
+
+	child = pipecmd(NULL, NULL, NULL, argv[0], "err", NULL);
+	if (!ok1(child > 0))
+		exit(1);
+	ok1(waitpid(child, &status, 0) == child);
+	ok1(WIFEXITED(status));
+	ok1(WEXITSTATUS(status) == 0);
+
+	// Can't run non-existent file, but errno set correctly.
+	child = pipecmd(NULL, NULL, NULL, "/doesnotexist", "in", NULL);
+	ok1(errno == ENOENT);
+	ok1(child < 0);
+
+	/* No fd leaks! */
+	ok1(close(3) == -1 && errno == EBADF);
+	ok1(close(4) == -1 && errno == EBADF);
+	ok1(close(5) == -1 && errno == EBADF);
+	ok1(close(6) == -1 && errno == EBADF);
+	ok1(close(7) == -1 && errno == EBADF);
+	ok1(close(8) == -1 && errno == EBADF);
+	ok1(close(9) == -1 && errno == EBADF);
+	ok1(close(10) == -1 && errno == EBADF);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/ptrint/_info

@@ -48,6 +48,7 @@ int main(int argc, char *argv[])
 
 	if (strcmp(argv[1], "depends") == 0) {
 		printf("ccan/build_assert\n");
+		printf("ccan/compiler\n");
 		return 0;
 	}
 	if (strcmp(argv[1], "testdepends") == 0) {

ccan/ccan/ptrint/ptrint.h

@@ -7,6 +7,7 @@
 #include <stddef.h>
 
 #include <ccan/build_assert/build_assert.h>
+#include <ccan/compiler/compiler.h>
 
 /*
  * This is a deliberately incomplete type, because it should never be
@@ -15,7 +16,7 @@
  */
 typedef struct ptrint ptrint_t;
 
-static inline ptrdiff_t ptr2int(const ptrint_t *p)
+CONST_FUNCTION static inline ptrdiff_t ptr2int(const ptrint_t *p)
 {
 	/*
 	 * ptrdiff_t is the right size by definition, but to avoid
@@ -26,7 +27,7 @@ static inline ptrdiff_t ptr2int(const ptrint_t *p)
 	return (const char *)p - (const char *)NULL;
 }
 
-static inline ptrint_t *int2ptr(ptrdiff_t i)
+CONST_FUNCTION static inline ptrint_t *int2ptr(ptrdiff_t i)
 {
 	return (ptrint_t *)((char *)NULL + i);
 }

ccan/ccan/tal/str/str.c

@@ -26,11 +26,9 @@ char *tal_strndup(const tal_t *ctx, const char *p, size_t n)
 	char *ret;
 
 	/* We have to let through NULL for take(). */
-	if (likely(p)) {
-		len = strlen(p);
-		if (len > n)
-			len = n;
-	} else
+	if (likely(p))
+		len = strnlen(p, n);
+	else
 		len = n;
 
 	ret = tal_dup_(ctx, p, 1, len, 1, false, TAL_LABEL(char, "[]"));
@@ -54,7 +52,7 @@ char *tal_fmt(const tal_t *ctx, const char *fmt, ...)
 static bool do_vfmt(char **buf, size_t off, const char *fmt, va_list ap)
 {
 	/* A decent guess to start. */
-	size_t max = strlen(fmt) * 2;
+	size_t max = strlen(fmt) * 2 + 1;
 	bool ok;
 
 	for (;;) {

ccan/ccan/tal/str/test/run-fmt-terminate.c

@@ -0,0 +1,22 @@
+#include <ccan/tal/str/str.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ccan/tal/str/str.c>
+#include <ccan/tap/tap.h>
+#include "helper.h"
+
+/* Empty format string: should still terminate! */
+int main(int argc, char *argv[])
+{
+	char *str;
+	const char *fmt = "";
+
+	plan_tests(1);
+	/* GCC complains about empty format string, complains about non-literal
+	 * with no args... */
+	str = tal_fmt(NULL, fmt, "");
+	ok1(!strcmp(str, ""));
+	tal_free(str);
+
+	return exit_status();
+}

ccan/ccan/tal/str/test/run-strndup.c

@@ -0,0 +1,22 @@
+#include <ccan/tal/str/str.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ccan/tal/str/str.c>
+#include <ccan/tap/tap.h>
+#include "helper.h"
+
+int main(int argc, char *argv[])
+{
+	char *str, *copy;
+
+	plan_tests(1);
+	str = malloc(5);
+	memcpy(str, "hello", 5);
+	/* We should be fine to strndup src without nul terminator. */
+	copy = tal_strndup(NULL, str, 5);
+	ok1(!strcmp(copy, "hello"));
+	tal_free(copy);
+	free(str);
+
+	return exit_status();
+}

ccan/ccan/tcon/_info

@@ -71,5 +71,10 @@ int main(int argc, char *argv[])
 		return 0;
 	}
 
+	if (strcmp(argv[1], "testdepends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
 	return 1;
 }

ccan/ccan/tcon/tcon.h

@@ -3,6 +3,8 @@
 #define CCAN_TCON_H
 #include "config.h"
 
+#include <stddef.h>
+
 /**
  * TCON - declare a _tcon type containing canary variables.
  * @decls: the semi-colon separated list of type canaries.
@@ -42,6 +44,77 @@
 #define TCON(decls) struct { decls; } _tcon[1]
 #endif
 
+/**
+ * TCON_WRAP - declare a wrapper type containing a base type and type canaries
+ * @basetype: the base type to wrap
+ * @decls: the semi-colon separated list of type canaries.
+ *
+ * This expands to a new type which includes the given base type, and
+ * also type canaries, similar to those created with TCON.
+ *
+ * The embedded base type value can be accessed using tcon_unwrap().
+ *
+ * Differences from using TCON()
+ * - The wrapper type will take either the size of the base type, or
+ *   the size of a single pointer, whichever is greater (regardless of
+ *   compiler)
+ * - A TCON_WRAP type may be included in another structure, and need
+ *   not be the last element.
+ *
+ * A type of "void *" will allow tcon_check() to pass on any (pointer) type.
+ *
+ * Example:
+ *	// Simply typesafe linked list.
+ *	struct list_head {
+ *		struct list_head *prev, *next;
+ *	};
+ *
+ *	typedef TCON_WRAP(struct list_head, char *canary) string_list_t;
+ *
+ *	// More complex: mapping from one type to another.
+ *	struct map {
+ *		void *contents;
+ *	};
+ *
+ *	typedef TCON_WRAP(struct map, char *charp_canary; int int_canary)
+ *		int_to_string_map_t;
+ */
+#define TCON_WRAP(basetype, decls) \
+	union {			   \
+		basetype _base;	   \
+		struct {	   \
+			decls;	   \
+		} *_tcon;	   \
+	}
+
+/**
+ * TCON_WRAP_INIT - an initializer for a variable declared with TCON_WRAP
+ * @...: Initializer for the base type (treated as variadic so commas
+ *       can be included)
+ *
+ * Converts converts an initializer suitable for a base type into one
+ * suitable for that type wrapped with TCON_WRAP.
+ *
+ * Example:
+ *	TCON_WRAP(int, char *canary) canaried_int = TCON_WRAP_INIT(17);
+ */
+#define TCON_WRAP_INIT(...)			\
+	{ ._base = __VA_ARGS__, }
+
+/**
+ * tcon_unwrap - Access the base type of a TCON_WRAP
+ * @ptr: pointer to an object declared with TCON_WRAP
+ *
+ * tcon_unwrap() returns a pointer to the base type of the TCON_WRAP()
+ * object pointer to by @ptr.
+ *
+ * Example:
+ *	TCON_WRAP(int, char *canary) canaried_int;
+ *
+ *	*tcon_unwrap(&canaried_int) = 17;
+ */
+#define tcon_unwrap(ptr) (&((ptr)->_base))
+
 /**
  * tcon_check - typecheck a typed container
  * @x: the structure containing the TCON.
@@ -88,6 +161,35 @@
 #define tcon_type(x, canary) void *
 #endif
 
+/**
+ * tcon_sizeof - the size of type within a container
+ * @x: the structure containing the TCON.
+ * @canary: which canary to check against.
+ */
+#define tcon_sizeof(x, canary) sizeof((x)->_tcon[0].canary)
+
+/**
+ * TCON_VALUE - encode an integer value in a type canary
+ * @canary: name of the value canary
+ * @val: positive integer compile time constant value
+ *
+ * This macro can be included inside the declarations in a TCON() or
+ * TCON_WRAP(), constructing a special "type" canary which encodes the
+ * integer value @val (which must be a compile time constant, and a
+ * positive integer in the range of size_t).
+ */
+#define TCON_VALUE(canary, val)	char _value_##canary[val]
+
+/**
+ * tcon_value - retrieve the value of a TCON_VALUE canary
+ * @x: the structure containing the TCON
+ * @canary: name of the value canary
+ *
+ * This macros expands to the value previously encoded into a TCON
+ * using TCON_VALUE().
+ */
+#define tcon_value(x, canary)	tcon_sizeof(x, _value_##canary)
+
 /**
  * tcon_ptr_type - pointer to the type within a container (or void *)
  * @x: the structure containing the TCON.
@@ -112,4 +214,154 @@
 #define tcon_cast(x, canary, expr) ((tcon_type((x), canary))(expr))
 #define tcon_cast_ptr(x, canary, expr) ((tcon_ptr_type((x), canary))(expr))
 
+/**
+ * TCON_CONTAINER - encode information on a specific member of a
+ *                  containing structure into a "type" canary
+ * @canary: name of the container canary
+ * @container: type of the container structure
+ * @member: name of the member
+ *
+ * Used in the declarations in TCON() or TCON_WRAP(), encode a
+ * "container canary".  This encodes the type of @container, the type
+ * of @member within it (with sufficient compiler support) and the
+ * offset of @member within @container.
+ */
+#if HAVE_TYPEOF
+#define TCON_CONTAINER(canary, container, member)			\
+	container _container_##canary;					\
+	typeof(((container *)0)->member) _member_##canary;		\
+	TCON_VALUE(_offset_##canary, offsetof(container, member))
+#else
+#define TCON_CONTAINER(canary, container, member)			\
+	container _container_##canary;					\
+	TCON_VALUE(_offset_##canary, offsetof(container, member))
+#endif
+
+/**
+ * tcon_container_check
+ * tcon_container_check_ptr
+ * tcon_container_type
+ * tcon_container_ptr_type
+ * tcon_container_sizeof
+ * tcon_container_cast
+ * tcon_container_cast_ptr
+ * @x: the structure containing the TCON.
+ * @canary: which container canary to check against.
+ *
+ * As tcon_check / tcon_check_ptr / tcon_type / tcon_ptr_type /
+ * tcon_sizeof / tcon_cast / tcon_cast_ptr, but use the type of the
+ * "container" type declared with TCON_CONTAINER, instead of a simple
+ * canary.
+ */
+#define tcon_container_check(x, canary, expr)		\
+	tcon_check(x, _container_##canary, expr)
+#define tcon_container_check_ptr(x, canary, expr)	\
+	tcon_check_ptr(x, _container_##canary, expr)
+#define tcon_container_type(x, canary)		\
+	tcon_type(x, _container_##canary)
+#define tcon_container_ptr_type(x, canary)	\
+	tcon_ptr_type(x, _container_##canary)
+#define tcon_container_sizeof(x, canary)	\
+	tcon_sizeof(x, _container_##canary)
+#define tcon_container_cast(x, canary, expr)	\
+	tcon_cast(x, _container_##canary, expr)
+#define tcon_container_cast_ptr(x, canary, expr)	\
+	tcon_cast_ptr(x, _container_##canary, expr)
+
+/**
+ * tcon_member_check
+ * tcon_member_check_ptr
+ * tcon_member_type
+ * tcon_member_ptr_type
+ * tcon_member_sizeof
+ * tcon_member_cast
+ * tcon_member_cast_ptr
+ * @x: the structure containing the TCON.
+ * @canary: which container canary to check against.
+ *
+ * As tcon_check / tcon_check_ptr / tcon_type / tcon_ptr_type /
+ * tcon_sizeof / tcon_cast / tcon_cast_ptr, but use the type of the
+ * "member" type declared with TCON_CONTAINER, instead of a simple
+ * canary.
+ */
+#define tcon_member_check(x, canary, expr)	\
+	tcon_check(x, _member_##canary, expr)
+#define tcon_member_check_ptr(x, canary, expr)		\
+	tcon_check_ptr(x, _member_##canary, expr)
+#define tcon_member_type(x, canary)		\
+	tcon_type(x, _member_##canary)
+#define tcon_member_ptr_type(x, canary)	\
+	tcon_ptr_type(x, _member_##canary)
+#define tcon_member_sizeof(x, canary)	\
+	tcon_sizeof(x, _member_##canary)
+#define tcon_member_cast(x, canary, expr)	\
+	tcon_cast(x, _member_##canary, expr)
+#define tcon_member_cast_ptr(x, canary, expr)	\
+	tcon_cast_ptr(x, _member_##canary, expr)
+
+/**
+ * tcon_offset - the offset of a member within a container, as
+ *               declared with TCON_CONTAINER
+ * @x: the structure containing the TCON.
+ * @canary: which container canary to check against.
+ */
+#define tcon_offset(x, canary)			\
+	tcon_value((x), _offset_##canary)
+
+/**
+ * tcon_container_of - get pointer to enclosing structure based on a
+ *                     container canary
+ * @x: the structure containing the TCON
+ * @canary: the name of the container canary
+ * @member_ptr: pointer to a member of the container
+ *
+ * @member_ptr must be a pointer to the member of a container
+ * structure previously recorded in @canary with TCON_CONTAINER.
+ *
+ * tcon_container_of() evaluates to a pointer to the container
+ * structure.  With sufficient compiler support, the pointer will be
+ * correctly typed, and the type of @member_ptr will be verified.
+ *
+ * Returns NULL if @member_ptr is NULL.
+ */
+#define tcon_container_of(x, canary, member_ptr)			\
+	tcon_container_cast_ptr(					\
+		tcon_member_check_ptr((x), canary, (member_ptr)),	\
+		canary, tcon_container_of_((member_ptr),		\
+					   tcon_offset((x), canary)))
+
+static inline void *tcon_container_of_(void *member_ptr, size_t offset)
+{
+	return member_ptr ? (char *)member_ptr - offset : NULL;
+}
+
+
+/**
+ * tcon_member_of - get pointer to enclosed member structure based on a
+ *                  container canary
+ * @x: the structure containing the TCON
+ * @canary: the name of the container canary
+ * @container_ptr: pointer to a container
+ *
+ * @container_ptr must be a pointer to a container structure
+ * previously recorded in @canary with TCON_CONTAINER.
+ *
+ * tcon_member_of() evaluates to a pointer to the member of the
+ * container recorded in @canary. With sufficient compiler support,
+ * the pointer will be correctly typed, and the type of @container_ptr
+ * will be verified.
+ *
+ * Returns NULL if @container_ptr is NULL.
+ */
+#define tcon_member_of(x, canary, container_ptr)			\
+	tcon_member_cast_ptr(						\
+		tcon_container_check_ptr((x), canary, (container_ptr)),	\
+		canary, tcon_member_of_((container_ptr),		\
+					tcon_offset((x), canary)))
+static inline void *tcon_member_of_(void *container_ptr, size_t offset)
+{
+	return container_ptr ? (char *)container_ptr + offset : NULL;
+}
+
+
 #endif /* CCAN_TCON_H */

ccan/ccan/tcon/test/compile_fail-container1.c

@@ -0,0 +1,39 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+struct info_tcon {
+	struct info_base base;
+	TCON(TCON_CONTAINER(concan, struct outer, inner));
+};
+
+int main(int argc, char *argv[])
+{
+	struct info_tcon info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	int *innerp = &ovar.outer_val;
+#else
+	struct inner *innerp = &ovar.inner;
+#endif
+
+	return tcon_container_of(&info, concan, innerp) == &ovar;
+}

ccan/ccan/tcon/test/compile_fail-container1w.c

@@ -0,0 +1,35 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+int main(int argc, char *argv[])
+{
+	TCON_WRAP(struct info_base,
+		  TCON_CONTAINER(concan, struct outer, inner)) info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	int *innerp = &ovar.outer_val;
+#else
+	struct inner *innerp = &ovar.inner;
+#endif
+
+	return tcon_container_of(&info, concan, innerp) == &ovar;
+}

ccan/ccan/tcon/test/compile_fail-container2.c

@@ -0,0 +1,39 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+struct info_tcon {
+	struct info_base base;
+	TCON(TCON_CONTAINER(concan, struct outer, inner));
+};
+
+int main(int argc, char *argv[])
+{
+	struct info_tcon info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	char *outerp = NULL;
+#else
+	struct outer *outerp = &ovar;
+#endif
+
+	return tcon_member_of(&info, concan, outerp) == &ovar.inner;
+}

ccan/ccan/tcon/test/compile_fail-container2w.c

@@ -0,0 +1,35 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+int main(int argc, char *argv[])
+{
+	TCON_WRAP(struct info_base,
+		  TCON_CONTAINER(concan, struct outer, inner)) info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	char *outerp = NULL;
+#else
+	struct outer *outerp = &ovar;
+#endif
+
+	return tcon_member_of(&info, concan, outerp) == &ovar.inner;
+}

ccan/ccan/tcon/test/compile_fail-container3.c

@@ -0,0 +1,40 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+struct info_tcon {
+	struct info_base base;
+	TCON(TCON_CONTAINER(concan, struct outer, inner));
+};
+
+int main(int argc, char *argv[])
+{
+	struct info_tcon info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	int *outerp;
+#else
+	struct outer *outerp;
+#endif
+
+	outerp = tcon_container_of(&info, concan, &ovar.inner);
+	return outerp != NULL;
+}

ccan/ccan/tcon/test/compile_fail-container3w.c

@@ -0,0 +1,36 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+int main(int argc, char *argv[])
+{
+	TCON_WRAP(struct info_base,
+		  TCON_CONTAINER(concan, struct outer, inner)) info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	int *outerp;
+#else
+	struct outer *outerp;
+#endif
+
+	outerp = tcon_container_of(&info, concan, &ovar.inner);
+	return outerp != NULL;
+}

ccan/ccan/tcon/test/compile_fail-container4.c

@@ -0,0 +1,40 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+struct info_tcon {
+	struct info_base base;
+	TCON(TCON_CONTAINER(concan, struct outer, inner));
+};
+
+int main(int argc, char *argv[])
+{
+	struct info_tcon info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	int *innerp;
+#else
+	struct inner *innerp;
+#endif
+
+	innerp = tcon_member_of(&info, concan, &ovar);
+	return innerp != NULL;
+}

ccan/ccan/tcon/test/compile_fail-container4w.c

@@ -0,0 +1,36 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct info_base {
+	char *infop;
+};
+
+int main(int argc, char *argv[])
+{
+	TCON_WRAP(struct info_base,
+		  TCON_CONTAINER(concan, struct outer, inner)) info;
+	struct outer ovar;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	int *innerp;
+#else
+	struct inner *innerp;
+#endif
+
+	innerp = tcon_member_of(&info, concan, &ovar);
+	return innerp != NULL;
+}

ccan/ccan/tcon/test/compile_fail-tcon_cast_wrap.c

@@ -0,0 +1,25 @@
+#include <ccan/tcon/tcon.h>
+#include <stdlib.h>
+
+struct container {
+	void *p;
+};
+
+int main(int argc, char *argv[])
+{
+	TCON_WRAP(struct container,
+		  int *tc1; char *tc2) icon;
+#ifdef FAIL
+#if !HAVE_TYPEOF
+#error We cannot detect type problems without HAVE_TYPEOF
+#endif
+	char *
+#else
+	int *
+#endif
+		x;
+
+	tcon_unwrap(&icon)->p = NULL;
+	x = tcon_cast(&icon, tc1, tcon_unwrap(&icon)->p);
+	return x != NULL ? 0 : 1;
+}

ccan/ccan/tcon/test/compile_fail-wrap.c

@@ -0,0 +1,20 @@
+#include <ccan/tcon/tcon.h>
+#include <stdlib.h>
+
+struct container {
+	void *p;
+};
+
+int main(int argc, char *argv[])
+{
+	TCON_WRAP(struct container, int *canary) icon;
+#ifdef FAIL
+	char *
+#else
+	int *
+#endif
+		x = NULL;
+
+	tcon_unwrap(tcon_check(&icon, canary, x))->p = x;
+	return 0;
+}

ccan/ccan/tcon/test/compile_ok-sizeof.c

@@ -0,0 +1,35 @@
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <stdlib.h>
+
+struct container {
+	void *p;
+};
+
+struct int_container {
+	struct container raw;
+	TCON(int tc);
+};
+
+struct charp_and_int_container {
+	struct container raw;
+	TCON(int tc1; char *tc2);
+};
+
+int main(int argc, char *argv[])
+{
+	struct int_container icon;
+	struct charp_and_int_container cicon;
+	TCON_WRAP(struct container, int tc) iconw;
+	TCON_WRAP(struct container, int tc1; char *tc2) ciconw;
+
+	BUILD_ASSERT(tcon_sizeof(&icon, tc) == sizeof(int));
+	BUILD_ASSERT(tcon_sizeof(&cicon, tc1) == sizeof(int));
+	BUILD_ASSERT(tcon_sizeof(&cicon, tc2) == sizeof(char *));
+
+	BUILD_ASSERT(tcon_sizeof(&iconw, tc) == sizeof(int));
+	BUILD_ASSERT(tcon_sizeof(&ciconw, tc1) == sizeof(int));
+	BUILD_ASSERT(tcon_sizeof(&ciconw, tc2) == sizeof(char *));
+
+	return 0;
+}

ccan/ccan/tcon/test/compile_ok-value.c

@@ -0,0 +1,51 @@
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <stdlib.h>
+#include <stddef.h>
+
+struct container {
+	void *p;
+};
+
+struct val_container {
+	struct container raw;
+	TCON(TCON_VALUE(fixed_val, 17));
+};
+
+struct other_struct {
+	char junk1;
+	int x1;
+	long junk2;
+	char *x2;
+	short junk3;
+};
+
+struct offs_container {
+	struct container raw;
+	TCON(TCON_VALUE(off1, offsetof(struct other_struct, x1));
+	     TCON_VALUE(off2, offsetof(struct other_struct, x2)));
+};
+
+int main(int argc, char *argv[])
+{
+	struct val_container valcon;
+	struct offs_container offscon;
+	TCON_WRAP(struct container, TCON_VALUE(fixed_val, 17)) valconw;
+	TCON_WRAP(struct container,
+		  TCON_VALUE(off1, offsetof(struct other_struct, x1));
+		  TCON_VALUE(off2, offsetof(struct other_struct, x2))) offsconw;
+
+	BUILD_ASSERT(tcon_value(&valcon, fixed_val) == 17);
+	BUILD_ASSERT(tcon_value(&valconw, fixed_val) == 17);
+
+	BUILD_ASSERT(tcon_value(&offscon, off1)
+		     == offsetof(struct other_struct, x1));
+	BUILD_ASSERT(tcon_value(&offscon, off2)
+		     == offsetof(struct other_struct, x2));
+	BUILD_ASSERT(tcon_value(&offsconw, off1)
+		     == offsetof(struct other_struct, x1));
+	BUILD_ASSERT(tcon_value(&offsconw, off2)
+		     == offsetof(struct other_struct, x2));
+
+	return 0;
+}

ccan/ccan/tcon/test/compile_ok-void.c

@@ -13,9 +13,15 @@ struct void_container {
 int main(int argc, char *argv[])
 {
 	struct void_container vcon;
+	TCON_WRAP(struct container, void *canary) vconw;
 
 	tcon_check(&vcon, canary, NULL)->raw.p = NULL;
 	tcon_check(&vcon, canary, argv[0])->raw.p = NULL;
 	tcon_check(&vcon, canary, main)->raw.p = NULL;
+
+	tcon_unwrap(tcon_check(&vconw, canary, NULL))->p = NULL;
+	tcon_unwrap(tcon_check(&vconw, canary, argv[0]))->p = NULL;
+	tcon_unwrap(tcon_check(&vconw, canary, main))->p = NULL;
+
 	return 0;
 }

ccan/ccan/tcon/test/compile_ok.c

@@ -1,4 +1,5 @@
 #include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
 #include <stdlib.h>
 
 struct container {
@@ -19,9 +20,19 @@ int main(int argc, char *argv[])
 {
 	struct int_container icon;
 	struct charp_and_int_container cicon;
+	TCON_WRAP(struct container, int tc) iconw;
+	TCON_WRAP(struct container, int tc1; char *tc2) ciconw;
 
 	tcon_check(&icon, tc, 7)->raw.p = NULL;
 	tcon_check(&cicon, tc1, 7)->raw.p = argv[0];
 	tcon_check(&cicon, tc2, argv[0])->raw.p = argv[0];
+
+	tcon_unwrap(tcon_check(&iconw, tc, 7))->p = NULL;
+	tcon_unwrap(tcon_check(&ciconw, tc1, 7))->p = argv[0];
+	tcon_unwrap(tcon_check(&ciconw, tc2, argv[0]))->p = argv[0];
+
+	BUILD_ASSERT(sizeof(iconw) == sizeof(struct container));
+	BUILD_ASSERT(sizeof(ciconw) == sizeof(struct container));
+
 	return 0;
 }

ccan/ccan/tcon/test/run-container.c

@@ -0,0 +1,59 @@
+#include <stdlib.h>
+
+#include <ccan/tcon/tcon.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+struct inner {
+	int inner_val;
+};
+
+struct outer {
+	int outer_val;
+	struct inner inner;
+};
+
+struct outer0 {
+	struct inner inner;
+	int outer0_val;
+};
+
+struct info_base {
+	char *infop;
+};
+
+struct info_tcon {
+	struct info_base base;
+	TCON(TCON_CONTAINER(fi, struct outer, inner);
+	     TCON_CONTAINER(fi2, struct outer0, inner));
+};
+
+int main(int argc, char *argv[])
+{
+	struct info_tcon info;
+	TCON_WRAP(struct info_base,
+		  TCON_CONTAINER(fi, struct outer, inner);
+		  TCON_CONTAINER(fi2, struct outer0, inner)) infow;
+	struct outer ovar;
+	struct outer0 ovar2;
+
+	plan_tests(12);
+
+	ok1(tcon_container_of(&info, fi, &ovar.inner) == &ovar);
+	ok1(tcon_member_of(&info, fi, &ovar) == &ovar.inner);
+	ok1(tcon_container_of(&infow, fi, &ovar.inner) == &ovar);
+	ok1(tcon_member_of(&infow, fi, &ovar) == &ovar.inner);
+
+	ok1(tcon_container_of(&info, fi2, &ovar2.inner) == &ovar2);
+	ok1(tcon_member_of(&info, fi2, &ovar2) == &ovar2.inner);
+	ok1(tcon_container_of(&infow, fi2, &ovar2.inner) == &ovar2);
+	ok1(tcon_member_of(&infow, fi2, &ovar2) == &ovar2.inner);
+
+	/* Check handling of NULLs */
+	ok1(tcon_container_of(&info, fi, NULL) == NULL);
+	ok1(tcon_member_of(&info, fi, NULL) == NULL);
+	ok1(tcon_container_of(&infow, fi, NULL) == NULL);
+	ok1(tcon_member_of(&infow, fi, NULL) == NULL);
+
+	return 0;
+}

ccan/ccan/tcon/test/run-wrap.c

@@ -0,0 +1,18 @@
+#include <ccan/tcon/tcon.h>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+
+typedef TCON_WRAP(int, char *canary) canaried_int;
+
+int main(int argc, char *argv[])
+{
+	canaried_int ci = TCON_WRAP_INIT(0);
+
+	plan_tests(2);
+
+	ok1(*tcon_unwrap(&ci) == 0);
+	*tcon_unwrap(&ci) = 17;
+	ok1(*tcon_unwrap(&ci) == 17);
+
+	return exit_status();
+}

ccan/ccan/timer/LICENSE

@@ -0,0 +1 @@
+../../licenses/LGPL-2.1

ccan/ccan/timer/_info

@@ -0,0 +1,80 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * timer - efficient implementation of rarely-expiring timers.
+ *
+ * This is a lazy implementation of timers: you can add and delete timers
+ * very quickly, and they are only sorted as their expiry approaches.
+ *
+ * This is a common case for timeouts, which must often be set, but
+ * rarely expire.
+ *
+ * Example:
+ *	// Silly example which outputs strings until timers expire.
+ *	#include <ccan/timer/timer.h>
+ *	#include <ccan/time/time.h>
+ *	#include <stdlib.h>
+ *	#include <stdio.h>
+ *
+ *	struct timed_string {
+ *		struct list_node node;
+ *		struct timer timer;
+ *		const char *string;
+ *	};
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct timers timers;
+ *		struct list_head strings;
+ *		struct timer *t;
+ *		struct timed_string *s;
+ *
+ *		timers_init(&timers, time_now());
+ *		list_head_init(&strings);
+ *
+ *		while (argv[1]) {
+ *			s = malloc(sizeof(*s));
+ *			s->string = argv[1];
+ *			timer_add(&timers, &s->timer,
+ *				  timeabs_add(time_now(),
+ *					      time_from_msec(atol(argv[2]))));
+ *			list_add_tail(&strings, &s->node);
+ *			argv += 2;
+ *		}
+ *
+ *		while (!list_empty(&strings)) {
+ *			struct timeabs now = time_now();
+ *			list_for_each(&strings, s, node)
+ *				printf("%s", s->string);
+ *			while ((t = timers_expire(&timers, now)) != NULL) {
+ *				s = container_of(t, struct timed_string, timer);
+ *				list_del_from(&strings, &s->node);
+ *				free(s);
+ *			}
+ *		}
+ *
+ *		exit(0);
+ *	}
+ *
+ * License: LGPL (v2.1 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/array_size\n");
+		printf("ccan/ilog\n");
+		printf("ccan/likely\n");
+		printf("ccan/list\n");
+		printf("ccan/time\n");
+		return 0;
+	}
+
+	return 1;
+}

ccan/ccan/timer/benchmarks/Makefile

@@ -0,0 +1,35 @@
+ALL:=expected-usage
+CCANDIR:=../../..
+CFLAGS:=-Wall -I$(CCANDIR) -O3 -flto
+LDFLAGS:=-O3 -flto
+LDLIBS:=-lrt
+
+OBJS:=time.o timer.o list.o opt_opt.o opt_parse.o opt_usage.o opt_helpers.o expected-usage.o
+
+default: $(ALL)
+
+expected-usage: $(OBJS)
+
+opt_parse.o: $(CCANDIR)/ccan/opt/parse.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+opt_usage.o: $(CCANDIR)/ccan/opt/usage.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+opt_helpers.o: $(CCANDIR)/ccan/opt/helpers.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+opt_opt.o: $(CCANDIR)/ccan/opt/opt.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+time.o: $(CCANDIR)/ccan/time/time.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+timer.o: $(CCANDIR)/ccan/timer/timer.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+list.o: $(CCANDIR)/ccan/list/list.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+clean:
+	$(RM) *.o $(ALL)

ccan/ccan/timer/benchmarks/benchmark.c

@@ -0,0 +1,53 @@
+#include <ccan/time/time.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef FIRST_APPROX
+#include "first-approx.c"
+#endif
+#ifdef SECOND_APPROX
+#include "second-approx.c"
+#endif
+#ifdef NO_APPROX
+#include "no-approx.c"
+#endif
+
+int main(int argc, char *argv[])
+{
+	struct timespec start, val, val2, end, diff;
+	unsigned int i, j, limit = atoi(argv[1] ?: "100000");
+	uint64_t val64;
+
+	val = start = time_now();
+	val64 = to_u64(start);
+	val2.tv_sec = 0;
+	val2.tv_nsec = 1;
+
+	for (j = 0; j < limit; j++) {
+		for (i = 0; i < limit; i++) {
+			val = time_add(val, val2);
+			val64 += to_u64(val2);
+		}
+	}
+
+	end = time_now();
+
+	printf("val64 says %lu.%09lu\n",
+	       from_u64(val64).tv_sec,
+	       from_u64(val64).tv_nsec);
+
+	printf("val says %lu.%09lu\n",
+	       val.tv_sec,
+	       val.tv_nsec);
+
+	if (time_greater(val, from_u64(val64)))
+		diff = time_sub(val, from_u64(val64));
+	else
+		diff = time_sub(from_u64(val64), val);
+
+	printf("Time %lluns, error = %i%%\n",
+	       (long long)time_to_nsec(time_sub(end, start)),
+	       (int)(100 * time_to_nsec(diff) / time_to_nsec(time_sub(val, start))));
+	return 0;
+}

ccan/ccan/timer/benchmarks/expected-usage.c

@@ -0,0 +1,71 @@
+/* We expect a timer to rarely go off, so benchmark that case:
+ * Every 1ms a connection comes in, we set up a 30 second timer for it.
+ * After 8192ms we finish the connection (and thus delete the timer).
+ */
+#include <ccan/timer/timer.h>
+#include <ccan/opt/opt.h>
+#include <ccan/array_size/array_size.h>
+#include <stdio.h>
+
+#define PER_CONN_TIME 8192
+#define CONN_TIMEOUT_MS 30000
+
+int main(int argc, char *argv[])
+{
+	struct timespec start, curr;
+	struct timers timers;
+	struct list_head expired;
+	struct timer t[PER_CONN_TIME];
+	unsigned int i, num;
+	bool check = false;
+
+	opt_register_noarg("-c|--check", opt_set_bool, &check,
+			   "Check timer structure during progress");
+
+	opt_parse(&argc, argv, opt_log_stderr_exit);
+
+	num = argv[1] ? atoi(argv[1]) : (check ? 100000 : 100000000);
+
+	list_head_init(&expired);
+	curr = start = time_now();
+	timers_init(&timers, start);
+
+	for (i = 0; i < num; i++) {
+		curr = time_add(curr, time_from_msec(1));
+		if (check)
+			timers_check(&timers, NULL);
+		if (timers_expire(&timers, curr))
+			abort();
+		if (check)
+			timers_check(&timers, NULL);
+
+		if (i >= PER_CONN_TIME) {
+			timer_del(&timers, &t[i%PER_CONN_TIME]);
+			if (check)
+				timers_check(&timers, NULL);
+		}
+		timer_add(&timers, &t[i%PER_CONN_TIME],
+			  time_add(curr, time_from_msec(CONN_TIMEOUT_MS)));
+		if (check)
+			timers_check(&timers, NULL);
+	}
+	if (num > PER_CONN_TIME) {
+		for (i = 0; i < PER_CONN_TIME; i++)
+			timer_del(&timers, &t[i]);
+	}
+
+	curr = time_sub(time_now(), start);
+	if (check)
+		timers_check(&timers, NULL);
+	timers_cleanup(&timers);
+	opt_free_table();
+
+	for (i = 0; i < ARRAY_SIZE(timers.level); i++)
+		if (!timers.level[i])
+			break;
+
+	printf("%u in %lu.%09lu (%u levels / %zu)\n",
+	       num, (long)curr.tv_sec, curr.tv_nsec,
+	       i, ARRAY_SIZE(timers.level));
+	return 0;
+}

ccan/ccan/timer/design.txt

@@ -0,0 +1,76 @@
+Cascading timer design.
+
+Inspired by the Linux kernel approach, documented roughly at:
+	https://lwn.net/Articles/152436/
+
+For easy description, we use whole seconds and powers of 10: in the
+implementation, we use powers of 2 (eg. 256 entries) and smaller
+granularities.
+
+We start with a simple data structure:
+
+struct timer_level {
+	struct timer_level *next;
+
+	/* Ten buckets: 	[0] [1] [2] [3] [4] [5] [6] [7] [8] [9] */
+	struct list_head bucket[10];
+};
+
+struct timers {
+	/* We can never have a timer before this, aka "now". */
+	time_t offset;
+
+	struct timer_level *level;
+
+	/* Anything too far in the future. */
+	struct list_head far;
+}
+
+The first level of timers holds anything which will happen in the next
+10 seconds.  The next level holds things which will happen in the next
+100 seconds.  And so on.
+
+When we want to add a new timer into the structure, we need to figure
+out first what level it goes into, and second, which bucket.  Say our
+offset is 500,000,001 (about Tue Nov 5, 1985 in Unix time).  And our
+timer is set to go off in 5 seconds, ie. 500,000,006.
+
+The level is easy: the difference between the timer and the offset is
+5, and that's less than 10, so it's in the first level.  The position,
+however, depends on the absolute time, in this case the last digit 6,
+so it's in bucket 6.
+
+Adding a timer at 500,000,123?  The difference is > 100 and < 1000, so
+it's in the third level.  The bucket is 1.  If there's no third level,
+we just add it to the 'far' list for stuff which is in the far future.
+
+Deleting a timer is as simple as removing it; there is no external
+bookkeeping in this scheme.  This matters, since timers used for
+timeouts are almost always deleted before they expire.
+
+Now, when a second passes, we need to know if there are any timers
+which are due.  We increment the offset to 500,000,002, and look in
+the first level, bucket 2 for any timers, so lookup is simple.
+
+We do this eight more times, and we increment the offset to
+500,000,010.  We've swept around back to bucket 0, though it may not
+be empty if we added more timers as we were going.
+
+But we need to look into the next level since a timer at 500,000,010
+added when the offset was 500,000,000 would have gone up there.  We
+empty bucket 1 (due to the '1' in 500,000,010) into these buckets,
+which will contain timers between 500,000,010 and 500,000,019, which
+all now are less than 10 seconds away, so belong in the bottom level.
+
+Similarly, at 500,000,020 we will empty bucket 1 of the second level
+into the first level.  And at 500,000,100 we will empty bucket 1 of
+the third level into the second level then bucket 0 of the second
+level into the first level.  We do it in this order, since emptying
+bucket 1 on the third level (500,000,100 - 500,000,199) may put more
+entries (500,000,100 - 500,000,109) into bucket 0 on the second level.
+
+When we get to 500,001,000 we should empty the fourth level.  If there
+is no fourth level, that's when we sort through the 'far' list and
+empty any which are less than 500,002,000.  If there are many entries
+in the far list, we should add more levels to reduce the number, or at
+least the frequency we have to check it.

ccan/ccan/timer/test/run-add.c

@@ -0,0 +1,53 @@
+#include <ccan/timer/timer.h>
+/* Include the C files directly. */
+#include <ccan/timer/timer.c>
+#include <ccan/tap/tap.h>
+
+/* More than 32 bits */
+#define MAX_ORD 34
+
+/* 0...17, 63, 64, 65, 127, 128, 129, 255, 256, 257, ... */
+static uint64_t next(uint64_t base)
+{
+	if (base > 16 && ((base - 1) & ((base - 1) >> 1)) == 0)
+		return base * 2 - 3;
+	return base+1;
+}
+
+int main(void)
+{
+	struct timers timers;
+	struct timer t;
+	uint64_t diff;
+	unsigned int i;
+	struct timeabs epoch = { { 0, 0 } };
+
+	/* This is how many tests you plan to run */
+	plan_tests(2 + (18 + (MAX_ORD - 4) * 3) * (18 + (MAX_ORD - 4) * 3));
+
+	timers_init(&timers, epoch);
+	ok1(timers_check(&timers, NULL));
+
+	for (i = 0; i < 4; i++)
+		add_level(&timers, i);
+
+	i = 0;
+	timer_init(&t);
+	for (diff = 0; diff < (1ULL << MAX_ORD)+2; diff = next(diff)) {
+		i++;
+		for (timers.base = 0;
+		     timers.base < (1ULL << MAX_ORD)+2;
+		     timers.base = next(timers.base)) {
+			timer_add(&timers, &t, grains_to_time(timers.base + diff));
+			ok1(timers_check(&timers, NULL));
+			timer_del(&timers, &t);
+		}
+	}
+
+	ok1(timers_check(&timers, NULL));
+
+	timers_cleanup(&timers);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/timer/test/run-corrupt.c

@@ -0,0 +1,74 @@
+#define CCAN_TIMER_DEBUG 1
+#include <ccan/timer/timer.h>
+/* Include the C files directly. */
+#include <ccan/timer/timer.c>
+#include <ccan/tap/tap.h>
+
+static void new_timer(struct timers *timers, unsigned long nsec)
+{
+	struct timer *timer;
+	struct timeabs when;
+
+	timer = malloc(sizeof(*timer));
+	timer_init(timer);
+	when.ts.tv_sec = 0; when.ts.tv_nsec = nsec;
+	timer_add(timers, timer, when);
+}
+
+static void update_and_expire(struct timers *timers)
+{
+	struct timeabs when;
+
+	timer_earliest(timers, &when);
+	free(timers_expire(timers, when));
+}
+
+int main(int argc, char *argv[])
+{
+	struct timeabs when;
+	struct timers timers;
+
+	plan_tests(7);
+	
+	when.ts.tv_sec = 0; when.ts.tv_nsec = 0;
+	timers_init(&timers, when);
+
+	/* Add these */
+	new_timer(&timers, 35000000);
+	new_timer(&timers, 38000000);
+	new_timer(&timers, 59000000);
+	new_timer(&timers, 65000000);
+	new_timer(&timers, 88000000);
+	new_timer(&timers, 125000000);
+	new_timer(&timers, 130000000);
+	new_timer(&timers, 152000000);
+	new_timer(&timers, 168000000);
+	/* Expire all but the last one. */
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	update_and_expire(&timers);
+	/* Add a new one. */
+	new_timer(&timers, 169000000);
+	ok1(timers_check(&timers, NULL));
+
+	/* Used to get the wrong one... */
+	timers_dump(&timers, stdout);
+	ok1(timer_earliest(&timers, &when));
+	ok1(when.ts.tv_nsec == 168000000);
+	free(timers_expire(&timers, when));
+
+	ok1(timer_earliest(&timers, &when));
+	ok1(when.ts.tv_nsec == 169000000);
+	free(timers_expire(&timers, when));
+
+	ok1(timers_check(&timers, NULL));
+	ok1(!timer_earliest(&timers, &when));
+	timers_cleanup(&timers);
+
+	return exit_status();
+}

ccan/ccan/timer/test/run-expiry.c

@@ -0,0 +1,29 @@
+#include <ccan/timer/timer.h>
+/* Include the C files directly. */
+#include <ccan/timer/timer.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	struct timers timers;
+	struct timer t;
+
+	/* This is how many tests you plan to run */
+	plan_tests(7);
+
+	timers_init(&timers, grains_to_time(1364984760903400ULL));
+	ok1(timers.base == 1364984760903400ULL);
+	timer_init(&t);
+	timer_add(&timers, &t, grains_to_time(1364984761003398ULL));
+	ok1(t.time == 1364984761003398ULL);
+	ok1(timers.first == 1364984761003398ULL);
+	ok1(!timers_expire(&timers, grains_to_time(1364984760903444ULL)));
+	ok1(timers_check(&timers, NULL));
+	ok1(!timers_expire(&timers, grains_to_time(1364984761002667ULL)));
+	ok1(timers_check(&timers, NULL));
+
+	timers_cleanup(&timers);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}