ccan: update.

This was from a different series, so I just cherry-picked it.

It adds ccan/membuf as a depenency of ccan/rbuf, though we don't use
it directly yet.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Makefile

@@ -68,6 +68,7 @@ CCAN_OBJS :=					\
 	ccan-isaac64.o				\
 	ccan-list.o				\
 	ccan-mem.o				\
+	ccan-membuf.o				\
 	ccan-noerr.o				\
 	ccan-opt-helpers.o			\
 	ccan-opt-parse.o			\
@@ -129,6 +130,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
+	$(CCANDIR)/ccan/membuf/membuf.h			\
 	$(CCANDIR)/ccan/noerr/noerr.h			\
 	$(CCANDIR)/ccan/opt/opt.h			\
 	$(CCANDIR)/ccan/opt/private.h			\
@@ -601,3 +603,5 @@ ccan-str-base32.o: $(CCANDIR)/ccan/str/base32/base32.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-utf8.o: $(CCANDIR)/ccan/utf8/utf8.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-membuf.o: $(CCANDIR)/ccan/membuf/membuf.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2442-ga8722345
+CCAN version: init-2446-g1b4ed377

ccan/ccan/crypto/shachain/tools/Makefile

@@ -5,7 +5,7 @@ CFLAGS=-Wall -Werror -O3 -I$(CCANDIR) -DSHACHAIN_BITS=48
 #CFLAGS=-Wall -Werror -g3 -I$(CCANDIR) -DSHACHAIN_BITS=48
 
 # 48 bit index for shachain.  This is what lightning uses.
-CCAN_OBJS:=ccan-str.o ccan-err.o ccan-hex.o ccan-shachain.o ccan-sha256.o ccan-rbuf.o
+CCAN_OBJS:=ccan-str.o ccan-err.o ccan-hex.o ccan-shachain.o ccan-sha256.o ccan-rbuf.o ccan-membuf.o
 
 all: shachain48
 
@@ -15,6 +15,7 @@ shachain48.o: $(CCANDIR)/ccan/crypto/shachain/shachain.h	\
 	$(CCANDIR)/ccan/str/hex/hex.h			\
 	$(CCANDIR)/ccan/str/str.h				\
 	$(CCANDIR)/ccan/err/err.h				\
+	$(CCANDIR)/ccan/membuf/membuf.h				\
 	$(CCANDIR)/ccan/rbuf/rbuf.h
 
 shachain48.o $(CCAN_OBJS): $(CCANDIR)/config.h
@@ -37,3 +38,5 @@ ccan-sha256.o: $(CCANDIR)/ccan/crypto/sha256/sha256.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-rbuf.o: $(CCANDIR)/ccan/rbuf/rbuf.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-membuf.o: $(CCANDIR)/ccan/membuf/membuf.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/ccan/crypto/shachain/tools/shachain48.c

@@ -16,9 +16,9 @@ int main(int argc, char *argv[])
 		char *p;
 
 		shachain_init(&s);
-		rbuf_init(&rbuf, STDIN_FILENO, malloc(size), size);
+		rbuf_init(&rbuf, STDIN_FILENO, malloc(size), size, membuf_realloc);
 
-		while ((p = rbuf_read_str(&rbuf, '\n', realloc)) != NULL) {
+		while ((p = rbuf_read_str(&rbuf, '\n')) != NULL) {
 			struct sha256 hash;
 			unsigned long long idx;
 

ccan/ccan/membuf/LICENSE

@@ -0,0 +1 @@
+../../licenses/BSD-MIT

ccan/ccan/membuf/_info

@@ -0,0 +1,51 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * membuf - simple linear memory buffer routines.
+ *
+ * It's common to want a linear memory buffer, where you can get memory on
+ * the end, and consume memory from the start.  The details of actually
+ * when to enlarge or move the buffer are slightly nontrivial, so they're
+ * encapsulated here.
+ *
+ * License: BSD-MIT
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ * #include <ccan/membuf/membuf.h>
+ * #include <string.h>
+ * #include <stdio.h>
+ *
+ * // Given "hello world" outputs helloworld
+ * // Given "hello there world" outputs hellothereworld
+ * int main(int argc, char *argv[])
+ * {
+ *	MEMBUF(char) charbuf;
+ *
+ *	membuf_init(&charbuf, malloc(10), 10, membuf_realloc);
+ *
+ *	for (int i = 1; i < argc; i++)
+ *		strcpy(membuf_add(&charbuf, strlen(argv[i])), argv[i]);
+ *
+ *	// This is dumb, we could do all at once, but shows technique.
+ *	while (membuf_num_elems(&charbuf) > 0)
+ *		printf("%c", *(char *)membuf_consume(&charbuf, 1));
+ *	printf("\n");
+ *	return 0;
+ * }
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/tcon\n");
+		return 0;
+	}
+
+	return 1;
+}

ccan/ccan/membuf/membuf.c

@@ -0,0 +1,60 @@
+/* MIT (BSD) license - see LICENSE file for details */
+#include <ccan/membuf/membuf.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+void membuf_init_(struct membuf *mb,
+		  void *elems, size_t num_elems, size_t elemsize,
+		  void *(*expandfn)(struct membuf *, void *, size_t))
+{
+
+	mb->start = mb->end = 0;
+	mb->max_elems = num_elems;
+	mb->elems = elems;
+	mb->expandfn = expandfn;
+}
+
+size_t membuf_prepare_space_(struct membuf *mb,
+			     size_t num_extra, size_t elemsize)
+{
+	char *oldstart = membuf_elems_(mb, elemsize);
+
+	/* Always reset in the trivial empty case. */
+	if (mb->start == mb->end)
+		mb->start = mb->end = 0;
+
+	if (membuf_num_space_(mb) >= num_extra)
+		return 0;
+
+	/* There are two ways to make space: enlarge buffer, and memmove
+	 * down.  We use a simple heuristic: if we are using less than half
+	 * the buffer, and memmove would get us sufficient space, do that. */
+	if (membuf_num_elems_(mb) <= mb->max_elems / 2
+	    && membuf_num_elems_(mb) + num_extra <= mb->max_elems) {
+		memmove(mb->elems, oldstart, (mb->end - mb->start) * elemsize);
+		mb->end -= mb->start;
+		mb->start = 0;
+	} else {
+		void *expand;
+
+		/* Since we're going to expand, at least double. */
+		if (num_extra < mb->max_elems)
+			num_extra = mb->max_elems;
+
+		expand = mb->expandfn(mb, mb->elems,
+				      (mb->max_elems + num_extra) * elemsize);
+		if (!expand) {
+			errno = ENOMEM;
+		} else {
+			mb->max_elems += num_extra;
+			mb->elems = expand;
+		}
+	}
+	return (char *)membuf_elems_(mb, elemsize) - oldstart;
+}
+
+void *membuf_realloc(struct membuf *mb, void *rawelems, size_t newsize)
+{
+	return realloc(rawelems, newsize);
+}

ccan/ccan/membuf/membuf.h

@@ -0,0 +1,236 @@
+/* MIT (BSD) license - see LICENSE file for details */
+#ifndef CCAN_MEMBUF_H
+#define CCAN_MEMBUF_H
+#include "config.h"
+#include <assert.h>
+#include <ccan/tcon/tcon.h>
+
+/**
+ * struct membuf - representation of a memory buffer.
+ *
+ * It's exposed here to allow you to embed it and so we can inline the
+ * trivial functions.
+ */
+struct membuf {
+	/* These are the cursors into buf elements */
+	size_t start;
+	size_t end;
+
+	/* Number of elements in buf */
+	size_t max_elems;
+	/* The buffer; at this low-level, untyped. */
+	char *elems;
+
+	void *(*expandfn)(struct membuf *, void *elems, size_t max_elems);
+};
+
+/**
+ * MEMBUF - declare a type-specific membuf
+ * @membertype: type for this buffer's values.
+ *
+ * You use this to create your own typed membuf.
+ *
+ * Example:
+ *	MEMBUF(int *) intp_membuf;
+ *	printf("Address of our int * membuf = %p\n", &intp_membuf);
+ */
+#define MEMBUF(membertype)					\
+	TCON_WRAP(struct membuf, membertype canary)
+
+/**
+ * membuf_init - initialize a type-specfic membuf.
+ * @mb: the MEMBUF() declared membuf.
+ * @elems: the initial buffer, if any.
+ * @max_elems: the initial space @elems, in number of elements.
+ * @expandfn: the function to enlarge buf (eg. membuf_realloc).
+ *
+ * Example:
+ *	membuf_init(&intp_membuf, NULL, 0, membuf_realloc);
+ */
+#define membuf_init(mb, elems, num, expandfn)				\
+	membuf_init_(tcon_unwrap(tcon_check_ptr((mb), canary, (elems))), \
+		     (elems), (num), tcon_sizeof((mb), canary), (expandfn))
+
+void membuf_init_(struct membuf *mb,
+		  void *elems, size_t max_elems, size_t elemsize,
+		  void *(*expandfn)(struct membuf *, void *, size_t));
+
+/**
+ * membuf_realloc - simple membuf helper to do realloc().
+ *
+ * Assumes initial buffer was NULL, or malloc().
+ */
+void *membuf_realloc(struct membuf *mb, void *rawelems, size_t newsize);
+
+/**
+ * membuf_num_elems - number of populated elements in the membuf.
+ * @mb: the MEMBUF() declared membuf.
+ */
+#define membuf_num_elems(mb) membuf_num_elems_(tcon_unwrap(mb))
+
+static inline size_t membuf_num_elems_(const struct membuf *mb)
+{
+	return mb->end - mb->start;
+}
+
+/**
+ * membuf_elems - pointer to the populated elements in the membuf.
+ * @mb: the MEMBUF() declared membuf.
+ */
+#define membuf_elems(mb)						\
+	tcon_cast_ptr(mb, canary,					\
+		      membuf_elems_(tcon_unwrap(mb), tcon_sizeof((mb), canary)))
+
+static inline void *membuf_elems_(const struct membuf *mb, size_t elemsize)
+{
+	return mb->elems + mb->start * elemsize;
+}
+
+/**
+ * membuf_consume - we've used up this many membuf_elems.
+ * @mb: the MEMBUF() declared membuf.
+ * @num: the number of elems.
+ *
+ * Returns a pointer to the old start of membuf, so you can mark consumed
+ * and actually process in a single call.
+ */
+#define membuf_consume(mb, num)						\
+	tcon_cast_ptr(mb, canary,					\
+		      membuf_consume_(tcon_unwrap(mb), (num),		\
+				      tcon_sizeof((mb), canary)))
+
+static inline void *membuf_consume_(struct membuf *mb,
+				    size_t num, size_t elemsize)
+{
+	void *old_start = membuf_elems_(mb, elemsize);
+	assert(num <= membuf_num_elems_(mb));
+	mb->start += num;
+
+	return old_start;
+}
+
+/**
+ * membuf_num_space - number of unpopulated elements at end of the membuf.
+ * @mb: the MEMBUF() declared membuf.
+ */
+#define membuf_num_space(mb) membuf_num_space_(tcon_unwrap(mb))
+
+static inline size_t membuf_num_space_(const struct membuf *mb)
+{
+	return mb->max_elems - mb->end;
+}
+
+/**
+ * membuf_space - pointer to the unpopulated elements at end of membuf.
+ * @mb: the MEMBUF() declared membuf.
+ */
+#define membuf_space(mb)						\
+	tcon_cast_ptr(mb, canary,					\
+		      membuf_space_(tcon_unwrap(mb), tcon_sizeof((mb), canary)))
+
+static inline void *membuf_space_(struct membuf *mb, size_t elemsize)
+{
+	return mb->elems + mb->end * elemsize;
+}
+
+/**
+ * membuf_added - declare that we've added this many elements.
+ * @mb: the MEMBUF() declared membuf.
+ * @n: the number of elements we added (must be < membuf_num_space()).
+ */
+#define membuf_added(mb, num)						\
+	membuf_added_(tcon_unwrap(mb), (num))
+
+static inline void membuf_added_(struct membuf *mb, size_t num)
+{
+	assert(num <= membuf_num_space_(mb));
+	mb->end += num;
+}
+
+/**
+ * membuf_prepare_space - internal routine to make sure we've got space.
+ * @mb: the MEMBUF() declared membuf.
+ * @num_extra: the minimum number of elements of space we need
+ *
+ * Usually you wouldn't call this yourself; see membuf_add() below.  But
+ * you might use this if you need to know about moves within mb->elements
+ * so you can adjust your own pointers/offsets.
+ *
+ * It returns the offset *in bytes* between the old locations and the new.
+ * This is because it may not be a whole number of elements, in the case
+ * of realloc!
+ *
+ * If you want to check for expandfn failure (which sets errno to
+ * ENOMEM), you can check if membuf_num_space() is < num_extra which will
+ * never otherwise happen.
+ */
+#define membuf_prepare_space(mb, num_extra)			\
+	membuf_prepare_space_(tcon_unwrap(mb),			\
+			      (num_extra),			\
+			      tcon_sizeof((mb), canary))
+
+size_t membuf_prepare_space_(struct membuf *mb,
+			     size_t num_extra, size_t elemsize);
+
+/**
+ * membuf_add - add to the end of the membuf.
+ * @mb: the MEMBUF() declared membuf.
+ * @num: the number of elements (must be that much space available!).
+ *
+ * Returns the pointer to the space just added, in case you want to
+ * populate it afterwards.
+ *
+ * Note that this may invalidate existing buf pointers!  If you want to
+ * avoid that, call membuf_prepare_space(mb, num) first.
+ */
+#define membuf_add(mb, num)						\
+	tcon_cast_ptr(mb, canary,					\
+		      membuf_add_(tcon_unwrap(mb), (num),		\
+				  tcon_sizeof((mb), canary)))
+
+static inline void *membuf_add_(struct membuf *mb, size_t num, size_t elemsize)
+{
+	void *oldend;
+	membuf_prepare_space_(mb, num, elemsize);
+
+	oldend = membuf_space_(mb, elemsize);
+	/* We assume expandfn succeeded. */
+	membuf_added_(mb, num);
+
+	return oldend;
+}
+
+/**
+ * membuf_unadd - remove this many added elements.
+ * @mb: the MEMBUF() declared membuf.
+ * @n: the number of elements we want to "unadd" (must be < membuf_num_elems()).
+ */
+#define membuf_unadd(mb, num)						\
+	membuf_unadd_(tcon_unwrap(mb), (num))
+
+static inline void membuf_unadd_(struct membuf *mb, size_t num)
+{
+	assert(num <= membuf_num_elems_(mb));
+	mb->end -= num;
+}
+
+/**
+ * membuf_cleanup - reset membuf, return elems array for freeing.
+ * @mb: the MEMBUF() declared membuf.
+ *
+ * The mb will be empty after this, and crash if you try to expand it.
+ * You can membuf_init() it again, however.
+ *
+ * Example:
+ *	free(membuf_cleanup(&intp_membuf));
+ */
+#define membuf_cleanup(mb) membuf_cleanup_(tcon_unwrap(mb))
+
+static inline void *membuf_cleanup_(struct membuf *mb)
+{
+	mb->start = mb->end = mb->max_elems = 0;
+	mb->expandfn = NULL;
+
+	return mb->elems;
+}
+#endif /* CCAN_MEMBUF_H */

ccan/ccan/membuf/test/run.c

@@ -0,0 +1,103 @@
+#include <ccan/membuf/membuf.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int num_realloc, num_memmove;
+
+void *memmove_test(void *dest, const void *src, size_t n);
+void *realloc_test(void *ptr, size_t size);
+
+void *memmove_test(void *dest, const void *src, size_t n)
+{
+	num_memmove++;
+	return memmove(dest, src, n);
+}
+
+void *realloc_test(void *ptr, size_t size)
+{
+	num_realloc++;
+	return realloc(ptr, size);
+}
+
+#undef memmove
+#define memmove memmove_test
+
+#undef realloc
+#define realloc realloc_test
+
+/* Include the C files directly. */
+#include <ccan/membuf/membuf.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	int prev_reallocs;
+	MEMBUF(int) intbuf;
+
+	/* This is how many tests you plan to run */
+	plan_tests(13 + 100 * 4 + 999);
+
+	membuf_init(&intbuf, malloc(10 * sizeof(int)), 10, membuf_realloc);
+	ok1(membuf_num_elems(&intbuf) == 0);
+	ok1(membuf_num_space(&intbuf) == 10);
+	ok1(membuf_space(&intbuf) != NULL);
+
+	/* Add 100 ints. */
+	for (int i = 0; i < 100; i++) {
+		memcpy(membuf_add(&intbuf, 1), &i, sizeof(i));
+		ok1(membuf_num_elems(&intbuf) == i+1);
+
+		/* Make sure membuf_elems works */
+		if (i == 0)
+			ok1(memcmp(membuf_elems(&intbuf), &i, sizeof(i)) == 0);
+	}
+
+
+	/* Pull 100 ints. */
+	for (int i = 0; i < 100; i++) {
+		ok1(memcmp(membuf_consume(&intbuf, 1), &i, sizeof(i)) == 0);
+		ok1(membuf_num_elems(&intbuf) == 100 - i - 1);
+	}
+
+	/* Should not have continuously realloced or memmoved */
+	ok1(num_realloc < 10);
+	ok1(num_memmove == 0);
+
+	/* Doing it again should give 0 reallocs. */
+	prev_reallocs = num_realloc;
+	for (int i = 0; i < 100; i++) {
+		memcpy(membuf_add(&intbuf, 1), &i, sizeof(i));
+		ok1(membuf_num_elems(&intbuf) == i+1);
+	}
+	ok1(num_realloc == prev_reallocs);
+	ok1(num_memmove == 0);
+
+	membuf_consume(&intbuf, 100);
+
+	/* Keep a single element in the queue, make sure we don't realloc! */
+	for (int i = 0; i < 1000; i++) {
+		memcpy(membuf_add(&intbuf, 1), &i, sizeof(i));
+		if (i > 0) {
+			int prev = i - 1;
+			ok1(memcmp(membuf_consume(&intbuf, 1),
+				   &prev, sizeof(prev)) == 0);
+		}
+	}
+
+	ok1(num_realloc == prev_reallocs);
+	/* Should have moved occasionally. */
+	ok1(num_memmove < 20);
+
+	ok1(membuf_consume(&intbuf, 1));
+	ok1(membuf_num_elems(&intbuf) == 0);
+
+	/* Force it to more-than-double; make sure that works! */
+	memset(membuf_add(&intbuf, 300), 0, 300*sizeof(int));
+	ok1(membuf_num_elems(&intbuf) == 300);
+
+	/* Free buffer so valgrind is happy. */
+	free(membuf_cleanup(&intbuf));
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/rbuf/_info

@@ -23,19 +23,19 @@
  *		char *word;
  *
  *		if (argv[1]) {
- *			if (!rbuf_open(&in, argv[1], NULL, 0))
+ *			if (!rbuf_open(&in, argv[1], NULL, 0, membuf_realloc))
  *				err(1, "Failed opening %s", argv[1]);
  *		} else
- *			rbuf_init(&in, STDIN_FILENO, NULL, 0);
+ *			rbuf_init(&in, STDIN_FILENO, NULL, 0, membuf_realloc);
  *
- *		while ((word = rbuf_read_str(&in, ' ', realloc)) != NULL)
+ *		while ((word = rbuf_read_str(&in, ' ')) != NULL)
  *			printf("%s*", word);
  *
  *		if (errno)
  *			err(1, "Reading %s", argv[1] ? argv[1] : "<stdin>");
  *
  *		// Free the buffer, just because we can.
- *		free(in.buf);
+ *		free(rbuf_cleanup(&in));
  *		return 0;
  *	}
  */
@@ -46,6 +46,7 @@ int main(int argc, char *argv[])
 		return 1;
 
 	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/membuf\n");
 		return 0;
 	}
 

ccan/ccan/rbuf/rbuf.c

@@ -7,21 +7,17 @@
 #include <string.h>
 #include <fcntl.h>
 
-bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max)
+bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max,
+	       void *(*expandfn)(struct membuf *, void *, size_t))
 {
 	int fd = open(name, O_RDONLY);
 	if (fd >= 0) {
-		rbuf_init(rbuf, fd, buf, buf_max);
+		rbuf_init(rbuf, fd, buf, buf_max, expandfn);
 		return true;
 	}
 	return false;
 }
 
-static size_t rem(const struct rbuf *buf)
-{
-	return buf->buf_end - (buf->start + buf->len);
-}
-
 size_t rbuf_good_size(int fd)
 {
 	struct stat st;
@@ -31,100 +27,78 @@ size_t rbuf_good_size(int fd)
 	return 4096;
 }
 
-static bool enlarge_buf(struct rbuf *buf, size_t len,
-			void *(*resize)(void *buf, size_t len))
-{
-	char *new;
-	if (!resize) {
-		errno = ENOMEM;
-		return false;
-	}
-	if (!len)
-		len = rbuf_good_size(buf->fd);
-	new = resize(buf->buf, len);
-	if (!new)
-		return false;
-	buf->start += (new - buf->buf);
-	buf->buf = new;
-	buf->buf_end = new + len;
-	return true;
-}
-
-static ssize_t get_more(struct rbuf *rbuf,
-			void *(*resize)(void *buf, size_t len))
-{
-	size_t r;
-
-	if (rbuf->start + rbuf->len == rbuf->buf_end) {
-		if (!enlarge_buf(rbuf, (rbuf->buf_end - rbuf->buf) * 2, resize))
-			return -1;
-	}
-
-	r = read(rbuf->fd, rbuf->start + rbuf->len, rem(rbuf));
-	if (r <= 0)
-		return r;
-
-	rbuf->len += r;
-	return r;
-}
-
-void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len))
+static ssize_t get_more(struct rbuf *rbuf)
 {
 	ssize_t r;
 
-	/* Move back to start of buffer if we're empty. */
-	if (!rbuf->len)
-		rbuf->start = rbuf->buf;
+	/* This is so we only call rbuf_good_size once. */
+	if (tcon_unwrap(&rbuf->m)->max_elems == 0)
+		membuf_prepare_space(&rbuf->m, rbuf_good_size(rbuf->fd));
+	else /* membuf doubles internally, so just ask for anything. */
+		membuf_prepare_space(&rbuf->m, 1);
 
-	while ((r = get_more(rbuf, resize)) != 0)
+	/* This happens if realloc fails (errno already ENOMEM) */
+	if (!membuf_num_space(&rbuf->m))
+		return -1;
+
+	r = read(rbuf->fd, membuf_space(&rbuf->m), membuf_num_space(&rbuf->m));
+	if (r <= 0)
+		return r;
+
+	membuf_add(&rbuf->m, r);
+	return r;
+}
+
+void *rbuf_fill_all(struct rbuf *rbuf)
+{
+	ssize_t r;
+
+	while ((r = get_more(rbuf)) != 0)
 		if (r < 0)
 			return NULL;
-	return rbuf->start;
+	return rbuf_start(rbuf);
 }
 
-void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len))
+void *rbuf_fill(struct rbuf *rbuf)
 {
-	if (!rbuf->len) {
-		rbuf->start = rbuf->buf;
-		if (get_more(rbuf, resize) < 0)
+	if (!rbuf_len(rbuf)) {
+		if (get_more(rbuf) < 0)
 			return NULL;
 	}
-	return rbuf->start;
+	return rbuf_start(rbuf);
 }
 
-char *rbuf_read_str(struct rbuf *rbuf, char term,
-		    void *(*resize)(void *buf, size_t len))
+char *rbuf_read_str(struct rbuf *rbuf, char term)
 {
-	char *p, *ret;
+	char *p;
 	ssize_t r = 0;
 	size_t prev = 0;
 
-	/* Move back to start of buffer if we're empty. */
-	if (!rbuf->len)
-		rbuf->start = rbuf->buf;
-
-	while (!(p = memchr(rbuf->start + prev, term, rbuf->len - prev))) {
+	while (!(p = memchr(membuf_elems(&rbuf->m) + prev,
+			    term,
+			    membuf_num_elems(&rbuf->m) - prev))) {
 		prev += r;
-		r = get_more(rbuf, resize);
+		r = get_more(rbuf);
 		if (r < 0)
 			return NULL;
 		/* EOF with no term. */
 		if (r == 0) {
+			char *ret;
+			size_t len = rbuf_len(rbuf);
+
 			/* Nothing read at all? */
-			if (!rbuf->len && term) {
+			if (!len && term) {
 				errno = 0;
 				return NULL;
 			}
+
 			/* Put term after input (get_more made room). */
-			assert(rbuf->start + rbuf->len < rbuf->buf_end);
-			rbuf->start[rbuf->len] = '\0';
-			ret = rbuf->start;
-			rbuf_consume(rbuf, rbuf->len);
+			assert(membuf_num_space(&rbuf->m) > 0);
+			ret = membuf_consume(&rbuf->m, len);
+			ret[len] = '\0';
 			return ret;
 		}
 	}
 	*p = '\0';
-	ret = rbuf->start;
-	rbuf_consume(rbuf, p + 1 - ret);
-	return ret;
+	return membuf_consume(&rbuf->m, p + 1 - (char *)rbuf_start(rbuf));
 }

ccan/ccan/rbuf/rbuf.h

@@ -5,41 +5,36 @@
 #include <limits.h> // For UCHAR_MAX
 #include <assert.h>
 #include <stdbool.h>
+#include <ccan/membuf/membuf.h>
 
 struct rbuf {
 	int fd;
-
-	/* Where to read next. */
-	char *start;
-	/* How much of what is there is valid. */
-	size_t len;
-
-	/* The entire buffer memory we have to work with. */
-	char *buf, *buf_end;
+	MEMBUF(char) m;
 };
 
 /**
  * rbuf_init - set up a buffer.
- * @buf: the struct rbuf.
+ * @rbuf: the struct rbuf.
  * @fd: the file descriptor.
  * @buf: the buffer to use.
  * @buf_max: the size of the buffer.
+ * @expandfn: usually membuf_realloc.
  */
-static inline void rbuf_init(struct rbuf *buf,
-			     int fd, char *buffer, size_t buf_max)
+static inline void rbuf_init(struct rbuf *rbuf,
+			     int fd, char *buffer, size_t buf_max,
+			     void *(*expandfn)(struct membuf *, void *, size_t))
 {
-	buf->fd = fd;
-	buf->start = buf->buf = buffer;
-	buf->len = 0;
-	buf->buf_end = buffer + buf_max;
+	rbuf->fd = fd;
+	membuf_init(&rbuf->m, buffer, buf_max, expandfn);
 }
 
 /**
  * rbuf_open - set up a buffer by opening a file.
- * @buf: the struct rbuf.
+ * @rbuf: the struct rbuf.
  * @filename: the filename
  * @buf: the buffer to use.
  * @buf_max: the size of the buffer.
+ * @expandfn: usually membuf_realloc.
  *
  * Returns false if the open fails.  If @buf_max is 0, then the buffer
  * will be resized to rbuf_good_size() on first rbuf_fill.
@@ -47,10 +42,11 @@ static inline void rbuf_init(struct rbuf *buf,
  * Example:
  *	struct rbuf in;
  *
- *	if (!rbuf_open(&in, "foo", NULL, 0))
+ *	if (!rbuf_open(&in, "foo", NULL, 0, membuf_realloc))
  *		err(1, "Could not open foo");
  */
-bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max);
+bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max,
+	       void *(*expandfn)(struct membuf *, void *, size_t));
 
 /**
  * rbuf_good_size - get a good buffer size for this fd.
@@ -62,73 +58,82 @@ size_t rbuf_good_size(int fd);
 
 /**
  * rbuf_fill - read into a buffer if it's empty.
- * @buf: the struct rbuf
- * @resize: the call to resize the buffer.
+ * @rbuf: the struct rbuf
  *
- * If @resize is needed and is NULL, or returns false, rbuf_read will
- * return NULL (with errno set to ENOMEM).  If a read fails, then NULL
- * is also returned.  If there is nothing more to read, it will return
- * NULL with errno set to 0.  Otherwise, returns @buf->start; @buf->len
- * is the valid length of the buffer.
+ * If @expandfn fails, rbuf_fill will return NULL (with errno set to ENOMEM).
+ * If a read fails, then NULL is also returned.  If there is nothing more to
+ * read, it will return NULL with errno set to 0.  Otherwise, returns first
+ * populated bytes (aka. rbuf_start()); rbuf_len() is the valid length of the
+ * buffer.
  *
  * You need to call rbuf_consume() to mark data in the buffer as
  * consumed.
  *
  * Example:
- *	while (rbuf_fill(&in, realloc)) {
- *		printf("%.*s\n", (int)in.len, in.start);
- *		rbuf_consume(&in, in.len);
+ *	while (rbuf_fill(&in)) {
+ *		printf("%.*s\n", (int)rbuf_len(&in), rbuf_start(&in));
+ *		rbuf_consume(&in, rbuf_len(&in));
  *	}
  *	if (errno)
  *		err(1, "reading foo");
  */
-void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
+void *rbuf_fill(struct rbuf *rbuf);
 
 /**
  * rbuf_consume - helper to use up data in a buffer.
- * @buf: the struct rbuf
+ * @rbuf: the struct rbuf
  * @len: the length (from @buf->start) you used.
  *
  * After rbuf_fill() you should indicate the data you've used with
  * rbuf_consume().  That way rbuf_fill() will know if it has anything
  * to do.
  */
-static inline void rbuf_consume(struct rbuf *buf, size_t len)
+static inline void rbuf_consume(struct rbuf *rbuf, size_t len)
 {
-	buf->len -= len;
-	buf->start += len;
+	membuf_consume(&rbuf->m, len);
+}
+
+/**
+ * rbuf_len - helper to determine how many bytes in rbuf
+ * @rbuf: the struct rbuf
+ */
+static inline size_t rbuf_len(const struct rbuf *rbuf)
+{
+	return membuf_num_elems(&rbuf->m);
+}
+
+/**
+ * rbuf_start - helper to get pointert to unconsumed bytes in rbuf
+ * @rbuf: the struct rbuf
+ */
+static inline char *rbuf_start(const struct rbuf *rbuf)
+{
+	return membuf_elems(&rbuf->m);
 }
 
 /**
  * rbuf_fill_all - read rest of file into a buffer.
- * @buf: the struct rbuf
- * @resize: the call to resize the buffer.
+ * @rbuf: the struct rbuf
  *
- * If @resize is needed and is NULL, or returns false, rbuf_read_all
- * will return NULL (with errno set to ENOMEM).  If a read fails,
- * then NULL is also returned, otherwise returns @buf->start.
+ * If a read or @expandfn fails then NULL returned, otherwise returns
+ * @rbuf->start.
  *
  * Example:
- *	if (!rbuf_fill_all(&in, realloc)) {
- *		if (errno)
- *			err(1, "reading foo");
- *	}
+ *	if (!rbuf_fill_all(&in))
+ *		err(1, "reading foo");
  */
-void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
+void *rbuf_fill_all(struct rbuf *rbuf);
 
 /**
  * rbuf_read_str - fill into a buffer up to a terminator, and consume string.
- * @buf: the struct rbuf
+ * @rbuf: the struct rbuf
  * @term: the character to terminate the read.
- * @resize: the call to resize the buffer.
  *
- * If @resize is needed and is NULL, or returns false, rbuf_read_str
- * will return NULL (with errno set to ENOMEM).  If a read fails,
- * then NULL is also returned, otherwise the next string.  It
- * replaces the terminator @term (if any) with NUL, otherwise NUL
+ * If a read or @expandfn fails, then NULL is returned, otherwise the next
+ * string.  It replaces the terminator @term (if any) with NUL, otherwise NUL
  * is placed after EOF.  If you need to, you can tell this has happened
- * because the nul terminator will be at @buf->start (normally it will
- * be at @buf->start - 1).
+ * because the nul terminator will be at rbuf_start(@rbuf) (normally it will be
+ * at rbuf_start(@rbuf) - 1).
  *
  * If there is nothing remaining to be read, NULL is returned with
  * errno set to 0, unless @term is NUL, in which case it returns the
@@ -140,7 +145,7 @@ void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
  * Example:
  *	char *line;
  *
- *	line = rbuf_read_str(&in, '\n', realloc);
+ *	line = rbuf_read_str(&in, '\n');
  *	if (!line) {
  *		if (errno)
  *			err(1, "reading foo");
@@ -150,7 +155,20 @@ void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
  *		printf("First line is %s\n", line);
  *
  */
-char *rbuf_read_str(struct rbuf *rbuf, char term,
-		    void *(*resize)(void *buf, size_t len));
+char *rbuf_read_str(struct rbuf *rbuf, char term);
 
+/**
+ * rbuf_cleanup - reset rbuf, return buffer for freeing.
+ * @rbuf: the struct rbuf
+ *
+ * The rbuf will be empty after this, and crash if you try to use it.
+ * You can rbuf_init() it again, however.
+ *
+ * Example:
+ *	free(rbuf_cleanup(&in));
+ */
+static inline char *rbuf_cleanup(struct rbuf *rbuf)
+{
+	return membuf_cleanup(&rbuf->m);
+}
 #endif /* CCAN_RBUF_H */

ccan/ccan/rbuf/test/run-all.c

@@ -7,6 +7,14 @@
 #include <fcntl.h>
 #include <stdlib.h>
 
+static bool test_realloc_fail;
+static void *test_realloc(struct membuf *mb, void *buf, size_t n)
+{
+	if (test_realloc_fail)
+		return NULL;
+	return realloc(buf, n);
+}
+
 int main(void)
 {
 	struct rbuf in;
@@ -24,25 +32,29 @@ int main(void)
 	}
 	close(fd);
 
-	ok1(rbuf_open(&in, "run-all-file", NULL, 0));
-	/* Can't fill without realloc. */
-	ok1(!rbuf_fill(&in, NULL));
+	ok1(rbuf_open(&in, "run-all-file", NULL, 0, test_realloc));
+	/* Can't fill if realloc fails. */
+	test_realloc_fail = true;
+	ok1(!rbuf_fill(&in));
 	ok1(errno == ENOMEM);
-	ok1(rbuf_fill(&in, realloc));
+	test_realloc_fail = false;
+	ok1(rbuf_fill(&in));
 	/* But can't load in whole file. */
-	ok1(!rbuf_fill_all(&in, NULL));
+	test_realloc_fail = true;
+	ok1(!rbuf_fill_all(&in));
 	ok1(errno == ENOMEM);
-	ok1(rbuf_fill_all(&in, realloc));
-	ok1(in.len == size);
+	test_realloc_fail = false;
+	ok1(rbuf_fill_all(&in));
+	ok1(rbuf_len(&in) == size);
 	for (i = 0; i * sizeof(buf) < size; i++) {
 		memset(buf, 0x42 + i, sizeof(buf));
-		if (memcmp(buf, in.start, sizeof(buf)) != 0) {
+		if (memcmp(buf, rbuf_start(&in), sizeof(buf)) != 0) {
 			fail("Bad buffer contents");
 			break;
 		}
 		rbuf_consume(&in, sizeof(buf));
 	}
-	free(in.buf);
+	free(rbuf_cleanup(&in));
 
 	/* This exits depending on whether all tests passed */
 	return exit_status();

ccan/ccan/rbuf/test/run-open.c

@@ -14,9 +14,9 @@ int main(void)
 	/* This is how many tests you plan to run */
 	plan_tests(5);
 
-	ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0));
+	ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0, NULL));
 	ok1(errno == ENOENT);
-	ok1(rbuf_open(&in, "test/run-open.c", NULL, 0));
+	ok1(rbuf_open(&in, "test/run-open.c", NULL, 0, NULL));
 	ok1(close(in.fd) == 0);
 	/* If this fails to stat, it should fall back */
 	ok1(rbuf_good_size(in.fd) == 4096);

ccan/ccan/rbuf/test/run-partial-read.c

@@ -27,7 +27,7 @@ int main(void)
 	int i, fd = open("test/run.c", O_RDONLY);
 
 	/* This is how many tests you plan to run */
-	plan_tests(140);
+	plan_tests(160);
 
 	/* Grab ourselves for comparison. */
 	buf[full_read(fd, buf, sizeof(buf))] = '\0';
@@ -41,26 +41,26 @@ int main(void)
 	}
 	lines[i] = NULL;
 
-	rbuf_init(&in, fd, malloc(31), 31);
+	rbuf_init(&in, fd, malloc(31), 31, membuf_realloc);
 	ok1(in.fd == fd);
-	ok1(in.buf_end - in.buf == 31);
-	p = rbuf_read_str(&in, '\n', NULL);
+	ok1(membuf_num_space(&in.m) == 31);
+	p = rbuf_read_str(&in, '\n');
 	ok1(p);
 	ok1(strcmp(p, lines[0]) == 0);
 
-	p = rbuf_read_str(&in, '\n', realloc);
+	p = rbuf_read_str(&in, '\n');
 	ok1(p);
 	ok1(strcmp(p, lines[1]) == 0);
 
 	for (i = 2; lines[i]; i++) {
-		ok1(p = rbuf_read_str(&in, '\n', realloc));
+		ok1(p = rbuf_read_str(&in, '\n'));
 		ok1(strcmp(p, lines[i]) == 0);
 	}
 
-	p = rbuf_read_str(&in, '\n', realloc);
+	p = rbuf_read_str(&in, '\n');
 	ok1(errno == 0);
 	ok1(p == NULL);
-	free(in.buf);
+	free(rbuf_cleanup(&in));
 
 	/* This exits depending on whether all tests passed */
 	return exit_status();

ccan/ccan/rbuf/test/run-term-eof.c

@@ -7,6 +7,14 @@
 #include <fcntl.h>
 #include <stdlib.h>
 
+static bool test_realloc_fail;
+static void *test_realloc(struct membuf *mb, void *buf, size_t n)
+{
+	if (test_realloc_fail)
+		return NULL;
+	return realloc(buf, n);
+}
+
 int main(void)
 {
 	struct rbuf in;
@@ -14,7 +22,7 @@ int main(void)
 	int fd = open("test/run-term-eof.c", O_RDONLY), len;
 
 	/* This is how many tests you plan to run */
-	plan_tests(6);
+	plan_tests(10);
 
 	/* Grab ourselves for comparison. */
 	len = read(fd, buf, sizeof(buf));
@@ -22,23 +30,35 @@ int main(void)
 	lseek(fd, SEEK_SET, 0);
 
 	/* We have exact-size buffer, which causes problems adding term. */
-	rbuf_init(&in, fd, malloc(len), len);
-	p = rbuf_read_str(&in, 64, NULL); /* At symbol does not appear. */
+	rbuf_init(&in, fd, malloc(len), len, test_realloc);
+	test_realloc_fail = true;
+	p = rbuf_read_str(&in, 64); /* At symbol does not appear. */
 	ok1(errno == ENOMEM);
 	ok1(!p);
 	/* This should succeed... */
-	p = rbuf_read_str(&in, 64, realloc);
+	test_realloc_fail = false;
+	p = rbuf_read_str(&in, 64);
 	ok1(p);
 	ok1(strcmp(p, buf) == 0);
-	free(in.buf);
+	ok1(rbuf_start(&in) == p + strlen(p));
+	free(rbuf_cleanup(&in));
 
 	/* Try again. */
 	lseek(fd, SEEK_SET, 0);
-	rbuf_init(&in, fd, malloc(len), len);
-	p = rbuf_read_str(&in, 64, realloc);
+	rbuf_init(&in, fd, malloc(len), len, test_realloc);
+	p = rbuf_read_str(&in, 64);
 	ok1(p);
 	ok1(strcmp(p, buf) == 0);
-	free(in.buf);
+	ok1(rbuf_start(&in) == p + strlen(p));
+	free(rbuf_cleanup(&in));
+
+	/* Normal case, we get rbuf_start after nul */
+	lseek(fd, SEEK_SET, 0);
+	rbuf_init(&in, fd, NULL, 0, test_realloc);
+	p = rbuf_read_str(&in, '^');
+	ok1(p);
+	ok1(rbuf_start(&in) == p + strlen(p) + 1);
+	free(rbuf_cleanup(&in));
 
 	return exit_status();
 }

ccan/ccan/rbuf/test/run.c

@@ -7,6 +7,14 @@
 #include <fcntl.h>
 #include <stdlib.h>
 
+static bool test_realloc_fail;
+static void *test_realloc(struct membuf *mb, void *buf, size_t n)
+{
+	if (test_realloc_fail)
+		return NULL;
+	return realloc(buf, n);
+}
+
 int main(void)
 {
 	struct rbuf in;
@@ -15,7 +23,7 @@ int main(void)
 	int i, fd = open("test/run.c", O_RDONLY), len;
 
 	/* This is how many tests you plan to run */
-	plan_tests(144);
+	plan_tests(164);
 
 	/* Grab ourselves for comparison. */
 	len = read(fd, buf, sizeof(buf));
@@ -30,39 +38,41 @@ int main(void)
 	}
 	lines[i] = NULL;
 
-	rbuf_init(&in, fd, malloc(31), 31);
+	rbuf_init(&in, fd, malloc(31), 31, test_realloc);
 	ok1(in.fd == fd);
-	ok1(in.buf_end - in.buf == 31);
-	p = rbuf_read_str(&in, '\n', NULL);
+	ok1(membuf_num_space(&in.m) == 31);
+	test_realloc_fail = true;
+	p = rbuf_read_str(&in, '\n');
 	ok1(p);
 	ok1(strcmp(p, lines[0]) == 0);
 
-	p = rbuf_read_str(&in, '\n', realloc);
+	test_realloc_fail = false;
+	p = rbuf_read_str(&in, '\n');
 	ok1(p);
 	ok1(strcmp(p, lines[1]) == 0);
 
 	for (i = 2; lines[i]; i++) {
-		ok1(p = rbuf_read_str(&in, '\n', realloc));
+		ok1(p = rbuf_read_str(&in, '\n'));
 		ok1(strcmp(p, lines[i]) == 0);
 	}
 
-	p = rbuf_read_str(&in, '\n', realloc);
+	p = rbuf_read_str(&in, '\n');
 	ok1(errno == 0);
 	ok1(p == NULL);
-	free(in.buf);
+	free(rbuf_cleanup(&in));
 
 	/* Another way of reading the entire (text) file. */
 	lseek(fd, SEEK_SET, 0);
-	rbuf_init(&in, fd, NULL, 0);
-	p = rbuf_read_str(&in, 0, realloc);
+	rbuf_init(&in, fd, NULL, 0, test_realloc);
+	p = rbuf_read_str(&in, 0);
 	ok1(p);
 	ok1(strlen(p) == len);
 
 	close(fd);
-	p = rbuf_read_str(&in, 0, realloc);
+	p = rbuf_read_str(&in, 0);
 	ok1(errno == EBADF);
 	ok1(!p);
-	free(in.buf);
+	free(rbuf_cleanup(&in));
 
 	return exit_status();
 }

connectd/tor_autoservice.c

@@ -26,7 +26,7 @@
 #define MAX_TOR_ONION_V2_ADDR_LEN 16
 #define MAX_TOR_ONION_V3_ADDR_LEN 56
 
-static void *buf_resize(void *buf, size_t len)
+static void *buf_resize(struct membuf *mb, void *buf, size_t len)
 {
 	tal_resize(&buf, len);
 	return buf;
@@ -49,7 +49,7 @@ static char *tor_response_line(struct rbuf *rbuf)
 {
 	char *line;
 
-	while ((line = rbuf_read_str(rbuf, '\n', buf_resize)) != NULL) {
+	while ((line = rbuf_read_str(rbuf, '\n')) != NULL) {
 		status_io(LOG_IO_IN, "torcontrol", line, strlen(line));
 
 		/* Weird response */
@@ -222,7 +222,7 @@ struct wireaddr *tor_autoservice(const tal_t *ctx,
 		err(1, "Connecting stream socket to Tor service");
 
 	buffer = tal_arr(tmpctx, char, rbuf_good_size(fd));
-	rbuf_init(&rbuf, fd, buffer, tal_count(buffer));
+	rbuf_init(&rbuf, fd, buffer, tal_count(buffer), buf_resize);
 
 	negotiate_auth(&rbuf, tor_password);
 	onion = make_onion(ctx, &rbuf, laddr);