wallet: Add tooling to extract SQL queries and generate driver info

This is the counterpart of the annotations we did in the last few commits. It
extracts queries, passes them through a driver-specific query rewriter and
dumps them into a driver-specific query-list, along with some metadata to
facilitate processing later on. The generated query list is then registered as
a `db_config` and will be loaded by the driver upon instantiation.

Signed-off-by: Christian Decker <decker.christian@gmail.com>

.travis.yml

@@ -5,7 +5,7 @@ notifications:
   email: false
 
 before_install:
-  sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8
+  sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8 gettext
 
 env:
   - ARCH=64 SOURCE_CHECK_ONLY=true COPTFLAGS="-O3"

.travis/build.sh

@@ -29,8 +29,7 @@ if [ ! -f dependencies/bin/bitcoind ]; then
 fi
 
 pyenv global 3.7.1
-pip3 install --user --quiet mako
-pip3 install --user --quiet -r tests/requirements.txt -r doc/requirements.txt
+pip3 install --user --quiet -r requirements.txt -r tests/requirements.txt -r doc/requirements.txt
 pip3 install --quiet \
      pytest-test-groups==1.0.3
 

configure

@@ -259,6 +259,20 @@ int main(void)
 	return 0;
 }
 /*END*/
+var=HAVE_SQLITE3
+desc=sqlite3
+style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE
+link=-lsqlite3
+code=
+#include <sqlite3.h>
+#include <stdio.h>
+
+int main(void)
+{
+	printf("%p\n", sqlite3_prepare_v2);
+	return 0;
+}
+/*END*/
 var=HAVE_GCC
 desc=compiler is GCC
 style=OUTSIDE_MAIN

devtools/sql-rewrite.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from mako.template import Template
+
+import sys
+
+
+class Sqlite3Rewriter(object):
+    def rewrite(self, query):
+        return query
+
+
+rewriters = {
+    "sqlite3": Sqlite3Rewriter(),
+}
+
+template = Template("""#ifndef LIGHTNINGD_WALLET_GEN_DB_${f.upper()}
+#define LIGHTNINGD_WALLET_GEN_DB_${f.upper()}
+
+#include <config.h>
+#include <wallet/db_common.h>
+
+#if HAVE_${f.upper()}
+
+struct db_query db_${f}_queries[] = {
+
+% for elem in queries:
+    {
+         .name = "${elem['name']}",
+         .query = "${elem['query']}",
+         .placeholders = ${elem['placeholders']}
+    },
+% endfor
+};
+
+#define DB_${f.upper()}_QUERY_COUNT ${len(queries)}
+
+#endif /* HAVE_${f.upper()} */
+
+#endif /* LIGHTNINGD_WALLET_GEN_DB_${f.upper()} */
+""")
+
+
+def extract_queries(pofile):
+    # Given a po-file, extract all queries and their associated names, and
+    # return them as a list.
+
+    def chunk(pofile):
+        # Chunk a given file into chunks separated by an empty line
+        with open(pofile, 'r') as f:
+            chunk = []
+            for line in f:
+                line = line.strip()
+                if line.strip() == "":
+                    yield chunk
+                    chunk = []
+                else:
+                    chunk.append(line.strip())
+            if chunk != []:
+                yield chunk
+
+    queries = []
+    for c in chunk(pofile):
+        name = c[0][3:]
+
+        # Skip other comments
+        i = 1
+        while c[i][0] == '#':
+            i += 1
+
+        # Strip header and surrounding quotes
+        query = c[i][7:][:-1]
+
+        queries.append({
+            'name': name,
+            'query': query,
+            'placeholders': query.count('?'),
+            'readonly': "true" if query.upper().startswith("SELECT") else "false",
+        })
+    return queries
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage:\n\t{} <statements.po-file> <output-dialect>".format(sys.argv[0]))
+        sys.exit(1)
+
+    dialect = sys.argv[2]
+
+    if dialect not in rewriters:
+        print("Unknown dialect {}. The following are available: {}".format(
+            dialect,
+            ", ".join(rewriters.keys())
+        ))
+        sys.exit(1)
+
+    rewriter = rewriters[dialect]
+
+    queries = extract_queries(sys.argv[1])
+    queries = rewriter.rewrite(queries)
+
+    print(template.render(f=dialect, queries=queries))

requirements.txt

@@ -0,0 +1,3 @@
+sqlparse==0.3.0
+mako==1.0.14
+mrkd==0.1.5

wallet/Makefile

@@ -11,7 +11,10 @@ WALLET_LIB_SRC :=		\
 	wallet/wallet.c		\
 	wallet/walletrpc.c
 
-WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o)
+WALLET_DB_DRIVERS :=		\
+	wallet/db_sqlite3.c
+
+WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o) $(WALLET_DB_DRIVERS:.c=.o)
 WALLET_LIB_HEADERS := $(WALLET_LIB_SRC:.c=.h)
 
 # Make sure these depend on everything.
@@ -26,7 +29,25 @@ check-source-bolt: $(WALLET_LIB_SRC:%=bolt-check/%) $(WALLET_LIB_HEADERS:%=bolt-
 
 clean: wallet-clean
 
+wallet/db_sqlite3.c: wallet/gen_db_sqlite3.c
+
+# The following files contain SQL-annotated statements that we need to extact
+SQL_FILES := 				\
+	wallet/db.c 			\
+	wallet/invoices.c		\
+	wallet/wallet.c 		\
+	wallet/test/run-db.c		\
+	wallet/test/run-wallet.c	\
+
+wallet/statements.po: $(SQL_FILES)
+	xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(SQL_FILES)
+
+wallet/gen_db_sqlite3.c:  wallet/statements.po devtools/sql-rewrite.py
+	 devtools/sql-rewrite.py wallet/statements.po sqlite3 > wallet/gen_db_sqlite3.c
+
 wallet-clean:
 	$(RM) $(WALLET_LIB_OBJS)
+	$(RM) wallet/statements.po
+	$(RM) wallet/gen_db_sqlite3.c
 
 include wallet/test/Makefile

wallet/db.c

@@ -9,6 +9,7 @@
 #include <lightningd/lightningd.h>
 #include <lightningd/log.h>
 #include <lightningd/plugin_hook.h>
+#include <wallet/db_common.h>
 
 #define DB_FILE "lightningd.sqlite3"
 #define NSEC_IN_SEC 1000000000

wallet/db.h

@@ -6,6 +6,7 @@
 #include <bitcoin/pubkey.h>
 #include <bitcoin/short_channel_id.h>
 #include <bitcoin/tx.h>
+#include <ccan/autodata/autodata.h>
 #include <ccan/short_types/short_types.h>
 #include <ccan/tal/tal.h>
 #include <ccan/time/time.h>

wallet/db_common.h

@@ -0,0 +1,56 @@
+#ifndef LIGHTNING_WALLET_DB_COMMON_H
+#define LIGHTNING_WALLET_DB_COMMON_H
+#include "config.h"
+#include <ccan/autodata/autodata.h>
+#include <ccan/short_types/short_types.h>
+
+struct db_query {
+	const char *name;
+	const char *query;
+
+	/* How many placeholders are in the query (and how many will we have
+	   to allocate when instantiating this query)? */
+	   size_t placeholders;
+};
+
+struct db_config {
+	const char *name;
+	struct db_query *queries;
+	size_t num_queries;
+};
+
+enum db_binding_type {
+	DB_BINDING_UNINITIALIZED = 0,
+	DB_BINDING_NULL,
+	DB_BINDING_BLOB,
+	DB_BINDING_TEXT,
+	DB_BINDING_UINT64,
+	DB_BINDING_INT,
+};
+
+struct db_binding {
+	enum db_binding_type type;
+	union {
+		int i;
+		u64 u64;
+		const char* text;
+		const u8 *blob;
+	} v;
+	size_t len;
+};
+
+struct db_stmt {
+	/* Which SQL statement are we trying to execute? */
+	struct db_query *query;
+
+	/* Which parameters are we binding to the statement? */
+	struct db_binding *bindings;
+
+	/* Where are we calling this statement from? */
+	const char *location;
+};
+
+/* Provide a way for DB backends to register themselves */
+AUTODATA_TYPE(db_backends, struct db_config);
+
+#endif /* LIGHTNING_WALLET_DB_COMMON_H */

wallet/db_sqlite3.c

@@ -0,0 +1,13 @@
+#include <wallet/db_common.h>
+#include "gen_db_sqlite3.c"
+#if HAVE_SQLITE3
+
+struct db_config db_sqlite3_config = {
+	.name = "sqlite3",
+	.queries = db_sqlite3_queries,
+	.num_queries = DB_SQLITE3_QUERY_COUNT,
+};
+
+AUTODATA(db_backends, &db_sqlite3_config);
+
+#endif

wallet/wallet.c

@@ -14,6 +14,7 @@
 #include <lightningd/peer_htlcs.h>
 #include <onchaind/gen_onchain_wire.h>
 #include <string.h>
+#include <wallet/db_common.h>
 
 #define SQLITE_MAX_UINT 0x7FFFFFFFFFFFFFFF
 #define DIRECTION_INCOMING 0