diff --git a/.travis.yml b/.travis.yml index f2b2a74b9..b703ea33c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ notifications: email: false before_install: - sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8 + sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8 gettext env: - ARCH=64 SOURCE_CHECK_ONLY=true COPTFLAGS="-O3" diff --git a/.travis/build.sh b/.travis/build.sh index f49381da9..63f67c42f 100755 --- a/.travis/build.sh +++ b/.travis/build.sh @@ -29,8 +29,7 @@ if [ ! -f dependencies/bin/bitcoind ]; then fi pyenv global 3.7.1 -pip3 install --user --quiet mako -pip3 install --user --quiet -r tests/requirements.txt -r doc/requirements.txt +pip3 install --user --quiet -r requirements.txt -r tests/requirements.txt -r doc/requirements.txt pip3 install --quiet \ pytest-test-groups==1.0.3 diff --git a/configure b/configure index 3d5ae56d5..3e1efd444 100755 --- a/configure +++ b/configure @@ -259,6 +259,20 @@ int main(void) return 0; } /*END*/ +var=HAVE_SQLITE3 +desc=sqlite3 +style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE +link=-lsqlite3 +code= +#include +#include + +int main(void) +{ + printf("%p\n", sqlite3_prepare_v2); + return 0; +} +/*END*/ var=HAVE_GCC desc=compiler is GCC style=OUTSIDE_MAIN diff --git a/devtools/sql-rewrite.py b/devtools/sql-rewrite.py new file mode 100755 index 000000000..db5cb03ae --- /dev/null +++ b/devtools/sql-rewrite.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +from mako.template import Template + +import sys + + +class Sqlite3Rewriter(object): + def rewrite(self, query): + return query + + +rewriters = { + "sqlite3": Sqlite3Rewriter(), +} + +template = Template("""#ifndef LIGHTNINGD_WALLET_GEN_DB_${f.upper()} +#define LIGHTNINGD_WALLET_GEN_DB_${f.upper()} + +#include +#include + +#if HAVE_${f.upper()} + +struct db_query db_${f}_queries[] = { + +% for elem in queries: + { + .name = "${elem['name']}", + .query = "${elem['query']}", + .placeholders = ${elem['placeholders']} + }, +% endfor +}; + +#define DB_${f.upper()}_QUERY_COUNT ${len(queries)} + +#endif /* HAVE_${f.upper()} */ + +#endif /* LIGHTNINGD_WALLET_GEN_DB_${f.upper()} */ +""") + + +def extract_queries(pofile): + # Given a po-file, extract all queries and their associated names, and + # return them as a list. + + def chunk(pofile): + # Chunk a given file into chunks separated by an empty line + with open(pofile, 'r') as f: + chunk = [] + for line in f: + line = line.strip() + if line.strip() == "": + yield chunk + chunk = [] + else: + chunk.append(line.strip()) + if chunk != []: + yield chunk + + queries = [] + for c in chunk(pofile): + name = c[0][3:] + + # Skip other comments + i = 1 + while c[i][0] == '#': + i += 1 + + # Strip header and surrounding quotes + query = c[i][7:][:-1] + + queries.append({ + 'name': name, + 'query': query, + 'placeholders': query.count('?'), + 'readonly': "true" if query.upper().startswith("SELECT") else "false", + }) + return queries + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage:\n\t{} ".format(sys.argv[0])) + sys.exit(1) + + dialect = sys.argv[2] + + if dialect not in rewriters: + print("Unknown dialect {}. The following are available: {}".format( + dialect, + ", ".join(rewriters.keys()) + )) + sys.exit(1) + + rewriter = rewriters[dialect] + + queries = extract_queries(sys.argv[1]) + queries = rewriter.rewrite(queries) + + print(template.render(f=dialect, queries=queries)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..5cca28d2b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +sqlparse==0.3.0 +mako==1.0.14 +mrkd==0.1.5 diff --git a/wallet/Makefile b/wallet/Makefile index 6f5c9e28e..53993c5b4 100644 --- a/wallet/Makefile +++ b/wallet/Makefile @@ -11,7 +11,10 @@ WALLET_LIB_SRC := \ wallet/wallet.c \ wallet/walletrpc.c -WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o) +WALLET_DB_DRIVERS := \ + wallet/db_sqlite3.c + +WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o) $(WALLET_DB_DRIVERS:.c=.o) WALLET_LIB_HEADERS := $(WALLET_LIB_SRC:.c=.h) # Make sure these depend on everything. @@ -26,7 +29,25 @@ check-source-bolt: $(WALLET_LIB_SRC:%=bolt-check/%) $(WALLET_LIB_HEADERS:%=bolt- clean: wallet-clean +wallet/db_sqlite3.c: wallet/gen_db_sqlite3.c + +# The following files contain SQL-annotated statements that we need to extact +SQL_FILES := \ + wallet/db.c \ + wallet/invoices.c \ + wallet/wallet.c \ + wallet/test/run-db.c \ + wallet/test/run-wallet.c \ + +wallet/statements.po: $(SQL_FILES) + xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(SQL_FILES) + +wallet/gen_db_sqlite3.c: wallet/statements.po devtools/sql-rewrite.py + devtools/sql-rewrite.py wallet/statements.po sqlite3 > wallet/gen_db_sqlite3.c + wallet-clean: $(RM) $(WALLET_LIB_OBJS) + $(RM) wallet/statements.po + $(RM) wallet/gen_db_sqlite3.c include wallet/test/Makefile diff --git a/wallet/db.c b/wallet/db.c index c64841e1b..5f83a7763 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -9,6 +9,7 @@ #include #include #include +#include #define DB_FILE "lightningd.sqlite3" #define NSEC_IN_SEC 1000000000 diff --git a/wallet/db.h b/wallet/db.h index 973d942e8..5c5d8c7b0 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/wallet/db_common.h b/wallet/db_common.h new file mode 100644 index 000000000..ae621b8ed --- /dev/null +++ b/wallet/db_common.h @@ -0,0 +1,56 @@ +#ifndef LIGHTNING_WALLET_DB_COMMON_H +#define LIGHTNING_WALLET_DB_COMMON_H +#include "config.h" +#include +#include + +struct db_query { + const char *name; + const char *query; + + /* How many placeholders are in the query (and how many will we have + to allocate when instantiating this query)? */ + size_t placeholders; +}; + +struct db_config { + const char *name; + struct db_query *queries; + size_t num_queries; +}; + +enum db_binding_type { + DB_BINDING_UNINITIALIZED = 0, + DB_BINDING_NULL, + DB_BINDING_BLOB, + DB_BINDING_TEXT, + DB_BINDING_UINT64, + DB_BINDING_INT, +}; + +struct db_binding { + enum db_binding_type type; + union { + int i; + u64 u64; + const char* text; + const u8 *blob; + } v; + size_t len; +}; + +struct db_stmt { + /* Which SQL statement are we trying to execute? */ + struct db_query *query; + + /* Which parameters are we binding to the statement? */ + struct db_binding *bindings; + + /* Where are we calling this statement from? */ + const char *location; +}; + +/* Provide a way for DB backends to register themselves */ +AUTODATA_TYPE(db_backends, struct db_config); + +#endif /* LIGHTNING_WALLET_DB_COMMON_H */ diff --git a/wallet/db_sqlite3.c b/wallet/db_sqlite3.c new file mode 100644 index 000000000..fd5e98090 --- /dev/null +++ b/wallet/db_sqlite3.c @@ -0,0 +1,13 @@ +#include +#include "gen_db_sqlite3.c" +#if HAVE_SQLITE3 + +struct db_config db_sqlite3_config = { + .name = "sqlite3", + .queries = db_sqlite3_queries, + .num_queries = DB_SQLITE3_QUERY_COUNT, +}; + +AUTODATA(db_backends, &db_sqlite3_config); + +#endif diff --git a/wallet/wallet.c b/wallet/wallet.c index e3181632b..ce6ef023f 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -14,6 +14,7 @@ #include #include #include +#include #define SQLITE_MAX_UINT 0x7FFFFFFFFFFFFFFF #define DIRECTION_INCOMING 0