db: vacuum after a db upgrade.

This is particularly useful after our recent field deletion:

before: 362,573,824 bytes
after: 124,190,720 bytes

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Changelog-Changed: db: removal of old HTLC information and vacuuming shrinks large lightningd.sqlite3 by a factor of 2-3.

wallet/db.c

@@ -1204,7 +1204,7 @@ static int db_get_version(struct db *db)
 /**
  * db_migrate - Apply all remaining migrations from the current version
  */
-static void db_migrate(struct lightningd *ld, struct db *db,
+static bool db_migrate(struct lightningd *ld, struct db *db,
 		       const struct ext_key *bip32_base)
 {
 	/* Attempt to read the version from the database */
@@ -1254,6 +1254,8 @@ static void db_migrate(struct lightningd *ld, struct db *db,
 		db_exec_prepared_v2(stmt);
 		tal_free(stmt);
 	}
+
+	return current != orig;
 }
 
 u32 db_data_version_get(struct db *db)
@@ -1272,14 +1274,22 @@ struct db *db_setup(const tal_t *ctx, struct lightningd *ld,
 		    const struct ext_key *bip32_base)
 {
 	struct db *db = db_open(ctx, ld->wallet_dsn);
+	bool migrated;
 	db->log = new_log(db, ld->log_book, NULL, "database");
 
 	db_begin_transaction(db);
 
-	db_migrate(ld, db, bip32_base);
+	migrated = db_migrate(ld, db, bip32_base);
 
 	db->data_version = db_data_version_get(db);
 	db_commit_transaction(db);
+
+	/* This needs to be done outside a transaction, apparently.
+	 * It's a good idea to do this every so often, and on db
+	 * upgrade is a reasonable time. */
+	if (migrated && !db->config->vacuum_fn(db))
+		db_fatal("Error vacuuming db: %s", db->error);
+
 	return db;
 }
 

wallet/db_common.h

@@ -140,7 +140,7 @@ struct db_config {
 	bool (*setup_fn)(struct db *db);
 	void (*teardown_fn)(struct db *db);
 
-	u32 (*version)(struct db *db);
+	bool (*vacuum_fn)(struct db *db);
 };
 
 /* Provide a way for DB backends to register themselves */

wallet/db_postgres.c

@@ -251,6 +251,20 @@ static void db_postgres_teardown(struct db *db)
 {
 }
 
+static bool db_postgres_vacuum(struct db *db)
+{
+	PGresult *res;
+	res = PQexec(db->conn, "VACUUM FULL;");
+	if (PQresultStatus(res) != PGRES_COMMAND_OK) {
+		db->error = tal_fmt(db, "BEGIN command failed: %s",
+				    PQerrorMessage(db->conn));
+		PQclear(res);
+		return false;
+	}
+	PQclear(res);
+	return true;
+}
+
 struct db_config db_postgres_config = {
     .name = "postgres",
     .queries = db_postgres_queries,
@@ -273,6 +287,7 @@ struct db_config db_postgres_config = {
     .count_changes_fn = db_postgres_count_changes,
     .setup_fn = db_postgres_setup,
     .teardown_fn = db_postgres_teardown,
+    .vacuum_fn = db_postgres_vacuum,
 };
 
 AUTODATA(db_backends, &db_postgres_config);

wallet/db_sqlite3.c

@@ -234,6 +234,20 @@ static u64 db_sqlite3_last_insert_id(struct db_stmt *stmt)
 	return sqlite3_last_insert_rowid(s);
 }
 
+static bool db_sqlite3_vacuum(struct db *db)
+{
+	int err;
+	sqlite3_stmt *stmt;
+
+	sqlite3_prepare_v2(db->conn, "VACUUM;", -1, &stmt, NULL);
+	err = sqlite3_step(stmt);
+	if (err != SQLITE_DONE)
+		db->error = tal_fmt(db, "%s", sqlite3_errmsg(db->conn));
+	sqlite3_finalize(stmt);
+
+	return err == SQLITE_DONE;
+}
+
 struct db_config db_sqlite3_config = {
 	.name = "sqlite3",
 	.queries = db_sqlite3_queries,
@@ -256,6 +270,8 @@ struct db_config db_sqlite3_config = {
 	.count_changes_fn = &db_sqlite3_count_changes,
 	.setup_fn = &db_sqlite3_setup,
 	.teardown_fn = &db_sqlite3_close,
+
+	.vacuum_fn = db_sqlite3_vacuum,
 };
 
 AUTODATA(db_backends, &db_sqlite3_config);