multi: add sqlite backend option

In this diff, we expose the option to use sqlite as a db backend.

config.go

@@ -1404,12 +1404,18 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser,
 		)
 	}
 
+	towerDir := filepath.Join(
+		cfg.Watchtower.TowerDir,
+		cfg.registeredChains.PrimaryChain().String(),
+		lncfg.NormalizeNetwork(cfg.ActiveNetParams.Name),
+	)
+
 	// Create the lnd directory and all other sub-directories if they don't
 	// already exist. This makes sure that directory trees are also created
 	// for files that point to outside the lnddir.
 	dirs := []string{
 		lndDir, cfg.DataDir, cfg.networkDir,
-		cfg.LetsEncryptDir, cfg.Watchtower.TowerDir,
+		cfg.LetsEncryptDir, towerDir, cfg.graphDatabaseDir(),
 		filepath.Dir(cfg.TLSCertPath), filepath.Dir(cfg.TLSKeyPath),
 		filepath.Dir(cfg.AdminMacPath), filepath.Dir(cfg.ReadMacPath),
 		filepath.Dir(cfg.InvoiceMacPath),
@@ -1616,6 +1622,47 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser,
 		cfg.DB.Bolt.NoFreelistSync = !cfg.SyncFreelist
 	}
 
+	// Parse any extra sqlite pragma options that may have been provided
+	// to determine if they override any of the defaults that we will
+	// otherwise add.
+	var (
+		defaultSynchronous = true
+		defaultAutoVacuum  = true
+		defaultFullfsync   = true
+	)
+	for _, option := range cfg.DB.Sqlite.PragmaOptions {
+		switch {
+		case strings.HasPrefix(option, "synchronous="):
+			defaultSynchronous = false
+
+		case strings.HasPrefix(option, "auto_vacuum="):
+			defaultAutoVacuum = false
+
+		case strings.HasPrefix(option, "fullfsync="):
+			defaultFullfsync = false
+
+		default:
+		}
+	}
+
+	if defaultSynchronous {
+		cfg.DB.Sqlite.PragmaOptions = append(
+			cfg.DB.Sqlite.PragmaOptions, "synchronous=full",
+		)
+	}
+
+	if defaultAutoVacuum {
+		cfg.DB.Sqlite.PragmaOptions = append(
+			cfg.DB.Sqlite.PragmaOptions, "auto_vacuum=incremental",
+		)
+	}
+
+	if defaultFullfsync {
+		cfg.DB.Sqlite.PragmaOptions = append(
+			cfg.DB.Sqlite.PragmaOptions, "fullfsync=true",
+		)
+	}
+
 	// Ensure that the user hasn't chosen a remote-max-htlc value greater
 	// than the protocol maximum.
 	maxRemoteHtlcs := uint16(input.MaxHTLCNumber / 2)

config_builder.go

@@ -257,7 +257,7 @@ func (d *DefaultWalletImpl) BuildWalletConfig(ctx context.Context,
 	var neutrinoCS *neutrino.ChainService
 	if mainChain.Node == "neutrino" {
 		neutrinoBackend, neutrinoCleanUp, err := initNeutrinoBackend(
-			d.cfg, mainChain.ChainDir, blockCache,
+			ctx, d.cfg, mainChain.ChainDir, blockCache,
 		)
 		if err != nil {
 			err := fmt.Errorf("unable to initialize neutrino "+
@@ -1173,7 +1173,7 @@ func importWatchOnlyAccounts(wallet *wallet.Wallet,
 
 // initNeutrinoBackend inits a new instance of the neutrino light client
 // backend given a target chain directory to store the chain state.
-func initNeutrinoBackend(cfg *Config, chainDir string,
+func initNeutrinoBackend(ctx context.Context, cfg *Config, chainDir string,
 	blockCache *blockcache.BlockCache) (*neutrino.ChainService,
 	func(), error) {
 
@@ -1200,13 +1200,26 @@ func initNeutrinoBackend(cfg *Config, chainDir string,
 		return nil, nil, err
 	}
 
-	dbName := filepath.Join(dbPath, "neutrino.db")
-	db, err := walletdb.Create(
-		"bdb", dbName, !cfg.SyncFreelist, cfg.DB.Bolt.DBTimeout,
+	var (
+		db  walletdb.DB
+		err error
 	)
+	switch {
+	case cfg.DB.Backend == kvdb.SqliteBackendName:
+		db, err = kvdb.Open(
+			kvdb.SqliteBackendName, ctx, cfg.DB.Sqlite, dbPath,
+			lncfg.SqliteNeutrinoDBName, lncfg.NSNeutrinoDB,
+		)
+
+	default:
+		dbName := filepath.Join(dbPath, "neutrino.db")
+		db, err = walletdb.Create(
+			"bdb", dbName, !cfg.SyncFreelist, cfg.DB.Bolt.DBTimeout,
+		)
+	}
 	if err != nil {
-		return nil, nil, fmt.Errorf("unable to create neutrino "+
-			"database: %v", err)
+		return nil, nil, fmt.Errorf("unable to create "+
+			"neutrino database: %v", err)
 	}
 
 	headerStateAssertion, err := parseHeaderStateAssertion(

lncfg/db.go

@@ -9,6 +9,7 @@ import (
 	"github.com/lightningnetwork/lnd/kvdb/etcd"
 	"github.com/lightningnetwork/lnd/kvdb/postgres"
 	"github.com/lightningnetwork/lnd/kvdb/sqlbase"
+	"github.com/lightningnetwork/lnd/kvdb/sqlite"
 	"github.com/lightningnetwork/lnd/lnwallet/btcwallet"
 )
 
@@ -20,12 +21,21 @@ const (
 	TowerServerDBName = "watchtower.db"
 	WalletDBName      = "wallet.db"
 
+	SqliteChannelDBName  = "channel.sqlite"
+	SqliteChainDBName    = "chain.sqlite"
+	SqliteNeutrinoDBName = "neutrino.sqlite"
+	SqliteTowerDBName    = "watchtower.sqlite"
+
 	BoltBackend                = "bolt"
 	EtcdBackend                = "etcd"
 	PostgresBackend            = "postgres"
+	SqliteBackend              = "sqlite"
 	DefaultBatchCommitInterval = 500 * time.Millisecond
 
 	defaultPostgresMaxConnections = 50
+	defaultSqliteMaxConnections   = 2
+
+	defaultSqliteBusyTimeout = 5 * time.Second
 
 	// NSChannelDB is the namespace name that we use for the combined graph
 	// and channel state DB.
@@ -48,6 +58,9 @@ const (
 
 	// NSWalletDB is the namespace name that we use for the wallet DB.
 	NSWalletDB = "walletdb"
+
+	// NSNeutrinoDB is the namespace name that we use for the neutrino DB.
+	NSNeutrinoDB = "neutrinodb"
 )
 
 // DB holds database configuration for LND.
@@ -64,6 +77,8 @@ type DB struct {
 
 	Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."`
 
+	Sqlite *sqlite.Config `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."`
+
 	NoGraphCache bool `long:"no-graph-cache" description:"Don't use the in-memory graph cache for path finding. Much slower but uses less RAM. Can only be used with a bolt database backend."`
 
 	PruneRevocation bool `long:"prune-revocation" description:"Run the optional migration that prunes the revocation logs to save disk space."`
@@ -86,13 +101,17 @@ func DefaultDB() *DB {
 		Postgres: &postgres.Config{
 			MaxConnections: defaultPostgresMaxConnections,
 		},
+		Sqlite: &sqlite.Config{
+			MaxConnections: defaultSqliteMaxConnections,
+			BusyTimeout:    defaultSqliteBusyTimeout,
+		},
 	}
 }
 
 // Validate validates the DB config.
 func (db *DB) Validate() error {
 	switch db.Backend {
-	case BoltBackend:
+	case BoltBackend, SqliteBackend:
 	case PostgresBackend:
 		if db.Postgres.Dsn == "" {
 			return fmt.Errorf("postgres dsn must be set")
@@ -104,8 +123,9 @@ func (db *DB) Validate() error {
 		}
 
 	default:
-		return fmt.Errorf("unknown backend, must be either '%v' or "+
-			"'%v'", BoltBackend, EtcdBackend)
+		return fmt.Errorf("unknown backend, must be either '%v', "+
+			"'%v', '%v' or '%v'", BoltBackend, EtcdBackend,
+			PostgresBackend, SqliteBackend)
 	}
 
 	// The path finding uses a manual read transaction that's open for a
@@ -144,6 +164,9 @@ func (db *DB) Init(ctx context.Context, dbPath string) error {
 
 	case db.Backend == PostgresBackend:
 		sqlbase.Init(db.Postgres.MaxConnections)
+
+	case db.Backend == SqliteBackend:
+		sqlbase.Init(db.Sqlite.MaxConnections)
 	}
 
 	return nil
@@ -187,7 +210,7 @@ type DatabaseBackends struct {
 	WalletDB btcwallet.LoaderOption
 
 	// Remote indicates whether the database backends are remote, possibly
-	// replicated instances or local bbolt backed databases.
+	// replicated instances or local bbolt or sqlite backed databases.
 	Remote bool
 
 	// CloseFuncs is a map of close functions for each of the initialized
@@ -291,6 +314,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSWalletDB] = etcdWalletBackend.Close
 
 		returnEarly = false
+
 		return &DatabaseBackends{
 			GraphDB:       etcdBackend,
 			ChanStateDB:   etcdBackend,
@@ -373,6 +397,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSWalletDB] = postgresWalletBackend.Close
 
 		returnEarly = false
+
 		return &DatabaseBackends{
 			GraphDB:       postgresBackend,
 			ChanStateDB:   postgresBackend,
@@ -392,6 +417,98 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 			Remote:     true,
 			CloseFuncs: closeFuncs,
 		}, nil
+
+	case SqliteBackend:
+		// Note that for sqlite, we put kv tables for the channel.db,
+		// wtclient.db and sphinxreplay.db all in the channel.sqlite db.
+		// The tables for wallet.db and macaroon.db are in the
+		// chain.sqlite db and watchtower.db tables are in the
+		// watchtower.sqlite db. The reason for the multiple sqlite dbs
+		// is twofold. The first reason is that it maintains the file
+		// structure that users are used to. The second reason is the
+		// fact that sqlite only supports one writer at a time which
+		// would cause deadlocks in the code due to the wallet db often
+		// being accessed during a write to another db.
+		sqliteBackend, err := kvdb.Open(
+			kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath,
+			SqliteChannelDBName, NSChannelDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening sqlite graph "+
+				"DB: %v", err)
+		}
+		closeFuncs[NSChannelDB] = sqliteBackend.Close
+
+		sqliteMacaroonBackend, err := kvdb.Open(
+			kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath,
+			SqliteChainDBName, NSMacaroonDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening sqlite "+
+				"macaroon DB: %v", err)
+		}
+		closeFuncs[NSMacaroonDB] = sqliteMacaroonBackend.Close
+
+		sqliteDecayedLogBackend, err := kvdb.Open(
+			kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath,
+			SqliteChannelDBName, NSDecayedLogDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening sqlite decayed "+
+				"log DB: %v", err)
+		}
+		closeFuncs[NSDecayedLogDB] = sqliteDecayedLogBackend.Close
+
+		sqliteTowerClientBackend, err := kvdb.Open(
+			kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath,
+			SqliteChannelDBName, NSTowerClientDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening sqlite tower "+
+				"client DB: %v", err)
+		}
+		closeFuncs[NSTowerClientDB] = sqliteTowerClientBackend.Close
+
+		sqliteTowerServerBackend, err := kvdb.Open(
+			kvdb.SqliteBackendName, ctx, db.Sqlite,
+			towerServerDBPath, SqliteTowerDBName, NSTowerServerDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening sqlite tower "+
+				"server DB: %v", err)
+		}
+		closeFuncs[NSTowerServerDB] = sqliteTowerServerBackend.Close
+
+		sqliteWalletBackend, err := kvdb.Open(
+			kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath,
+			SqliteChainDBName, NSWalletDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening sqlite macaroon "+
+				"DB: %v", err)
+		}
+		closeFuncs[NSWalletDB] = sqliteWalletBackend.Close
+
+		returnEarly = false
+
+		return &DatabaseBackends{
+			GraphDB:       sqliteBackend,
+			ChanStateDB:   sqliteBackend,
+			HeightHintDB:  sqliteBackend,
+			MacaroonDB:    sqliteMacaroonBackend,
+			DecayedLogDB:  sqliteDecayedLogBackend,
+			TowerClientDB: sqliteTowerClientBackend,
+			TowerServerDB: sqliteTowerServerBackend,
+			// The wallet loader will attempt to use/create the
+			// wallet in the replicated remote DB if we're running
+			// in a clustered environment. This will ensure that all
+			// members of the cluster have access to the same wallet
+			// state.
+			WalletDB: btcwallet.LoaderWithExternalWalletDB(
+				sqliteWalletBackend,
+			),
+			CloseFuncs: closeFuncs,
+		}, nil
 	}
 
 	// We're using all bbolt based databases by default.
@@ -477,6 +594,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 	}
 
 	returnEarly = false
+
 	return &DatabaseBackends{
 		GraphDB:       boltBackend,
 		ChanStateDB:   boltBackend,

sample-lnd.conf

@@ -1199,7 +1199,8 @@ litecoin.node=ltcd
 [db]
 
 ; The selected database backend. The current default backend is "bolt". lnd
-; also has experimental support for etcd, a replicated backend.
+; also has experimental support for etcd, a replicated backend, postgres and
+; sqlite.
 ; db.backend=bolt
 
 ; The maximum interval the graph database will wait between attempting to flush
@@ -1275,6 +1276,22 @@ litecoin.node=ltcd
 ; Otherwise errors may occur in lnd under high-load conditions.
 ; db.postgres.maxconnections=
 
+[sqlite]
+; Sqlite connection timeout. Valid time units are {s, m, h}. Set to zero to
+; disable.
+; db.sqlite.timeout=0s
+
+; Maximum number of connections to the sqlite db. Set to zero for unlimited.
+; db.sqlite.maxconnections=0
+
+; The maximum amount of time to wait to execute a query if the db is locked.
+; db.sqlite.busytimeout=10s
+
+; Raw pragma option pairs to be used when opening the sqlite db. The flag
+; can be specified multiple times to set multiple options.
+; db.sqlite.pragmaoptions=auto_vacuum=incremental
+; db.sqlite.pragmaoptions=temp_store=MEMORY
+
 [bolt]
 
 ; If true, prevents the database from syncing its freelist to disk.