Merge #20202: wallet: Make BDB support optional

d52f502b1e Fix mock SQLiteDatabases (Andrew Chow)
99309ab3e9 Allow disabling BDB in configure with --without-bdb (Andrew Chow)
ee47f11f73 GUI: Force descriptor wallets when BDB is not compiled (Andrew Chow)
71e40b33bd RPC: Require descriptors=True for createwallet when BDB is not compiled (Andrew Chow)
6ebc41bf9c Enforce salvage is only for BDB wallets (Andrew Chow)
a58b719cf7 Do not compile BDB things when USE_BDB is defined (Andrew Chow)
b33af48210 Include wallet/bdb.h where it is actually being used (Andrew Chow)

Pull request description:

  Adds a `--without-bdb` option to `configure` which disables the compilation of the BDB stuff. Legacy wallets will not be created when BDB is not compiled. A legacy-sqlite wallet can be loaded, but we will not create them.

  Based on #20156 to resolve the situation where both `--without-sqlite` and `--without-bdb` are provided. In that case, the wallet is disabled and `--disable-wallet` is effectively set.

ACKs for top commit:
  laanwj:
    Code review ACK d52f502b1e

Tree-SHA512: 5a92ba7a542acc2e27003e9d4e5940e0d02d5c1f110db06cdcab831372bfd83e8d89c269caff31dd5bff062c1cf5f04683becff12bd23a33be731676f346553d

build-aux/m4/bitcoin_find_bdb48.m4

@@ -6,7 +6,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[
   AC_ARG_VAR(BDB_CFLAGS, [C compiler flags for BerkeleyDB, bypasses autodetection])
   AC_ARG_VAR(BDB_LIBS, [Linker flags for BerkeleyDB, bypasses autodetection])
 
-  if test "x$BDB_CFLAGS" = "x"; then
+  if test "x$use_bdb" = "xno"; then
+    use_bdb=no
+  elif test "x$BDB_CFLAGS" = "x"; then
     AC_MSG_CHECKING([for Berkeley DB C++ headers])
     BDB_CPPFLAGS=
     bdbpath=X
@@ -44,25 +46,30 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[
       ],[])
     done
     if test "x$bdbpath" = "xX"; then
+      use_bdb=no
       AC_MSG_RESULT([no])
-      AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for wallet functionality (--disable-wallet to disable wallet functionality)])
+      AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)])
     elif test "x$bdb48path" = "xX"; then
       BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx)
       AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[
-        AC_MSG_WARN([Found Berkeley DB other than 4.8; wallets opened by this build will not be portable!])
+        AC_MSG_WARN([Found Berkeley DB other than 4.8; BDB wallets opened by this build will not be portable!])
       ],[
-        AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable wallets (--with-incompatible-bdb to ignore or --disable-wallet to disable wallet functionality)])
+        AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable BDB wallets (--with-incompatible-bdb to ignore or --without-bdb to disable BDB wallet support)])
       ])
+      use_bdb=yes
     else
       BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdb48path}],db_cxx)
       bdbpath="${bdb48path}"
+      use_bdb=yes
     fi
   else
     BDB_CPPFLAGS=${BDB_CFLAGS}
   fi
   AC_SUBST(BDB_CPPFLAGS)
 
-  if test "x$BDB_LIBS" = "x"; then
+  if test "x$use_bdb" = "xno"; then
+    use_bdb=no
+  elif test "x$BDB_LIBS" = "x"; then
     # TODO: Ideally this could find the library version and make sure it matches the headers being used
     for searchlib in db_cxx-4.8 db_cxx db4_cxx; do
       AC_CHECK_LIB([$searchlib],[main],[
@@ -71,8 +78,12 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[
       ])
     done
     if test "x$BDB_LIBS" = "x"; then
-        AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for wallet functionality (--disable-wallet to disable wallet functionality)])
+        AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)])
     fi
   fi
-  AC_SUBST(BDB_LIBS)
+  if test "x$use_bdb" != "xno"; then
+    AC_SUBST(BDB_LIBS)
+    AC_DEFINE([USE_BDB], [1], [Define if BDB support should be compiled in])
+    use_bdb=yes
+  fi
 ])

build_msvc/bitcoin_config.h

@@ -35,6 +35,12 @@
 /* Define to 1 to enable wallet functions */
 #define ENABLE_WALLET 1
 
+/* Define to 1 to enable BDB wallet */
+#define USE_BDB 1
+
+/* Define to 1 to enable SQLite wallet */
+#define USE_SQLITE 1
+
 /* Define to 1 to enable ZMQ functions */
 #define ENABLE_ZMQ 1
 

build_msvc/bitcoind/bitcoind.vcxproj

@@ -62,6 +62,10 @@
                    Replace="@EXEEXT@" By=".exe"></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
                   Replace="@ENABLE_WALLET_TRUE@" By=""></ReplaceInFile>
+    <ReplaceInFile FilePath="$(ConfigIniOut)"
+                  Replace="@USE_BDB_TRUE@" By=""></ReplaceInFile>
+    <ReplaceInFile FilePath="$(ConfigIniOut)"
+                  Replace="@USE_SQLITE_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
                   Replace="@BUILD_BITCOIN_CLI_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"

build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in

@@ -8,6 +8,9 @@
     <ConfigurationType>StaticLibrary</ConfigurationType>
   </PropertyGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\src\wallet\bdb.cpp" />
+    <ClCompile Include="..\..\src\wallet\salvage.cpp" />
+    <ClCompile Include="..\..\src\wallet\sqlite.cpp" />
 @SOURCE_FILES@
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

configure.ac

@@ -115,7 +115,7 @@ AC_ARG_ENABLE([wallet],
   [AS_HELP_STRING([--disable-wallet],
   [disable wallet (enabled by default)])],
   [enable_wallet=$enableval],
-  [enable_wallet=yes])
+  [enable_wallet=auto])
 
 AC_ARG_WITH([sqlite],
   [AS_HELP_STRING([--with-sqlite=yes|no|auto],
@@ -123,6 +123,12 @@ AC_ARG_WITH([sqlite],
   [use_sqlite=$withval],
   [use_sqlite=auto])
 
+AC_ARG_WITH([bdb],
+  [AS_HELP_STRING([--without-bdb],
+  [disable bdb wallet support (default is enabled if wallet is enabled)])],
+  [use_bdb=$withval],
+  [use_bdb=auto])
+
 AC_ARG_WITH([miniupnpc],
   [AS_HELP_STRING([--with-miniupnpc],
   [enable UPNP (default is yes if libminiupnpc is found)])],
@@ -1237,6 +1243,14 @@ if test x$enable_wallet != xno; then
       fi
     fi
     AC_MSG_RESULT([$use_sqlite])
+
+    dnl Disable wallet if both --without-bdb and --without-sqlite
+    if test "x$use_bdb$use_sqlite" = "xnono"; then
+        if test "x$enable_wallet" = "xyes"; then
+            AC_MSG_ERROR([wallet functionality requested but no BDB or SQLite support available.])
+        fi
+        enable_wallet=no
+    fi
 fi
 
 dnl Check for libminiupnpc (optional)
@@ -1492,6 +1506,7 @@ AC_MSG_CHECKING([if wallet should be enabled])
 if test x$enable_wallet != xno; then
   AC_MSG_RESULT(yes)
   AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions])
+  enable_wallet=yes
 
 else
   AC_MSG_RESULT(no)
@@ -1591,6 +1606,7 @@ AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux])
 AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows])
 AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes])
 AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"])
+AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"])
 AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes])
 AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes])
@@ -1655,6 +1671,7 @@ AC_SUBST(SHANI_CXXFLAGS)
 AC_SUBST(ARM_CRC_CXXFLAGS)
 AC_SUBST(LIBTOOL_APP_LDFLAGS)
 AC_SUBST(USE_SQLITE)
+AC_SUBST(USE_BDB)
 AC_SUBST(USE_UPNP)
 AC_SUBST(USE_QRCODE)
 AC_SUBST(BOOST_LIBS)
@@ -1732,6 +1749,7 @@ echo "  multiprocess  = $build_multiprocess"
 echo "  with wallet   = $enable_wallet"
 if test "x$enable_wallet" != "xno"; then
     echo "    with sqlite = $use_sqlite"
+    echo "    with bdb    = $use_bdb"
 fi
 echo "  with gui / qt = $bitcoin_enable_qt"
 if test x$bitcoin_enable_qt != xno; then

doc/build-unix.md

@@ -41,7 +41,7 @@ Optional dependencies:
  Library     | Purpose          | Description
  ------------|------------------|----------------------
  miniupnpc   | UPnP Support     | Firewall-jumping support
- libdb4.8    | Berkeley DB      | Wallet storage (only needed when wallet enabled)
+ libdb4.8    | Berkeley DB      | Optional, wallet storage (only needed when wallet enabled)
  qt          | GUI              | GUI toolkit (only needed when GUI enabled)
  libqrencode | QR codes in GUI  | Optional for generating QR codes (only needed when GUI enabled)
  univalue    | Utility          | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure)

doc/dependencies.md

@@ -33,7 +33,7 @@ Some dependencies are not needed in all configurations. The following are some f
 
 #### Options passed to `./configure`
 * MiniUPnPc is not needed with  `--with-miniupnpc=no`.
-* Berkeley DB is not needed with `--disable-wallet`.
+* Berkeley DB is not needed with `--disable-wallet` or `--without-bdb`.
 * SQLite is not needed with `--disable-wallet` or `--without-sqlite`.
 * Qt is not needed with `--without-gui`.
 * If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`.

src/Makefile.am

@@ -360,7 +360,6 @@ libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(SQLITE_CFLAG
 libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libbitcoin_wallet_a_SOURCES = \
   interfaces/wallet.cpp \
-  wallet/bdb.cpp \
   wallet/coincontrol.cpp \
   wallet/context.cpp \
   wallet/crypter.cpp \
@@ -370,7 +369,6 @@ libbitcoin_wallet_a_SOURCES = \
   wallet/load.cpp \
   wallet/rpcdump.cpp \
   wallet/rpcwallet.cpp \
-  wallet/salvage.cpp \
   wallet/scriptpubkeyman.cpp \
   wallet/wallet.cpp \
   wallet/walletdb.cpp \
@@ -381,6 +379,9 @@ libbitcoin_wallet_a_SOURCES = \
 if USE_SQLITE
 libbitcoin_wallet_a_SOURCES += wallet/sqlite.cpp
 endif
+if USE_BDB
+libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp
+endif
 
 libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
 libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)

src/Makefile.test.include

@@ -292,7 +292,6 @@ BITCOIN_TESTS =\
 
 if ENABLE_WALLET
 BITCOIN_TESTS += \
-  wallet/test/db_tests.cpp \
   wallet/test/psbt_wallet_tests.cpp \
   wallet/test/wallet_tests.cpp \
   wallet/test/walletdb_tests.cpp \
@@ -302,6 +301,10 @@ BITCOIN_TESTS += \
   wallet/test/ismine_tests.cpp \
   wallet/test/scriptpubkeyman_tests.cpp
 
+if USE_BDB
+BITCOIN_TESTS += wallet/test/db_tests.cpp
+endif
+
 BITCOIN_TEST_SUITE += \
   wallet/test/wallet_test_fixture.cpp \
   wallet/test/wallet_test_fixture.h \

src/qt/createwalletdialog.cpp

@@ -51,12 +51,15 @@ CreateWalletDialog::CreateWalletDialog(QWidget* parent) :
         }
     });
 
-    #ifndef USE_SQLITE
+#ifndef USE_SQLITE
         ui->descriptor_checkbox->setToolTip(tr("Compiled without sqlite support (required for descriptor wallets)"));
         ui->descriptor_checkbox->setEnabled(false);
         ui->descriptor_checkbox->setChecked(false);
-    #endif
-
+#endif
+#ifndef USE_BDB
+        ui->descriptor_checkbox->setEnabled(false);
+        ui->descriptor_checkbox->setChecked(true);
+#endif
 }
 
 CreateWalletDialog::~CreateWalletDialog()

src/qt/rpcconsole.cpp

@@ -25,6 +25,9 @@
 #include <univalue.h>
 
 #ifdef ENABLE_WALLET
+#ifdef USE_BDB
+#include <wallet/bdb.h>
+#endif
 #include <wallet/db.h>
 #include <wallet/wallet.h>
 #endif

src/wallet/init.cpp

@@ -15,6 +15,9 @@
 #include <util/moneystr.h>
 #include <util/system.h>
 #include <util/translation.h>
+#ifdef USE_BDB
+#include <wallet/bdb.h>
+#endif
 #include <wallet/coincontrol.h>
 #include <wallet/wallet.h>
 #include <walletinitinterface.h>
@@ -68,9 +71,14 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const
 #endif
     argsman.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET);
 
+#ifdef USE_BDB
     argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
     argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
     argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
+#else
+    argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"});
+#endif
+
     argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
 
     argsman.AddHiddenArgs({"-zapwallettxes"});

src/wallet/rpcwallet.cpp

@@ -2757,6 +2757,12 @@ static RPCHelpMan createwallet()
         warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet"));
     }
 
+#ifndef USE_BDB
+    if (!(flags & WALLET_FLAG_DESCRIPTORS)) {
+        throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without bdb support (required for legacy wallets)");
+    }
+#endif
+
     DatabaseOptions options;
     DatabaseStatus status;
     options.require_create = true;

src/wallet/salvage.cpp

@@ -6,6 +6,7 @@
 #include <fs.h>
 #include <streams.h>
 #include <util/translation.h>
+#include <wallet/bdb.h>
 #include <wallet/salvage.h>
 #include <wallet/wallet.h>
 #include <wallet/walletdb.h>
@@ -27,6 +28,7 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v
     DatabaseStatus status;
     options.require_existing = true;
     options.verify = false;
+    options.require_format = DatabaseFormat::BERKELEY;
     std::unique_ptr<WalletDatabase> database = MakeDatabase(file_path, options, status, error);
     if (!database) return false;
 

src/wallet/scriptpubkeyman.cpp

@@ -3,12 +3,15 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <key_io.h>
+#include <logging.h>
 #include <outputtype.h>
 #include <script/descriptor.h>
 #include <script/sign.h>
 #include <util/bip32.h>
 #include <util/strencodings.h>
 #include <util/string.h>
+#include <util/system.h>
+#include <util/time.h>
 #include <util/translation.h>
 #include <wallet/scriptpubkeyman.h>
 

src/wallet/scriptpubkeyman.h

@@ -11,6 +11,7 @@
 #include <script/standard.h>
 #include <util/error.h>
 #include <util/message.h>
+#include <util/time.h>
 #include <wallet/crypter.h>
 #include <wallet/ismine.h>
 #include <wallet/walletdb.h>

src/wallet/sqlite.cpp

@@ -206,7 +206,9 @@ void SQLiteDatabase::Open()
     }
 
     if (m_db == nullptr) {
-        TryCreateDirectories(m_dir_path);
+        if (!m_mock) {
+            TryCreateDirectories(m_dir_path);
+        }
         int ret = sqlite3_open_v2(m_file_path.c_str(), &m_db, flags, nullptr);
         if (ret != SQLITE_OK) {
             throw std::runtime_error(strprintf("SQLiteDatabase: Failed to open database: %s\n", sqlite3_errstr(ret)));

src/wallet/walletdb.cpp

@@ -14,7 +14,9 @@
 #include <util/system.h>
 #include <util/time.h>
 #include <util/translation.h>
+#ifdef USE_BDB
 #include <wallet/bdb.h>
+#endif
 #ifdef USE_SQLITE
 #include <wallet/sqlite.h>
 #endif
@@ -1011,9 +1013,11 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
 
     Optional<DatabaseFormat> format;
     if (exists) {
+#ifdef USE_BDB
         if (ExistsBerkeleyDatabase(path)) {
             format = DatabaseFormat::BERKELEY;
         }
+#endif
 #ifdef USE_SQLITE
         if (ExistsSQLiteDatabase(path)) {
             if (format) {
@@ -1052,15 +1056,31 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
     // Format is not set when a db doesn't already exist, so use the format specified by the options if it is set.
     if (!format && options.require_format) format = options.require_format;
 
+    // If the format is not specified or detected, choose the default format based on what is available. We prefer BDB over SQLite for now.
+    if (!format) {
 #ifdef USE_SQLITE
-    if (format && format == DatabaseFormat::SQLITE) {
-        return MakeSQLiteDatabase(path, options, status, error);
-    }
-#else
-    assert(format != DatabaseFormat::SQLITE);
+        format = DatabaseFormat::SQLITE;
 #endif
+#ifdef USE_BDB
+        format = DatabaseFormat::BERKELEY;
+#endif
+    }
 
+    if (format == DatabaseFormat::SQLITE) {
+#ifdef USE_SQLITE
+        return MakeSQLiteDatabase(path, options, status, error);
+#endif
+        error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", path.string()));
+        status = DatabaseStatus::FAILED_BAD_FORMAT;
+        return nullptr;
+    }
+
+#ifdef USE_BDB
     return MakeBerkeleyDatabase(path, options, status, error);
+#endif
+    error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", path.string()));
+    status = DatabaseStatus::FAILED_BAD_FORMAT;
+    return nullptr;
 }
 
 /** Return object for accessing dummy database with no read/write capabilities. */
@@ -1072,5 +1092,9 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase()
 /** Return object for accessing temporary in-memory database. */
 std::unique_ptr<WalletDatabase> CreateMockWalletDatabase()
 {
+#ifdef USE_BDB
     return MakeUnique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), "");
+#elif USE_SQLITE
+    return MakeUnique<SQLiteDatabase>("", "", true);
+#endif
 }

src/wallet/walletdb.h

@@ -8,7 +8,6 @@
 
 #include <amount.h>
 #include <script/sign.h>
-#include <wallet/bdb.h>
 #include <wallet/db.h>
 #include <wallet/walletutil.h>
 #include <key.h>

src/wallet/wallettool.cpp

@@ -122,6 +122,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name)
             WalletShowInfo(wallet_instance.get());
             wallet_instance->Close();
         } else if (command == "salvage") {
+#ifdef USE_BDB
             bilingual_str error;
             std::vector<bilingual_str> warnings;
             bool ret = RecoverDatabaseFile(path, error, warnings);
@@ -134,6 +135,10 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name)
                 }
             }
             return ret;
+#else
+            tfm::format(std::cerr, "Salvage command is not available as BDB support is not compiled");
+            return false;
+#endif
         }
     } else {
         tfm::format(std::cerr, "Invalid command: %s\n", command);

src/wallet/walletutil.cpp

@@ -7,7 +7,11 @@
 #include <logging.h>
 #include <util/system.h>
 
+#ifdef USE_BDB
 bool ExistsBerkeleyDatabase(const fs::path& path);
+#else
+#   define ExistsBerkeleyDatabase(path)  (false)
+#endif
 #ifdef USE_SQLITE
 bool ExistsSQLiteDatabase(const fs::path& path);
 #else