Merge bitcoin/bitcoin#25667: assumeutxo: snapshot initialization

bf95976061 doc: add note about snapshot chainstate init (James O'Beirne)
e4d7995286 test: add testcases for snapshot initialization (James O'Beirne)
cced4e7336 test: move-only-ish: factor out LoadVerifyActivateChainstate() (James O'Beirne)
51fc9241c0 test: allow on-disk coins and block tree dbs in tests (James O'Beirne)
3c361391b8 test: add reset_chainstate parameter for snapshot unittests (James O'Beirne)
00b357c215 validation: add ResetChainstates() (James O'Beirne)
3a29dfbfb2 move-only: test: make snapshot chainstate setup reusable (James O'Beirne)
8153bd9247 blockmanager: avoid undefined behavior during FlushBlockFile (James O'Beirne)
ad67ff377c validation: remove snapshot datadirs upon validation failure (James O'Beirne)
34d1590331 add utilities for deleting on-disk leveldb data (James O'Beirne)
252abd1e8b init: add utxo snapshot detection (James O'Beirne)
f9f1735f13 validation: rename snapshot chainstate dir (James O'Beirne)
d14bebf100 db: add StoragePath to CDBWrapper/CCoinsViewDB (James O'Beirne)

Pull request description:

  This is part of the [assumeutxo project](https://github.com/bitcoin/bitcoin/projects/11) (parent PR: https://github.com/bitcoin/bitcoin/pull/15606)

  ---

  Half of the replacement for #24232. The original PR grew larger than expected throughout the review process.

  This change adds the ability to initialize a snapshot-based chainstate during init if one is detected on disk. This is of course unused as of now (aside from in unittests) given that we haven't yet enabled actually loading snapshots.

  Don't be scared! There are some big move-only commits in here.

  Accompanying changes include:

  - moving the snapshot coinsdb directory from being called `chainstate_[base blockhash]` to `chainstate_snapshot`, since we only support one snapshot in use at a time. This simplifies some logic, but it necessitates writing that base blockhash out to a file within the coinsdb dir. See [discussion here](https://github.com/bitcoin/bitcoin/pull/24232#discussion_r832762880).
  - adding a simple fix in `FlushBlockFile()` that avoids a crash when attemping to flush to disk before `LoadBlockIndexDB()` is called, which happens when calling `MaybeRebalanceCaches()` during multiple chainstate init.
  - improving the unittest to allow testing with on-disk chainstates - necessary to test a simulated restart and re-initialization.

ACKs for top commit:
  naumenkogs:
    utACK bf95976061
  ariard:
    Code Review ACK bf9597606
  ryanofsky:
    Code review ACK bf95976061. Changes since last review: rebasing, switching from CAutoFile to AutoFile, adding comments, switching from BOOST_CHECK to Assert in test util, using chainman.GetMutex() in tests, destroying one ChainstateManager before creating a new one in tests
  fjahr:
    utACK bf95976061
  aureleoules:
    ACK bf95976061

Tree-SHA512: 15ae75caf19f8d12a12d2647c52897904d27b265a7af6b4ae7b858592eeadb8f9da6c2394b6baebec90adc28742c053e3eb506119577dae7c1e722ebb3b7bcc0

doc/design/assumeutxo.md

@@ -76,8 +76,9 @@ original chainstate remains in use as active.
 
 Once the snapshot chainstate is loaded and validated, it is promoted to active
 chainstate and a sync to tip begins. A new chainstate directory is created in the
-datadir for the snapshot chainstate called
-`chainstate_[SHA256 blockhash of snapshot base block]`.
+datadir for the snapshot chainstate called `chainstate_snapshot`. When this directory
+is present in the datadir, the snapshot chainstate will be detected and loaded as
+active on node startup (via `DetectSnapshotChainstate()`).
 
 |    |    |
 | ---------- | ----------- |

src/Makefile.am

@@ -396,6 +396,7 @@ libbitcoin_node_a_SOURCES = \
   node/minisketchwrapper.cpp \
   node/psbt.cpp \
   node/transaction.cpp \
+  node/utxo_snapshot.cpp \
   node/validation_cache_args.cpp \
   noui.cpp \
   policy/fees.cpp \
@@ -902,6 +903,7 @@ libbitcoinkernel_la_SOURCES = \
   node/blockstorage.cpp \
   node/chainstate.cpp \
   node/interface_ui.cpp \
+  node/utxo_snapshot.cpp \
   policy/feerate.cpp \
   policy/fees.cpp \
   policy/packages.cpp \

src/dbwrapper.cpp

@@ -128,7 +128,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
 }
 
 CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
-    : m_name{fs::PathToString(path.stem())}
+    : m_name{fs::PathToString(path.stem())}, m_path{path}, m_is_memory{fMemory}
 {
     penv = nullptr;
     readoptions.verify_checksums = true;

src/dbwrapper.h

@@ -39,6 +39,10 @@ public:
 
 class CDBWrapper;
 
+namespace dbwrapper {
+    using leveldb::DestroyDB;
+}
+
 /** These should be considered an implementation detail of the specific database.
  */
 namespace dbwrapper_private {
@@ -219,6 +223,12 @@ private:
 
     std::vector<unsigned char> CreateObfuscateKey() const;
 
+    //! path to filesystem storage
+    const fs::path m_path;
+
+    //! whether or not the database resides in memory
+    bool m_is_memory;
+
 public:
     /**
      * @param[in] path        Location in the filesystem where leveldb data will be stored.
@@ -268,6 +278,14 @@ public:
         return WriteBatch(batch, fSync);
     }
 
+    //! @returns filesystem path to the on-disk data.
+    std::optional<fs::path> StoragePath() {
+        if (m_is_memory) {
+            return {};
+        }
+        return m_path;
+    }
+
     template <typename K>
     bool Exists(const K& key) const
     {

src/node/blockstorage.cpp

@@ -524,6 +524,16 @@ void BlockManager::FlushUndoFile(int block_file, bool finalize)
 void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo)
 {
     LOCK(cs_LastBlockFile);
+
+    if (m_blockfile_info.size() < 1) {
+        // Return if we haven't loaded any blockfiles yet. This happens during
+        // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
+        // then calls FlushStateToDisk()), resulting in a call to this function before we
+        // have populated `m_blockfile_info` via LoadBlockIndexDB().
+        return;
+    }
+    assert(static_cast<int>(m_blockfile_info.size()) > m_last_blockfile);
+
     FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize);
     if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
         AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");

src/node/chainstate.cpp

@@ -48,10 +48,15 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
     }
 
     LOCK(cs_main);
-    chainman.InitializeChainstate(options.mempool);
     chainman.m_total_coinstip_cache = cache_sizes.coins;
     chainman.m_total_coinsdb_cache = cache_sizes.coins_db;
 
+    // Load the fully validated chainstate.
+    chainman.InitializeChainstate(options.mempool);
+
+    // Load a chain created from a UTXO snapshot, if any exist.
+    chainman.DetectSnapshotChainstate(options.mempool);
+
     auto& pblocktree{chainman.m_blockman.m_block_tree_db};
     // new CBlockTreeDB tries to delete the existing file, which
     // fails if it's still open from the previous loop. Close it first:
@@ -98,12 +103,20 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")};
     }
 
+    // Conservative value which is arbitrarily chosen, as it will ultimately be changed
+    // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure
+    // that the sum of the two caches (40%) does not exceed the allowable amount
+    // during this temporary initialization state.
+    double init_cache_fraction = 0.2;
+
     // At this point we're either in reindex or we've loaded a useful
     // block tree into BlockIndex()!
 
     for (Chainstate* chainstate : chainman.GetAll()) {
+        LogPrintf("Initializing chainstate %s\n", chainstate->ToString());
+
         chainstate->InitCoinsDB(
-            /*cache_size_bytes=*/cache_sizes.coins_db,
+            /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction,
             /*in_memory=*/options.coins_db_in_memory,
             /*should_wipe=*/options.reindex || options.reindex_chainstate);
 
@@ -125,7 +138,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         }
 
         // The on-disk coinsdb is now in a good state, create the cache
-        chainstate->InitCoinsCache(cache_sizes.coins);
+        chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction);
         assert(chainstate->CanFlushToDisk());
 
         if (!is_coinsview_empty(chainstate)) {
@@ -146,6 +159,11 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         };
     }
 
+    // Now that chainstates are loaded and we're able to flush to
+    // disk, rebalance the coins caches to desired levels based
+    // on the condition of each chainstate.
+    chainman.MaybeRebalanceCaches();
+
     return {ChainstateLoadStatus::SUCCESS, {}};
 }
 

src/node/utxo_snapshot.cpp

@@ -0,0 +1,91 @@
+// Copyright (c) 2022 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <node/utxo_snapshot.h>
+
+#include <fs.h>
+#include <logging.h>
+#include <streams.h>
+#include <uint256.h>
+#include <util/system.h>
+#include <validation.h>
+
+#include <cstdio>
+#include <optional>
+
+namespace node {
+
+bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate)
+{
+    AssertLockHeld(::cs_main);
+    assert(snapshot_chainstate.m_from_snapshot_blockhash);
+
+    const std::optional<fs::path> chaindir = snapshot_chainstate.CoinsDB().StoragePath();
+    assert(chaindir); // Sanity check that chainstate isn't in-memory.
+    const fs::path write_to = *chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME;
+
+    FILE* file{fsbridge::fopen(write_to, "wb")};
+    AutoFile afile{file};
+    if (afile.IsNull()) {
+        LogPrintf("[snapshot] failed to open base blockhash file for writing: %s\n",
+                  fs::PathToString(write_to));
+        return false;
+    }
+    afile << *snapshot_chainstate.m_from_snapshot_blockhash;
+
+    if (afile.fclose() != 0) {
+        LogPrintf("[snapshot] failed to close base blockhash file %s after writing\n",
+                  fs::PathToString(write_to));
+        return false;
+    }
+    return true;
+}
+
+std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir)
+{
+    if (!fs::exists(chaindir)) {
+        LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " /* Continued */
+            "exists at path %s\n", fs::PathToString(chaindir));
+        return std::nullopt;
+    }
+    const fs::path read_from = chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME;
+    const std::string read_from_str = fs::PathToString(read_from);
+
+    if (!fs::exists(read_from)) {
+        LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " /* Continued */
+            "exists at path %s. Try deleting %s and calling loadtxoutset again?\n",
+            fs::PathToString(chaindir), read_from_str);
+        return std::nullopt;
+    }
+
+    uint256 base_blockhash;
+    FILE* file{fsbridge::fopen(read_from, "rb")};
+    AutoFile afile{file};
+    if (afile.IsNull()) {
+        LogPrintf("[snapshot] failed to open base blockhash file for reading: %s\n",
+            read_from_str);
+        return std::nullopt;
+    }
+    afile >> base_blockhash;
+
+    if (std::fgetc(afile.Get()) != EOF) {
+        LogPrintf("[snapshot] warning: unexpected trailing data in %s\n", read_from_str);
+    } else if (std::ferror(afile.Get())) {
+        LogPrintf("[snapshot] warning: i/o error reading %s\n", read_from_str);
+    }
+    return base_blockhash;
+}
+
+std::optional<fs::path> FindSnapshotChainstateDir()
+{
+    fs::path possible_dir =
+        gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX));
+
+    if (fs::exists(possible_dir)) {
+        return possible_dir;
+    }
+    return std::nullopt;
+}
+
+} // namespace node

src/node/utxo_snapshot.h

@@ -6,8 +6,14 @@
 #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H
 #define BITCOIN_NODE_UTXO_SNAPSHOT_H
 
+#include <fs.h>
 #include <uint256.h>
 #include <serialize.h>
+#include <validation.h>
+
+#include <optional>
+
+extern RecursiveMutex cs_main;
 
 namespace node {
 //! Metadata describing a serialized version of a UTXO set from which an
@@ -33,6 +39,33 @@ public:
 
     SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); }
 };
+
+//! The file in the snapshot chainstate dir which stores the base blockhash. This is
+//! needed to reconstruct snapshot chainstates on init.
+//!
+//! Because we only allow loading a single snapshot at a time, there will only be one
+//! chainstate directory with this filename present within it.
+const fs::path SNAPSHOT_BLOCKHASH_FILENAME{"base_blockhash"};
+
+//! Write out the blockhash of the snapshot base block that was used to construct
+//! this chainstate. This value is read in during subsequent initializations and
+//! used to reconstruct snapshot-based chainstates.
+bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate)
+    EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+//! Read the blockhash of the snapshot base block that was used to construct the
+//! chainstate.
+std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir)
+    EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+//! Suffix appended to the chainstate (leveldb) dir when created based upon
+//! a snapshot.
+constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot";
+
+
+//! Return a path to the snapshot-based chainstate dir, if one exists.
+std::optional<fs::path> FindSnapshotChainstateDir();
+
 } // namespace node
 
 #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H

src/streams.h

@@ -487,12 +487,14 @@ public:
     AutoFile(const AutoFile&) = delete;
     AutoFile& operator=(const AutoFile&) = delete;
 
-    void fclose()
+    int fclose()
     {
+        int retval{0};
         if (file) {
-            ::fclose(file);
+            retval = ::fclose(file);
             file = nullptr;
         }
+        return retval;
     }
 
     /** Get wrapped FILE* with transfer of ownership.

src/test/util/chainstate.h

@@ -11,6 +11,7 @@
 #include <node/context.h>
 #include <node/utxo_snapshot.h>
 #include <rpc/blockchain.h>
+#include <test/util/setup_common.h>
 #include <validation.h>
 
 #include <univalue.h>
@@ -20,11 +21,24 @@ const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){};
 /**
  * Create and activate a UTXO snapshot, optionally providing a function to
  * malleate the snapshot.
+ *
+ * If `reset_chainstate` is true, reset the original chainstate back to the genesis
+ * block. This allows us to simulate more realistic conditions in which a snapshot is
+ * loaded into an otherwise mostly-uninitialized datadir. It also allows us to test
+ * conditions that would otherwise cause shutdowns based on the IBD chainstate going
+ * past the snapshot it generated.
  */
 template<typename F = decltype(NoMalleation)>
 static bool
-CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F malleation = NoMalleation)
+CreateAndActivateUTXOSnapshot(
+    TestingSetup* fixture,
+    F malleation = NoMalleation,
+    bool reset_chainstate = false,
+    bool in_memory_chainstate = false)
 {
+    node::NodeContext& node = fixture->m_node;
+    fs::path root = fixture->m_path_root;
+
     // Write out a snapshot to the test's tempdir.
     //
     int height;
@@ -47,7 +61,38 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma
 
     malleation(auto_infile, metadata);
 
-    return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory=*/true);
+    if (reset_chainstate) {
+        {
+            // What follows is code to selectively reset chainstate data without
+            // disturbing the existing BlockManager instance, which is needed to
+            // recognize the headers chain previously generated by the chainstate we're
+            // removing. Without those headers, we can't activate the snapshot below.
+            //
+            // This is a stripped-down version of node::LoadChainstate which
+            // preserves the block index.
+            LOCK(::cs_main);
+            uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash();
+            node.chainman->ResetChainstates();
+            node.chainman->InitializeChainstate(node.mempool.get());
+            Chainstate& chain = node.chainman->ActiveChainstate();
+            Assert(chain.LoadGenesisBlock());
+            // These cache values will be corrected shortly in `MaybeRebalanceCaches`.
+            chain.InitCoinsDB(1 << 20, true, false, "");
+            chain.InitCoinsCache(1 << 20);
+            chain.CoinsTip().SetBestBlock(gen_hash);
+            chain.setBlockIndexCandidates.insert(node.chainman->m_blockman.LookupBlockIndex(gen_hash));
+            chain.LoadChainTip();
+            node.chainman->MaybeRebalanceCaches();
+        }
+        BlockValidationState state;
+        if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) {
+            throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString()));
+        }
+        Assert(
+            0 == WITH_LOCK(node.chainman->GetMutex(), return node.chainman->ActiveHeight()));
+    }
+
+    return node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate);
 }
 
 

src/test/util/setup_common.cpp

@@ -220,17 +220,12 @@ ChainTestingSetup::~ChainTestingSetup()
     m_node.chainman.reset();
 }
 
-TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args)
-    : ChainTestingSetup(chainName, extra_args)
+void TestingSetup::LoadVerifyActivateChainstate()
 {
-    // Ideally we'd move all the RPC tests to the functional testing framework
-    // instead of unit tests, but for now we need these here.
-    RegisterAllCoreRPCCommands(tableRPC);
-
     node::ChainstateLoadOptions options;
     options.mempool = Assert(m_node.mempool.get());
-    options.block_tree_db_in_memory = true;
-    options.coins_db_in_memory = true;
+    options.block_tree_db_in_memory = m_block_tree_db_in_memory;
+    options.coins_db_in_memory = m_coins_db_in_memory;
     options.reindex = node::fReindex;
     options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false);
     options.prune = node::fPruneMode;
@@ -246,6 +241,22 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const
     if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) {
         throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString()));
     }
+}
+
+TestingSetup::TestingSetup(
+    const std::string& chainName,
+    const std::vector<const char*>& extra_args,
+    const bool coins_db_in_memory,
+    const bool block_tree_db_in_memory)
+    : ChainTestingSetup(chainName, extra_args),
+      m_coins_db_in_memory(coins_db_in_memory),
+      m_block_tree_db_in_memory(block_tree_db_in_memory)
+{
+    // Ideally we'd move all the RPC tests to the functional testing framework
+    // instead of unit tests, but for now we need these here.
+    RegisterAllCoreRPCCommands(tableRPC);
+
+    LoadVerifyActivateChainstate();
 
     m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>());
     m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman,
@@ -263,8 +274,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const
     }
 }
 
-TestChain100Setup::TestChain100Setup(const std::string& chain_name, const std::vector<const char*>& extra_args)
-    : TestingSetup{chain_name, extra_args}
+TestChain100Setup::TestChain100Setup(
+        const std::string& chain_name,
+        const std::vector<const char*>& extra_args,
+        const bool coins_db_in_memory,
+        const bool block_tree_db_in_memory)
+    : TestingSetup{CBaseChainParams::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory}
 {
     SetMockTime(1598887952);
     constexpr std::array<unsigned char, 32> vchKey = {

src/test/util/setup_common.h

@@ -107,7 +107,16 @@ struct ChainTestingSetup : public BasicTestingSetup {
 /** Testing setup that configures a complete environment.
  */
 struct TestingSetup : public ChainTestingSetup {
-    explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {});
+    bool m_coins_db_in_memory{true};
+    bool m_block_tree_db_in_memory{true};
+
+    void LoadVerifyActivateChainstate();
+
+    explicit TestingSetup(
+        const std::string& chainName = CBaseChainParams::MAIN,
+        const std::vector<const char*>& extra_args = {},
+        const bool coins_db_in_memory = true,
+        const bool block_tree_db_in_memory = true);
 };
 
 /** Identical to TestingSetup, but chain set to regtest */
@@ -124,8 +133,11 @@ class CScript;
  * Testing fixture that pre-creates a 100-block REGTEST-mode block chain
  */
 struct TestChain100Setup : public TestingSetup {
-    TestChain100Setup(const std::string& chain_name = CBaseChainParams::REGTEST,
-                      const std::vector<const char*>& extra_args = {});
+    TestChain100Setup(
+        const std::string& chain_name = CBaseChainParams::REGTEST,
+        const std::vector<const char*>& extra_args = {},
+        const bool coins_db_in_memory = true,
+        const bool block_tree_db_in_memory = true);
 
     /**
      * Create a new block with just given transactions, coinbase paying to

src/test/validation_chainstate_tests.cpp

@@ -89,7 +89,8 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup)
     // After adding some blocks to the tip, best block should have changed.
     BOOST_CHECK(::g_best_block != curr_tip);
 
-    BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root));
+    BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(
+        this, NoMalleation, /*reset_chainstate=*/ true));
 
     // Ensure our active chain is the snapshot chainstate.
     BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive()));

src/test/validation_chainstatemanager_tests.cpp

@@ -10,6 +10,7 @@
 #include <sync.h>
 #include <test/util/chainstate.h>
 #include <test/util/setup_common.h>
+#include <timedata.h>
 #include <uint256.h>
 #include <validation.h>
 #include <validationinterface.h>
@@ -63,7 +64,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager)
     // Create a snapshot-based chainstate.
     //
     const uint256 snapshot_blockhash = GetRandHash();
-    Chainstate& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(
+    Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot(
         &mempool, snapshot_blockhash));
     chainstates.push_back(&c2);
 
@@ -133,7 +134,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches)
 
     // Create a snapshot-based chainstate.
     //
-    Chainstate& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash()));
+    Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash()));
     chainstates.push_back(&c2);
     c2.InitCoinsDB(
         /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false);
@@ -154,162 +155,240 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches)
     BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1);
 }
 
-//! Test basic snapshot activation.
-BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup)
-{
-    ChainstateManager& chainman = *Assert(m_node.chainman);
-
-    size_t initial_size;
-    size_t initial_total_coins{100};
-
-    // Make some initial assertions about the contents of the chainstate.
+struct SnapshotTestSetup : TestChain100Setup {
+    // Run with coinsdb on the filesystem to support, e.g., moving invalidated
+    // chainstate dirs to "*_invalid".
+    //
+    // Note that this means the tests run considerably slower than in-memory DB
+    // tests, but we can't otherwise test this functionality since it relies on
+    // destructive filesystem operations.
+    SnapshotTestSetup() : TestChain100Setup{
+                              {},
+                              {},
+                              /*coins_db_in_memory=*/false,
+                              /*block_tree_db_in_memory=*/false,
+                          }
     {
-        LOCK(::cs_main);
-        CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip();
-        initial_size = ibd_coinscache.GetCacheSize();
-        size_t total_coins{0};
-
-        for (CTransactionRef& txn : m_coinbase_txns) {
-            COutPoint op{txn->GetHash(), 0};
-            BOOST_CHECK(ibd_coinscache.HaveCoin(op));
-            total_coins++;
-        }
-
-        BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
-        BOOST_CHECK_EQUAL(initial_size, initial_total_coins);
     }
 
-    // Snapshot should refuse to load at this height.
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root));
-    BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash);
-    BOOST_CHECK(!chainman.SnapshotBlockhash());
-
-    // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can
-    // be found.
-    constexpr int snapshot_height = 110;
-    mineBlocks(10);
-    initial_size += 10;
-    initial_total_coins += 10;
-
-    // Should not load malleated snapshots
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
-        m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
-            // A UTXO is missing but count is correct
-            metadata.m_coins_count -= 1;
-
-            COutPoint outpoint;
-            Coin coin;
-
-            auto_infile >> outpoint;
-            auto_infile >> coin;
-    }));
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
-        m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
-            // Coins count is larger than coins in file
-            metadata.m_coins_count += 1;
-    }));
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
-        m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
-            // Coins count is smaller than coins in file
-            metadata.m_coins_count -= 1;
-    }));
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
-        m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
-            // Wrong hash
-            metadata.m_base_blockhash = uint256::ZERO;
-    }));
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
-        m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
-            // Wrong hash
-            metadata.m_base_blockhash = uint256::ONE;
-    }));
-
-    BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root));
-
-    // Ensure our active chain is the snapshot chainstate.
-    BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull());
-    BOOST_CHECK_EQUAL(
-        *chainman.ActiveChainstate().m_from_snapshot_blockhash,
-        *chainman.SnapshotBlockhash());
-
-    // Ensure that the genesis block was not marked assumed-valid.
-    BOOST_CHECK(WITH_LOCK(::cs_main, return !chainman.ActiveChain().Genesis()->IsAssumedValid()));
-
-    const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params());
-    const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip());
-
-    BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx);
-
-    // To be checked against later when we try loading a subsequent snapshot.
-    uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()};
-
-    // Make some assertions about the both chainstates. These checks ensure the
-    // legacy chainstate hasn't changed and that the newly created chainstate
-    // reflects the expected content.
+    std::tuple<Chainstate*, Chainstate*> SetupSnapshot()
     {
-        LOCK(::cs_main);
-        int chains_tested{0};
+        ChainstateManager& chainman = *Assert(m_node.chainman);
 
-        for (Chainstate* chainstate : chainman.GetAll()) {
-            BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
-            CCoinsViewCache& coinscache = chainstate->CoinsTip();
+        BOOST_CHECK(!chainman.IsSnapshotActive());
 
-            // Both caches will be empty initially.
-            BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize());
+        {
+            LOCK(::cs_main);
+            BOOST_CHECK(!chainman.IsSnapshotValidated());
+            BOOST_CHECK(!node::FindSnapshotChainstateDir());
+        }
 
+        size_t initial_size;
+        size_t initial_total_coins{100};
+
+        // Make some initial assertions about the contents of the chainstate.
+        {
+            LOCK(::cs_main);
+            CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip();
+            initial_size = ibd_coinscache.GetCacheSize();
             size_t total_coins{0};
 
             for (CTransactionRef& txn : m_coinbase_txns) {
                 COutPoint op{txn->GetHash(), 0};
-                BOOST_CHECK(coinscache.HaveCoin(op));
+                BOOST_CHECK(ibd_coinscache.HaveCoin(op));
                 total_coins++;
             }
 
-            BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize());
             BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
-            chains_tested++;
+            BOOST_CHECK_EQUAL(initial_size, initial_total_coins);
         }
 
-        BOOST_CHECK_EQUAL(chains_tested, 2);
-    }
+        Chainstate& validation_chainstate = chainman.ActiveChainstate();
 
-    // Mine some new blocks on top of the activated snapshot chainstate.
-    constexpr size_t new_coins{100};
-    mineBlocks(new_coins);  // Defined in TestChain100Setup.
+        // Snapshot should refuse to load at this height.
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this));
+        BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash);
+        BOOST_CHECK(!chainman.SnapshotBlockhash());
 
-    {
-        LOCK(::cs_main);
-        size_t coins_in_active{0};
-        size_t coins_in_background{0};
-        size_t coins_missing_from_background{0};
+        // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can
+        // be found.
+        constexpr int snapshot_height = 110;
+        mineBlocks(10);
+        initial_size += 10;
+        initial_total_coins += 10;
 
-        for (Chainstate* chainstate : chainman.GetAll()) {
-            BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
-            CCoinsViewCache& coinscache = chainstate->CoinsTip();
-            bool is_background = chainstate != &chainman.ActiveChainstate();
+        // Should not load malleated snapshots
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+            this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
+                // A UTXO is missing but count is correct
+                metadata.m_coins_count -= 1;
 
-            for (CTransactionRef& txn : m_coinbase_txns) {
-                COutPoint op{txn->GetHash(), 0};
-                if (coinscache.HaveCoin(op)) {
-                    (is_background ? coins_in_background : coins_in_active)++;
-                } else if (is_background) {
-                    coins_missing_from_background++;
+                COutPoint outpoint;
+                Coin coin;
+
+                auto_infile >> outpoint;
+                auto_infile >> coin;
+        }));
+
+        BOOST_CHECK(!node::FindSnapshotChainstateDir());
+
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+            this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
+                // Coins count is larger than coins in file
+                metadata.m_coins_count += 1;
+        }));
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+            this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
+                // Coins count is smaller than coins in file
+                metadata.m_coins_count -= 1;
+        }));
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+            this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
+                // Wrong hash
+                metadata.m_base_blockhash = uint256::ZERO;
+        }));
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+            this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
+                // Wrong hash
+                metadata.m_base_blockhash = uint256::ONE;
+        }));
+
+        BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this));
+        BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir()));
+
+        // Ensure our active chain is the snapshot chainstate.
+        BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull());
+        BOOST_CHECK_EQUAL(
+            *chainman.ActiveChainstate().m_from_snapshot_blockhash,
+            *chainman.SnapshotBlockhash());
+
+        Chainstate& snapshot_chainstate = chainman.ActiveChainstate();
+
+        {
+            LOCK(::cs_main);
+
+            fs::path found = *node::FindSnapshotChainstateDir();
+
+            // Note: WriteSnapshotBaseBlockhash() is implicitly tested above.
+            BOOST_CHECK_EQUAL(
+                *node::ReadSnapshotBaseBlockhash(found),
+                *chainman.SnapshotBlockhash());
+
+            // Ensure that the genesis block was not marked assumed-valid.
+            BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid());
+        }
+
+        const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params());
+        const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip());
+
+        BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx);
+
+        // To be checked against later when we try loading a subsequent snapshot.
+        uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()};
+
+        // Make some assertions about the both chainstates. These checks ensure the
+        // legacy chainstate hasn't changed and that the newly created chainstate
+        // reflects the expected content.
+        {
+            LOCK(::cs_main);
+            int chains_tested{0};
+
+            for (Chainstate* chainstate : chainman.GetAll()) {
+                BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
+                CCoinsViewCache& coinscache = chainstate->CoinsTip();
+
+                // Both caches will be empty initially.
+                BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize());
+
+                size_t total_coins{0};
+
+                for (CTransactionRef& txn : m_coinbase_txns) {
+                    COutPoint op{txn->GetHash(), 0};
+                    BOOST_CHECK(coinscache.HaveCoin(op));
+                    total_coins++;
+                }
+
+                BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize());
+                BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
+                chains_tested++;
+            }
+
+            BOOST_CHECK_EQUAL(chains_tested, 2);
+        }
+
+        // Mine some new blocks on top of the activated snapshot chainstate.
+        constexpr size_t new_coins{100};
+        mineBlocks(new_coins);  // Defined in TestChain100Setup.
+
+        {
+            LOCK(::cs_main);
+            size_t coins_in_active{0};
+            size_t coins_in_background{0};
+            size_t coins_missing_from_background{0};
+
+            for (Chainstate* chainstate : chainman.GetAll()) {
+                BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
+                CCoinsViewCache& coinscache = chainstate->CoinsTip();
+                bool is_background = chainstate != &chainman.ActiveChainstate();
+
+                for (CTransactionRef& txn : m_coinbase_txns) {
+                    COutPoint op{txn->GetHash(), 0};
+                    if (coinscache.HaveCoin(op)) {
+                        (is_background ? coins_in_background : coins_in_active)++;
+                    } else if (is_background) {
+                        coins_missing_from_background++;
+                    }
                 }
             }
+
+            BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins);
+            BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins);
+            BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins);
         }
 
-        BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins);
-        BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins);
-        BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins);
+        // Snapshot should refuse to load after one has already loaded.
+        BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this));
+
+        // Snapshot blockhash should be unchanged.
+        BOOST_CHECK_EQUAL(
+            *chainman.ActiveChainstate().m_from_snapshot_blockhash,
+            loaded_snapshot_blockhash);
+        return std::make_tuple(&validation_chainstate, &snapshot_chainstate);
     }
 
-    // Snapshot should refuse to load after one has already loaded.
-    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root));
+    // Simulate a restart of the node by flushing all state to disk, clearing the
+    // existing ChainstateManager, and unloading the block index.
+    //
+    // @returns a reference to the "restarted" ChainstateManager
+    ChainstateManager& SimulateNodeRestart()
+    {
+        ChainstateManager& chainman = *Assert(m_node.chainman);
 
-    // Snapshot blockhash should be unchanged.
-    BOOST_CHECK_EQUAL(
-        *chainman.ActiveChainstate().m_from_snapshot_blockhash,
-        loaded_snapshot_blockhash);
+        BOOST_TEST_MESSAGE("Simulating node restart");
+        {
+            LOCK(::cs_main);
+            for (Chainstate* cs : chainman.GetAll()) {
+                cs->ForceFlushStateToDisk();
+            }
+            chainman.ResetChainstates();
+            BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0);
+            const ChainstateManager::Options chainman_opts{
+                .chainparams = ::Params(),
+                .adjusted_time_callback = GetAdjustedTime,
+            };
+            // For robustness, ensure the old manager is destroyed before creating a
+            // new one.
+            m_node.chainman.reset();
+            m_node.chainman.reset(new ChainstateManager(chainman_opts));
+        }
+        return *Assert(m_node.chainman);
+    }
+};
+
+//! Test basic snapshot activation.
+BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup)
+{
+    this->SetupSnapshot();
 }
 
 //! Test LoadBlockIndex behavior when multiple chainstates are in use.
@@ -374,7 +453,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup)
     BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid);
 
     Chainstate& cs2 = WITH_LOCK(::cs_main,
-        return chainman.InitializeChainstate(&mempool, GetRandHash()));
+        return chainman.ActivateExistingSnapshot(&mempool, GetRandHash()));
 
     reload_all_block_indexes();
 
@@ -390,4 +469,59 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup)
     BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes);
 }
 
+//! Ensure that snapshot chainstates initialize properly when found on disk.
+BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup)
+{
+    this->SetupSnapshot();
+
+    ChainstateManager& chainman = *Assert(m_node.chainman);
+
+    fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir();
+    BOOST_CHECK(fs::exists(snapshot_chainstate_dir));
+    BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot");
+
+    BOOST_CHECK(chainman.IsSnapshotActive());
+    const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(),
+        return chainman.ActiveTip()->GetBlockHash());
+
+    auto all_chainstates = chainman.GetAll();
+    BOOST_CHECK_EQUAL(all_chainstates.size(), 2);
+
+    // Test that simulating a shutdown (resetting ChainstateManager) and then performing
+    // chainstate reinitializing successfully cleans up the background-validation
+    // chainstate data, and we end up with a single chainstate that is at tip.
+    ChainstateManager& chainman_restarted = this->SimulateNodeRestart();
+
+    BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate");
+
+    // This call reinitializes the chainstates.
+    this->LoadVerifyActivateChainstate();
+
+    {
+        LOCK(chainman_restarted.GetMutex());
+        BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 2);
+        BOOST_CHECK(chainman_restarted.IsSnapshotActive());
+        BOOST_CHECK(!chainman_restarted.IsSnapshotValidated());
+
+        BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash);
+        BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210);
+    }
+
+    BOOST_TEST_MESSAGE(
+        "Ensure we can mine blocks on top of the initialized snapshot chainstate");
+    mineBlocks(10);
+    {
+        LOCK(chainman_restarted.GetMutex());
+        BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220);
+
+        // Background chainstate should be unaware of new blocks on the snapshot
+        // chainstate.
+        for (Chainstate* cs : chainman_restarted.GetAll()) {
+            if (cs != &chainman_restarted.ActiveChainstate()) {
+                BOOST_CHECK_EQUAL(cs->m_chain.Height(), 110);
+            }
+        }
+    }
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/txdb.h

@@ -9,6 +9,7 @@
 #include <coins.h>
 #include <dbwrapper.h>
 #include <sync.h>
+#include <fs.h>
 
 #include <memory>
 #include <optional>
@@ -72,6 +73,9 @@ public:
 
     //! Dynamically alter the underlying leveldb cache size.
     void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
+
+    //! @returns filesystem path to on-disk storage or std::nullopt if in memory.
+    std::optional<fs::path> StoragePath() { return m_db->StoragePath(); }
 };
 
 /** Access to the block database (blocks/index/) */

src/validation.cpp

@@ -1513,7 +1513,7 @@ void Chainstate::InitCoinsDB(
     fs::path leveldb_name)
 {
     if (m_from_snapshot_blockhash) {
-        leveldb_name += "_" + m_from_snapshot_blockhash->ToString();
+        leveldb_name += node::SNAPSHOT_CHAINSTATE_SUFFIX;
     }
 
     m_coins_views = std::make_unique<CoinsViews>(
@@ -4804,28 +4804,15 @@ std::vector<Chainstate*> ChainstateManager::GetAll()
     return out;
 }
 
-Chainstate& ChainstateManager::InitializeChainstate(
-    CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash)
+Chainstate& ChainstateManager::InitializeChainstate(CTxMemPool* mempool)
 {
     AssertLockHeld(::cs_main);
-    bool is_snapshot = snapshot_blockhash.has_value();
-    std::unique_ptr<Chainstate>& to_modify =
-        is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate;
+    assert(!m_ibd_chainstate);
+    assert(!m_active_chainstate);
 
-    if (to_modify) {
-        throw std::logic_error("should not be overwriting a chainstate");
-    }
-    to_modify.reset(new Chainstate(mempool, m_blockman, *this, snapshot_blockhash));
-
-    // Snapshot chainstates and initial IBD chaintates always become active.
-    if (is_snapshot || (!is_snapshot && !m_active_chainstate)) {
-        LogPrintf("Switching active chainstate to %s\n", to_modify->ToString());
-        m_active_chainstate = to_modify.get();
-    } else {
-        throw std::logic_error("unexpected chainstate activation");
-    }
-
-    return *to_modify;
+    m_ibd_chainstate = std::make_unique<Chainstate>(mempool, m_blockman, *this);
+    m_active_chainstate = m_ibd_chainstate.get();
+    return *m_active_chainstate;
 }
 
 const AssumeutxoData* ExpectedAssumeutxo(
@@ -4840,6 +4827,46 @@ const AssumeutxoData* ExpectedAssumeutxo(
     return nullptr;
 }
 
+static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot)
+    EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
+{
+    AssertLockHeld(::cs_main);
+
+    if (is_snapshot) {
+        fs::path base_blockhash_path = db_path / node::SNAPSHOT_BLOCKHASH_FILENAME;
+
+        if (fs::exists(base_blockhash_path)) {
+            bool removed = fs::remove(base_blockhash_path);
+            if (!removed) {
+                LogPrintf("[snapshot] failed to remove file %s\n",
+                          fs::PathToString(base_blockhash_path));
+            }
+        } else {
+            LogPrintf("[snapshot] snapshot chainstate dir being removed lacks %s file\n",
+                    fs::PathToString(node::SNAPSHOT_BLOCKHASH_FILENAME));
+        }
+    }
+
+    std::string path_str = fs::PathToString(db_path);
+    LogPrintf("Removing leveldb dir at %s\n", path_str);
+
+    // We have to destruct before this call leveldb::DB in order to release the db
+    // lock, otherwise `DestroyDB` will fail. See `leveldb::~DBImpl()`.
+    const bool destroyed = dbwrapper::DestroyDB(path_str, {}).ok();
+
+    if (!destroyed) {
+        LogPrintf("error: leveldb DestroyDB call failed on %s\n", path_str);
+    }
+
+    // Datadir should be removed from filesystem; otherwise initialization may detect
+    // it on subsequent statups and get confused.
+    //
+    // If the base_blockhash_path removal above fails in the case of snapshot
+    // chainstates, this will return false since leveldb won't remove a non-empty
+    // directory.
+    return destroyed && !fs::exists(db_path);
+}
+
 bool ChainstateManager::ActivateSnapshot(
         AutoFile& coins_file,
         const SnapshotMetadata& metadata,
@@ -4897,11 +4924,34 @@ bool ChainstateManager::ActivateSnapshot(
             static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC));
     }
 
-    const bool snapshot_ok = this->PopulateAndValidateSnapshot(
+    bool snapshot_ok = this->PopulateAndValidateSnapshot(
         *snapshot_chainstate, coins_file, metadata);
 
+    // If not in-memory, persist the base blockhash for use during subsequent
+    // initialization.
+    if (!in_memory) {
+        LOCK(::cs_main);
+        if (!node::WriteSnapshotBaseBlockhash(*snapshot_chainstate)) {
+            snapshot_ok = false;
+        }
+    }
     if (!snapshot_ok) {
-        WITH_LOCK(::cs_main, this->MaybeRebalanceCaches());
+        LOCK(::cs_main);
+        this->MaybeRebalanceCaches();
+
+        // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir
+        // has been created, so only attempt removal if we got that far.
+        if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) {
+            // We have to destruct leveldb::DB in order to release the db lock, otherwise
+            // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`.
+            // Destructing the chainstate (and so resetting the coinsviews object) does this.
+            snapshot_chainstate.reset();
+            bool removed = DeleteCoinsDBFromDisk(*snapshot_datadir, /*is_snapshot=*/true);
+            if (!removed) {
+                AbortNode(strprintf("Failed to remove snapshot chainstate dir (%s). "
+                    "Manually remove it before restarting.\n", fs::PathToString(*snapshot_datadir)));
+            }
+        }
         return false;
     }
 
@@ -5175,6 +5225,13 @@ void ChainstateManager::MaybeRebalanceCaches()
     }
 }
 
+void ChainstateManager::ResetChainstates()
+{
+    m_ibd_chainstate.reset();
+    m_snapshot_chainstate.reset();
+    m_active_chainstate = nullptr;
+}
+
 ChainstateManager::~ChainstateManager()
 {
     LOCK(::cs_main);
@@ -5186,3 +5243,31 @@ ChainstateManager::~ChainstateManager()
         i.clear();
     }
 }
+
+bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool)
+{
+    assert(!m_snapshot_chainstate);
+    std::optional<fs::path> path = node::FindSnapshotChainstateDir();
+    if (!path) {
+        return false;
+    }
+    std::optional<uint256> base_blockhash = node::ReadSnapshotBaseBlockhash(*path);
+    if (!base_blockhash) {
+        return false;
+    }
+    LogPrintf("[snapshot] detected active snapshot chainstate (%s) - loading\n",
+        fs::PathToString(*path));
+
+    this->ActivateExistingSnapshot(mempool, *base_blockhash);
+    return true;
+}
+
+Chainstate& ChainstateManager::ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash)
+{
+    assert(!m_snapshot_chainstate);
+    m_snapshot_chainstate =
+        std::make_unique<Chainstate>(mempool, m_blockman, *this, base_blockhash);
+    LogPrintf("[snapshot] switching active chainstate to %s\n", m_snapshot_chainstate->ToString());
+    m_active_chainstate = m_snapshot_chainstate.get();
+    return *m_snapshot_chainstate;
+}

src/validation.h

@@ -926,17 +926,11 @@ public:
     //! coins databases. This will be split somehow across chainstates.
     int64_t m_total_coinsdb_cache{0};
 
-    //! Instantiate a new chainstate and assign it based upon whether it is
-    //! from a snapshot.
+    //! Instantiate a new chainstate.
     //!
     //! @param[in] mempool              The mempool to pass to the chainstate
     //                                  constructor
-    //! @param[in] snapshot_blockhash   If given, signify that this chainstate
-    //!                                 is based on a snapshot.
-    Chainstate& InitializeChainstate(
-        CTxMemPool* mempool,
-        const std::optional<uint256>& snapshot_blockhash = std::nullopt)
-        LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+    Chainstate& InitializeChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 
     //! Get all chainstates currently being used.
     std::vector<Chainstate*> GetAll();
@@ -1050,6 +1044,17 @@ public:
      *  information. */
     void ReportHeadersPresync(const arith_uint256& work, int64_t height, int64_t timestamp);
 
+    //! When starting up, search the datadir for a chainstate based on a UTXO
+    //! snapshot that is in the process of being validated.
+    bool DetectSnapshotChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+    void ResetChainstates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+    //! Switch the active chainstate to one based on a UTXO snapshot that was loaded
+    //! previously.
+    Chainstate& ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash)
+        EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
     ~ChainstateManager();
 };
 

test/functional/feature_init.py

@@ -55,7 +55,6 @@ class InitStressTest(BitcoinTestFramework):
             b'Loading P2P addresses',
             b'Loading banlist',
             b'Loading block index',
-            b'Switching active chainstate',
             b'Checking all blk files are present',
             b'Loaded best chain:',
             b'init message: Verifying blocks',

test/lint/lint-circular-dependencies.py

@@ -14,6 +14,7 @@ import sys
 EXPECTED_CIRCULAR_DEPENDENCIES = (
     "chainparamsbase -> util/system -> chainparamsbase",
     "node/blockstorage -> validation -> node/blockstorage",
+    "node/utxo_snapshot -> validation -> node/utxo_snapshot",
     "policy/fees -> txmempool -> policy/fees",
     "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel",
     "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel",