Auto reindex when old database is detected

This implements automatic reindexing of old database, turned on by default. When an old database is detected and `auto_reindex` is turned on the database will be destroyed so that `electrs` can re-sync. The user can still turn this off to check db format. The help message says "or inconsistent" - this is not implemented but we could in the future and it seems reasonable to explicitly state that this option will control that thing as well. Note: 0.8.x versions of `electrs` didn't contain format information - so we check if the database is empty and assuming it's legacy if it's not empty but without format information. The code is also restructured a bit and a few tests are added.
2024-11-19 01:43:29 +01:00 · 2021-09-13 11:50:21 +02:00 · 2021-09-13 11:50:21 +02:00 · 577f54195b
commit 577f54195b
parent 0a04888673
6 changed files with 205 additions and 29 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -348,6 +348,7 @@ dependencies = [
 "serde_derive",
 "serde_json",
 "signal-hook",
+ "tempfile",
 "tiny_http",
 ]

@ -671,6 +672,12 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"

+[[package]]
+name = "ppv-lite86"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.29"
@ -740,9 +747,9 @@ checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
 dependencies = [
 "autocfg 0.1.7",
 "libc",
- "rand_chacha",
+ "rand_chacha 0.1.1",
 "rand_core 0.4.2",
- "rand_hc",
+ "rand_hc 0.1.0",
 "rand_isaac",
 "rand_jitter",
 "rand_os",
@ -751,6 +758,18 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "rand"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+ "rand_hc 0.3.0",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.1.1"
@ -761,6 +780,16 @@ dependencies = [
 "rand_core 0.3.1",
 ]

+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.3",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.3.1"
@ -776,6 +805,15 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"

+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.1.0"
@ -785,6 +823,15 @@ dependencies = [
 "rand_core 0.3.1",
 ]

+[[package]]
+name = "rand_hc"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
+dependencies = [
+ "rand_core 0.6.3",
+]
+
 [[package]]
 name = "rand_isaac"
 version = "0.1.1"
@ -908,6 +955,15 @@ version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"

+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "rocksdb"
 version = "0.15.0"
@ -947,7 +1003,7 @@ version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97d03ceae636d0fed5bae6a7f4f664354c5f4fcedf6eef053fef17e49f837d0a"
 dependencies = [
- "rand",
+ "rand 0.6.5",
 "secp256k1-sys",
 "serde",
 ]
@ -1034,6 +1090,20 @@ dependencies = [
 "unicode-xid",
 ]

+[[package]]
+name = "tempfile"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "rand 0.8.4",
+ "redox_syscall",
+ "remove_dir_all",
+ "winapi",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.2"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -52,3 +52,6 @@ features = ["zstd", "snappy"]

 [build-dependencies]
 configure_me_codegen = "0.4"
+
+[dev-dependencies]
+tempfile = "3.2"
--- a/internal/config_specification.toml
+++ b/internal/config_specification.toml
@ -17,6 +17,11 @@ count = true
 name = "timestamp"
 doc = "Prepend log lines with a timestamp"

+[[switch]]
+name = "auto_reindex"
+doc = "Automatically reindex the database if it's inconsistent or in old format"
+default = true
+
 [[param]]
 name = "db_dir"
 type = "std::path::PathBuf"
--- a/src/config.rs
+++ b/src/config.rs
@ -132,6 +132,7 @@ pub struct Config {
    pub wait_duration: Duration,
    pub index_batch_size: usize,
    pub index_lookup_limit: Option<usize>,
+    pub auto_reindex: bool,
    pub ignore_mempool: bool,
    pub sync_once: bool,
    pub server_banner: String,
@ -294,6 +295,7 @@ impl Config {
            wait_duration: Duration::from_secs(config.wait_duration_secs),
            index_batch_size: config.index_batch_size,
            index_lookup_limit,
+            auto_reindex: config.auto_reindex,
            ignore_mempool: config.ignore_mempool,
            sync_once: config.sync_once,
            server_banner: config.server_banner,
--- a/src/db.rs
+++ b/src/db.rs
@ -31,9 +31,7 @@ struct Options {
 /// RocksDB wrapper for index storage
 pub struct DBStore {
    db: rocksdb::DB,
-    path: PathBuf,
    bulk_import: AtomicBool,
-    cfs: Vec<&'static str>,
 }

 const CONFIG_CF: &str = "config";
@ -42,6 +40,8 @@ const TXID_CF: &str = "txid";
 const FUNDING_CF: &str = "funding";
 const SPENDING_CF: &str = "spending";

+const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF];
+
 const CONFIG_KEY: &str = "C";
 const TIP_KEY: &[u8] = b"T";

@ -53,6 +53,15 @@ struct Config {

 const CURRENT_FORMAT: u64 = 0;

+impl Default for Config {
+    fn default() -> Self {
+        Config {
+            compacted: false,
+            format: CURRENT_FORMAT,
+        }
+    }
+}
+
 fn default_opts() -> rocksdb::Options {
    let mut opts = rocksdb::Options::default();
    opts.set_keep_log_file_num(10);
@ -68,19 +77,20 @@ fn default_opts() -> rocksdb::Options {
 }

 impl DBStore {
-    /// Opens a new RocksDB at the specified location.
-    pub fn open(path: &Path) -> Result<Self> {
-        let cfs = vec![CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF];
-        let cf_descriptors: Vec<rocksdb::ColumnFamilyDescriptor> = cfs
+    fn create_cf_descriptors() -> Vec<rocksdb::ColumnFamilyDescriptor> {
+        COLUMN_FAMILIES
            .iter()
            .map(|&name| rocksdb::ColumnFamilyDescriptor::new(name, default_opts()))
-            .collect();
+            .collect()
+    }

+    fn open_internal(path: &Path) -> Result<Self> {
        let mut db_opts = default_opts();
        db_opts.create_if_missing(true);
        db_opts.create_missing_column_families(true);
-        let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, cf_descriptors)
-            .with_context(|| format!("failed to open DB: {:?}", path))?;
+
+        let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, Self::create_cf_descriptors())
+            .with_context(|| format!("failed to open DB: {}", path.display()))?;
        let live_files = db.live_files()?;
        info!(
            "{:?}: {} SST files, {} GB, {} Grows",
@ -91,15 +101,55 @@ impl DBStore {
        );
        let store = DBStore {
            db,
-            path: path.to_path_buf(),
-            cfs,
            bulk_import: AtomicBool::new(true),
        };
+        Ok(store)
+    }

+    fn is_legacy_format(&self) -> bool {
+        // In legacy DB format, all data was stored in a single (default) column family.
+        self.db
+            .iterator(rocksdb::IteratorMode::Start)
+            .next()
+            .is_some()
+    }
+
+    /// Opens a new RocksDB at the specified location.
+    pub fn open(path: &Path, auto_reindex: bool) -> Result<Self> {
+        let mut store = Self::open_internal(path)?;
        let config = store.get_config();
        debug!("DB {:?}", config);
-        if config.format != CURRENT_FORMAT {
-            bail!("unsupported DB format {}, re-index required", config.format);
+        let mut config = config.unwrap_or_default(); // use default config when DB is empty
+
+        let reindex_cause = if store.is_legacy_format() {
+            Some("legacy format".to_owned())
+        } else if config.format != CURRENT_FORMAT {
+            Some(format!(
+                "unsupported format {} != {}",
+                config.format, CURRENT_FORMAT
+            ))
+        } else {
+            None
+        };
+        if let Some(cause) = reindex_cause {
+            if !auto_reindex {
+                bail!("re-index required due to {}", cause);
+            }
+            warn!(
+                "Database needs to be re-indexed due to {}, going to delete {}",
+                cause,
+                path.display()
+            );
+            // close DB before deletion
+            drop(store);
+            rocksdb::DB::destroy(&default_opts(), &path).with_context(|| {
+                format!(
+                    "re-index required but the old database ({}) can not be deleted",
+                    path.display()
+                )
+            })?;
+            store = Self::open_internal(path)?;
+            config = Config::default(); // re-init config after dropping DB
        }
        if config.compacted {
            store.start_compactions();
@ -190,13 +240,13 @@ impl DBStore {
    }

    pub(crate) fn flush(&self) {
-        let mut config = self.get_config();
-        for name in &self.cfs {
+        let mut config = self.get_config().unwrap_or_default();
+        for name in COLUMN_FAMILIES {
            let cf = self.db.cf_handle(name).expect("missing CF");
            self.db.flush_cf(cf).expect("CF flush failed");
        }
        if !config.compacted {
-            for name in &self.cfs {
+            for name in COLUMN_FAMILIES {
                info!("starting {} compaction", name);
                let cf = self.db.cf_handle(name).expect("missing CF");
                self.db.compact_range_cf(cf, None::<&[u8]>, None::<&[u8]>);
@ -220,7 +270,7 @@ impl DBStore {

    fn start_compactions(&self) {
        self.bulk_import.store(false, Ordering::Relaxed);
-        for name in &self.cfs {
+        for name in COLUMN_FAMILIES {
            let cf = self.db.cf_handle(name).expect("missing CF");
            self.db
                .set_options_cf(cf, &[("disable_auto_compactions", "false")])
@ -239,15 +289,11 @@ impl DBStore {
            .expect("DB::put failed");
    }

-    fn get_config(&self) -> Config {
+    fn get_config(&self) -> Option<Config> {
        self.db
            .get_cf(self.config_cf(), CONFIG_KEY)
            .expect("DB::get failed")
            .map(|value| serde_json::from_slice(&value).expect("failed to deserialize Config"))
-            .unwrap_or_else(|| Config {
-                compacted: false,
-                format: CURRENT_FORMAT,
-            })
    }
 }

@ -275,6 +321,58 @@ impl<'a> Iterator for ScanIterator<'a> {

 impl Drop for DBStore {
    fn drop(&mut self) {
-        info!("closing DB at {:?}", self.path);
+        info!("closing DB at {}", self.db.path().display());
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DBStore, CURRENT_FORMAT};
+
+    #[test]
+    fn test_reindex_new_format() {
+        let dir = tempfile::tempdir().unwrap();
+        {
+            let store = DBStore::open(dir.path(), false).unwrap();
+            let mut config = store.get_config().unwrap();
+            config.format += 1;
+            store.set_config(config);
+        };
+        assert_eq!(
+            DBStore::open(dir.path(), false).err().unwrap().to_string(),
+            format!(
+                "re-index required due to unsupported format {} != {}",
+                CURRENT_FORMAT + 1,
+                CURRENT_FORMAT
+            )
+        );
+        {
+            let store = DBStore::open(dir.path(), true).unwrap();
+            store.flush();
+            let config = store.get_config().unwrap();
+            assert_eq!(config.format, CURRENT_FORMAT);
+            assert_eq!(store.is_legacy_format(), false);
+        }
+    }
+
+    #[test]
+    fn test_reindex_legacy_format() {
+        let dir = tempfile::tempdir().unwrap();
+        {
+            let mut db_opts = rocksdb::Options::default();
+            db_opts.create_if_missing(true);
+            let db = rocksdb::DB::open(&db_opts, dir.path()).unwrap();
+            db.put(b"F", b"").unwrap(); // insert legacy DB compaction marker (in 'default' column family)
+        };
+        assert_eq!(
+            DBStore::open(dir.path(), false).err().unwrap().to_string(),
+            format!("re-index required due to legacy format",)
+        );
+        {
+            let store = DBStore::open(dir.path(), true).unwrap();
+            store.flush();
+            let config = store.get_config().unwrap();
+            assert_eq!(config.format, CURRENT_FORMAT);
+        }
    }
 }
--- a/src/tracker.rs
+++ b/src/tracker.rs
@ -1,8 +1,6 @@
 use anyhow::{Context, Result};
 use bitcoin::{BlockHash, Txid};

-use std::path::Path;
-
 use crate::{
    cache::Cache,
    chain::Chain,
@ -27,7 +25,7 @@ pub struct Tracker {
 impl Tracker {
    pub fn new(config: &Config) -> Result<Self> {
        let metrics = Metrics::new(config.monitoring_addr)?;
-        let store = DBStore::open(Path::new(&config.db_path))?;
+        let store = DBStore::open(&config.db_path, config.auto_reindex)?;
        let chain = Chain::new(config.network);
        Ok(Self {
            index: Index::load(store, chain, &metrics, config.index_lookup_limit)