diff --git a/Cargo.lock b/Cargo.lock index 9438192..8fd8723 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -348,6 +348,7 @@ dependencies = [ "serde_derive", "serde_json", "signal-hook", + "tempfile", "tiny_http", ] @@ -671,6 +672,12 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + [[package]] name = "proc-macro2" version = "1.0.29" @@ -740,9 +747,9 @@ checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ "autocfg 0.1.7", "libc", - "rand_chacha", + "rand_chacha 0.1.1", "rand_core 0.4.2", - "rand_hc", + "rand_hc 0.1.0", "rand_isaac", "rand_jitter", "rand_os", @@ -751,6 +758,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.1.1" @@ -761,6 +780,16 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -776,6 +805,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + [[package]] name = "rand_hc" version = "0.1.0" @@ -785,6 +823,15 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.3", +] + [[package]] name = "rand_isaac" version = "0.1.1" @@ -908,6 +955,15 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rocksdb" version = "0.15.0" @@ -947,7 +1003,7 @@ version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d03ceae636d0fed5bae6a7f4f664354c5f4fcedf6eef053fef17e49f837d0a" dependencies = [ - "rand", + "rand 0.6.5", "secp256k1-sys", "serde", ] @@ -1034,6 +1090,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.4", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.2" diff --git a/Cargo.toml b/Cargo.toml index 96a7561..a468488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,3 +52,6 @@ features = ["zstd", "snappy"] [build-dependencies] configure_me_codegen = "0.4" + +[dev-dependencies] +tempfile = "3.2" diff --git a/internal/config_specification.toml b/internal/config_specification.toml index 7b937e9..9738e1f 100644 --- a/internal/config_specification.toml +++ b/internal/config_specification.toml @@ -17,6 +17,11 @@ count = true name = "timestamp" doc = "Prepend log lines with a timestamp" +[[switch]] +name = "auto_reindex" +doc = "Automatically reindex the database if it's inconsistent or in old format" +default = true + [[param]] name = "db_dir" type = "std::path::PathBuf" diff --git a/src/config.rs b/src/config.rs index c92572f..46dd31f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -132,6 +132,7 @@ pub struct Config { pub wait_duration: Duration, pub index_batch_size: usize, pub index_lookup_limit: Option, + pub auto_reindex: bool, pub ignore_mempool: bool, pub sync_once: bool, pub server_banner: String, @@ -294,6 +295,7 @@ impl Config { wait_duration: Duration::from_secs(config.wait_duration_secs), index_batch_size: config.index_batch_size, index_lookup_limit, + auto_reindex: config.auto_reindex, ignore_mempool: config.ignore_mempool, sync_once: config.sync_once, server_banner: config.server_banner, diff --git a/src/db.rs b/src/db.rs index 386dbac..03ded1e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -31,9 +31,7 @@ struct Options { /// RocksDB wrapper for index storage pub struct DBStore { db: rocksdb::DB, - path: PathBuf, bulk_import: AtomicBool, - cfs: Vec<&'static str>, } const CONFIG_CF: &str = "config"; @@ -42,6 +40,8 @@ const TXID_CF: &str = "txid"; const FUNDING_CF: &str = "funding"; const SPENDING_CF: &str = "spending"; +const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF]; + const CONFIG_KEY: &str = "C"; const TIP_KEY: &[u8] = b"T"; @@ -53,6 +53,15 @@ struct Config { const CURRENT_FORMAT: u64 = 0; +impl Default for Config { + fn default() -> Self { + Config { + compacted: false, + format: CURRENT_FORMAT, + } + } +} + fn default_opts() -> rocksdb::Options { let mut opts = rocksdb::Options::default(); opts.set_keep_log_file_num(10); @@ -68,19 +77,20 @@ fn default_opts() -> rocksdb::Options { } impl DBStore { - /// Opens a new RocksDB at the specified location. - pub fn open(path: &Path) -> Result { - let cfs = vec![CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF]; - let cf_descriptors: Vec = cfs + fn create_cf_descriptors() -> Vec { + COLUMN_FAMILIES .iter() .map(|&name| rocksdb::ColumnFamilyDescriptor::new(name, default_opts())) - .collect(); + .collect() + } + fn open_internal(path: &Path) -> Result { let mut db_opts = default_opts(); db_opts.create_if_missing(true); db_opts.create_missing_column_families(true); - let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, cf_descriptors) - .with_context(|| format!("failed to open DB: {:?}", path))?; + + let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, Self::create_cf_descriptors()) + .with_context(|| format!("failed to open DB: {}", path.display()))?; let live_files = db.live_files()?; info!( "{:?}: {} SST files, {} GB, {} Grows", @@ -91,15 +101,55 @@ impl DBStore { ); let store = DBStore { db, - path: path.to_path_buf(), - cfs, bulk_import: AtomicBool::new(true), }; + Ok(store) + } + fn is_legacy_format(&self) -> bool { + // In legacy DB format, all data was stored in a single (default) column family. + self.db + .iterator(rocksdb::IteratorMode::Start) + .next() + .is_some() + } + + /// Opens a new RocksDB at the specified location. + pub fn open(path: &Path, auto_reindex: bool) -> Result { + let mut store = Self::open_internal(path)?; let config = store.get_config(); debug!("DB {:?}", config); - if config.format != CURRENT_FORMAT { - bail!("unsupported DB format {}, re-index required", config.format); + let mut config = config.unwrap_or_default(); // use default config when DB is empty + + let reindex_cause = if store.is_legacy_format() { + Some("legacy format".to_owned()) + } else if config.format != CURRENT_FORMAT { + Some(format!( + "unsupported format {} != {}", + config.format, CURRENT_FORMAT + )) + } else { + None + }; + if let Some(cause) = reindex_cause { + if !auto_reindex { + bail!("re-index required due to {}", cause); + } + warn!( + "Database needs to be re-indexed due to {}, going to delete {}", + cause, + path.display() + ); + // close DB before deletion + drop(store); + rocksdb::DB::destroy(&default_opts(), &path).with_context(|| { + format!( + "re-index required but the old database ({}) can not be deleted", + path.display() + ) + })?; + store = Self::open_internal(path)?; + config = Config::default(); // re-init config after dropping DB } if config.compacted { store.start_compactions(); @@ -190,13 +240,13 @@ impl DBStore { } pub(crate) fn flush(&self) { - let mut config = self.get_config(); - for name in &self.cfs { + let mut config = self.get_config().unwrap_or_default(); + for name in COLUMN_FAMILIES { let cf = self.db.cf_handle(name).expect("missing CF"); self.db.flush_cf(cf).expect("CF flush failed"); } if !config.compacted { - for name in &self.cfs { + for name in COLUMN_FAMILIES { info!("starting {} compaction", name); let cf = self.db.cf_handle(name).expect("missing CF"); self.db.compact_range_cf(cf, None::<&[u8]>, None::<&[u8]>); @@ -220,7 +270,7 @@ impl DBStore { fn start_compactions(&self) { self.bulk_import.store(false, Ordering::Relaxed); - for name in &self.cfs { + for name in COLUMN_FAMILIES { let cf = self.db.cf_handle(name).expect("missing CF"); self.db .set_options_cf(cf, &[("disable_auto_compactions", "false")]) @@ -239,15 +289,11 @@ impl DBStore { .expect("DB::put failed"); } - fn get_config(&self) -> Config { + fn get_config(&self) -> Option { self.db .get_cf(self.config_cf(), CONFIG_KEY) .expect("DB::get failed") .map(|value| serde_json::from_slice(&value).expect("failed to deserialize Config")) - .unwrap_or_else(|| Config { - compacted: false, - format: CURRENT_FORMAT, - }) } } @@ -275,6 +321,58 @@ impl<'a> Iterator for ScanIterator<'a> { impl Drop for DBStore { fn drop(&mut self) { - info!("closing DB at {:?}", self.path); + info!("closing DB at {}", self.db.path().display()); + } +} + +#[cfg(test)] +mod tests { + use super::{DBStore, CURRENT_FORMAT}; + + #[test] + fn test_reindex_new_format() { + let dir = tempfile::tempdir().unwrap(); + { + let store = DBStore::open(dir.path(), false).unwrap(); + let mut config = store.get_config().unwrap(); + config.format += 1; + store.set_config(config); + }; + assert_eq!( + DBStore::open(dir.path(), false).err().unwrap().to_string(), + format!( + "re-index required due to unsupported format {} != {}", + CURRENT_FORMAT + 1, + CURRENT_FORMAT + ) + ); + { + let store = DBStore::open(dir.path(), true).unwrap(); + store.flush(); + let config = store.get_config().unwrap(); + assert_eq!(config.format, CURRENT_FORMAT); + assert_eq!(store.is_legacy_format(), false); + } + } + + #[test] + fn test_reindex_legacy_format() { + let dir = tempfile::tempdir().unwrap(); + { + let mut db_opts = rocksdb::Options::default(); + db_opts.create_if_missing(true); + let db = rocksdb::DB::open(&db_opts, dir.path()).unwrap(); + db.put(b"F", b"").unwrap(); // insert legacy DB compaction marker (in 'default' column family) + }; + assert_eq!( + DBStore::open(dir.path(), false).err().unwrap().to_string(), + format!("re-index required due to legacy format",) + ); + { + let store = DBStore::open(dir.path(), true).unwrap(); + store.flush(); + let config = store.get_config().unwrap(); + assert_eq!(config.format, CURRENT_FORMAT); + } } } diff --git a/src/tracker.rs b/src/tracker.rs index f40e581..f671c5b 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,8 +1,6 @@ use anyhow::{Context, Result}; use bitcoin::{BlockHash, Txid}; -use std::path::Path; - use crate::{ cache::Cache, chain::Chain, @@ -27,7 +25,7 @@ pub struct Tracker { impl Tracker { pub fn new(config: &Config) -> Result { let metrics = Metrics::new(config.monitoring_addr)?; - let store = DBStore::open(Path::new(&config.db_path))?; + let store = DBStore::open(&config.db_path, config.auto_reindex)?; let chain = Chain::new(config.network); Ok(Self { index: Index::load(store, chain, &metrics, config.index_lookup_limit)