mirror of
https://github.com/lightningdevkit/rust-lightning.git
synced 2025-03-15 15:39:09 +01:00
Merge pull request #3481 from tnull/2024-12-add-kvstore-migration-ext
Add `MigratableKVStore` trait
This commit is contained in:
commit
dcc531ebed
3 changed files with 253 additions and 77 deletions
|
@ -1,7 +1,7 @@
|
|||
//! Objects related to [`FilesystemStore`] live here.
|
||||
use crate::utils::{check_namespace_key_validity, is_valid_kvstore_str};
|
||||
|
||||
use lightning::util::persist::KVStore;
|
||||
use lightning::util::persist::{KVStore, MigratableKVStore};
|
||||
use lightning::util::string::PrintableString;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
@ -316,84 +316,13 @@ impl KVStore for FilesystemStore {
|
|||
let entry = entry?;
|
||||
let p = entry.path();
|
||||
|
||||
if let Some(ext) = p.extension() {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
// Clean up any trash files lying around.
|
||||
if ext == "trash" {
|
||||
fs::remove_file(p).ok();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if ext == "tmp" {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let metadata = p.metadata()?;
|
||||
|
||||
// We allow the presence of directories in the empty primary namespace and just skip them.
|
||||
if metadata.is_dir() {
|
||||
if !dir_entry_is_key(&p)? {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If we otherwise don't find a file at the given path something went wrong.
|
||||
if !metadata.is_file() {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of {}/{}: file couldn't be accessed.",
|
||||
PrintableString(primary_namespace),
|
||||
PrintableString(secondary_namespace)
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of {}/{}: file couldn't be accessed.",
|
||||
PrintableString(primary_namespace),
|
||||
PrintableString(secondary_namespace)
|
||||
);
|
||||
return Err(lightning::io::Error::new(lightning::io::ErrorKind::Other, msg));
|
||||
}
|
||||
let key = get_key_from_dir_entry(&p, &prefixed_dest)?;
|
||||
|
||||
match p.strip_prefix(&prefixed_dest) {
|
||||
Ok(stripped_path) => {
|
||||
if let Some(relative_path) = stripped_path.to_str() {
|
||||
if is_valid_kvstore_str(relative_path) {
|
||||
keys.push(relative_path.to_string())
|
||||
}
|
||||
} else {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of {}/{}: file path is not valid UTF-8",
|
||||
PrintableString(primary_namespace),
|
||||
PrintableString(secondary_namespace)
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of {}/{}: file path is not valid UTF-8",
|
||||
PrintableString(primary_namespace),
|
||||
PrintableString(secondary_namespace)
|
||||
);
|
||||
return Err(lightning::io::Error::new(
|
||||
lightning::io::ErrorKind::Other,
|
||||
msg,
|
||||
));
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of {}/{}: {}",
|
||||
PrintableString(primary_namespace),
|
||||
PrintableString(secondary_namespace),
|
||||
e
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of {}/{}: {}",
|
||||
PrintableString(primary_namespace),
|
||||
PrintableString(secondary_namespace),
|
||||
e
|
||||
);
|
||||
return Err(lightning::io::Error::new(lightning::io::ErrorKind::Other, msg));
|
||||
},
|
||||
}
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
self.garbage_collect_locks();
|
||||
|
@ -402,10 +331,172 @@ impl KVStore for FilesystemStore {
|
|||
}
|
||||
}
|
||||
|
||||
fn dir_entry_is_key(p: &Path) -> Result<bool, lightning::io::Error> {
|
||||
if let Some(ext) = p.extension() {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
// Clean up any trash files lying around.
|
||||
if ext == "trash" {
|
||||
fs::remove_file(p).ok();
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
if ext == "tmp" {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
let metadata = p.metadata().map_err(|e| {
|
||||
let msg = format!(
|
||||
"Failed to list keys at path {}: {}",
|
||||
PrintableString(p.to_str().unwrap_or_default()),
|
||||
e
|
||||
);
|
||||
lightning::io::Error::new(lightning::io::ErrorKind::Other, msg)
|
||||
})?;
|
||||
|
||||
// We allow the presence of directories in the empty primary namespace and just skip them.
|
||||
if metadata.is_dir() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// If we otherwise don't find a file at the given path something went wrong.
|
||||
if !metadata.is_file() {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys at path {}: file couldn't be accessed.",
|
||||
PrintableString(p.to_str().unwrap_or_default())
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys at path {}: file couldn't be accessed.",
|
||||
PrintableString(p.to_str().unwrap_or_default())
|
||||
);
|
||||
return Err(lightning::io::Error::new(lightning::io::ErrorKind::Other, msg));
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn get_key_from_dir_entry(p: &Path, base_path: &Path) -> Result<String, lightning::io::Error> {
|
||||
match p.strip_prefix(&base_path) {
|
||||
Ok(stripped_path) => {
|
||||
if let Some(relative_path) = stripped_path.to_str() {
|
||||
if is_valid_kvstore_str(relative_path) {
|
||||
return Ok(relative_path.to_string());
|
||||
} else {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of path {}: file path is not valid key",
|
||||
PrintableString(p.to_str().unwrap_or_default())
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of path {}: file path is not valid key",
|
||||
PrintableString(p.to_str().unwrap_or_default())
|
||||
);
|
||||
return Err(lightning::io::Error::new(lightning::io::ErrorKind::Other, msg));
|
||||
}
|
||||
} else {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of path {}: file path is not valid UTF-8",
|
||||
PrintableString(p.to_str().unwrap_or_default())
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of path {}: file path is not valid UTF-8",
|
||||
PrintableString(p.to_str().unwrap_or_default())
|
||||
);
|
||||
return Err(lightning::io::Error::new(lightning::io::ErrorKind::Other, msg));
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of path {}: {}",
|
||||
PrintableString(p.to_str().unwrap_or_default()),
|
||||
e
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of path {}: {}",
|
||||
PrintableString(p.to_str().unwrap_or_default()),
|
||||
e
|
||||
);
|
||||
return Err(lightning::io::Error::new(lightning::io::ErrorKind::Other, msg));
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
impl MigratableKVStore for FilesystemStore {
|
||||
fn list_all_keys(&self) -> Result<Vec<(String, String, String)>, lightning::io::Error> {
|
||||
let prefixed_dest = &self.data_dir;
|
||||
if !prefixed_dest.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut keys = Vec::new();
|
||||
|
||||
'primary_loop: for primary_entry in fs::read_dir(prefixed_dest)? {
|
||||
let primary_path = primary_entry?.path();
|
||||
|
||||
if dir_entry_is_key(&primary_path)? {
|
||||
let primary_namespace = String::new();
|
||||
let secondary_namespace = String::new();
|
||||
let key = get_key_from_dir_entry(&primary_path, prefixed_dest)?;
|
||||
keys.push((primary_namespace, secondary_namespace, key));
|
||||
continue 'primary_loop;
|
||||
}
|
||||
|
||||
// The primary_entry is actually also a directory.
|
||||
'secondary_loop: for secondary_entry in fs::read_dir(&primary_path)? {
|
||||
let secondary_path = secondary_entry?.path();
|
||||
|
||||
if dir_entry_is_key(&secondary_path)? {
|
||||
let primary_namespace = get_key_from_dir_entry(&primary_path, prefixed_dest)?;
|
||||
let secondary_namespace = String::new();
|
||||
let key = get_key_from_dir_entry(&secondary_path, &primary_path)?;
|
||||
keys.push((primary_namespace, secondary_namespace, key));
|
||||
continue 'secondary_loop;
|
||||
}
|
||||
|
||||
// The secondary_entry is actually also a directory.
|
||||
for tertiary_entry in fs::read_dir(&secondary_path)? {
|
||||
let tertiary_entry = tertiary_entry?;
|
||||
let tertiary_path = tertiary_entry.path();
|
||||
|
||||
if dir_entry_is_key(&tertiary_path)? {
|
||||
let primary_namespace =
|
||||
get_key_from_dir_entry(&primary_path, prefixed_dest)?;
|
||||
let secondary_namespace =
|
||||
get_key_from_dir_entry(&secondary_path, &primary_path)?;
|
||||
let key = get_key_from_dir_entry(&tertiary_path, &secondary_path)?;
|
||||
keys.push((primary_namespace, secondary_namespace, key));
|
||||
} else {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Failed to list keys of path {}: only two levels of namespaces are supported",
|
||||
PrintableString(tertiary_path.to_str().unwrap_or_default())
|
||||
);
|
||||
let msg = format!(
|
||||
"Failed to list keys of path {}: only two levels of namespaces are supported",
|
||||
PrintableString(tertiary_path.to_str().unwrap_or_default())
|
||||
);
|
||||
return Err(lightning::io::Error::new(
|
||||
lightning::io::ErrorKind::Other,
|
||||
msg,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::test_utils::{do_read_write_remove_list_persist, do_test_store};
|
||||
use crate::test_utils::{
|
||||
do_read_write_remove_list_persist, do_test_data_migration, do_test_store,
|
||||
};
|
||||
|
||||
use bitcoin::Txid;
|
||||
|
||||
|
@ -438,6 +529,19 @@ mod tests {
|
|||
do_read_write_remove_list_persist(&fs_store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_migration() {
|
||||
let mut source_temp_path = std::env::temp_dir();
|
||||
source_temp_path.push("test_data_migration_source");
|
||||
let mut source_store = FilesystemStore::new(source_temp_path);
|
||||
|
||||
let mut target_temp_path = std::env::temp_dir();
|
||||
target_temp_path.push("test_data_migration_target");
|
||||
let mut target_store = FilesystemStore::new(target_temp_path);
|
||||
|
||||
do_test_data_migration(&mut source_store, &mut target_store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_if_monitors_is_not_dir() {
|
||||
let store = FilesystemStore::new("test_monitors_is_not_dir".into());
|
||||
|
|
|
@ -3,7 +3,10 @@ use lightning::ln::functional_test_utils::{
|
|||
connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block,
|
||||
create_network, create_node_cfgs, create_node_chanmgrs, send_payment,
|
||||
};
|
||||
use lightning::util::persist::{read_channel_monitors, KVStore, KVSTORE_NAMESPACE_KEY_MAX_LEN};
|
||||
use lightning::util::persist::{
|
||||
migrate_kv_store_data, read_channel_monitors, KVStore, MigratableKVStore,
|
||||
KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN,
|
||||
};
|
||||
use lightning::util::test_utils;
|
||||
use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event};
|
||||
|
||||
|
@ -59,6 +62,41 @@ pub(crate) fn do_read_write_remove_list_persist<K: KVStore + RefUnwindSafe>(kv_s
|
|||
assert_eq!(listed_keys.len(), 0);
|
||||
}
|
||||
|
||||
pub(crate) fn do_test_data_migration<S: MigratableKVStore, T: MigratableKVStore>(
|
||||
source_store: &mut S, target_store: &mut T,
|
||||
) {
|
||||
// We fill the source with some bogus keys.
|
||||
let dummy_data = [42u8; 32];
|
||||
let num_primary_namespaces = 2;
|
||||
let num_secondary_namespaces = 2;
|
||||
let num_keys = 3;
|
||||
for i in 0..num_primary_namespaces {
|
||||
let primary_namespace =
|
||||
format!("testspace{}", KVSTORE_NAMESPACE_KEY_ALPHABET.chars().nth(i).unwrap());
|
||||
for j in 0..num_secondary_namespaces {
|
||||
let secondary_namespace =
|
||||
format!("testsubspace{}", KVSTORE_NAMESPACE_KEY_ALPHABET.chars().nth(j).unwrap());
|
||||
for k in 0..num_keys {
|
||||
let key =
|
||||
format!("testkey{}", KVSTORE_NAMESPACE_KEY_ALPHABET.chars().nth(k).unwrap());
|
||||
source_store
|
||||
.write(&primary_namespace, &secondary_namespace, &key, &dummy_data)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
let total_num_entries = num_primary_namespaces * num_secondary_namespaces * num_keys;
|
||||
let all_keys = source_store.list_all_keys().unwrap();
|
||||
assert_eq!(all_keys.len(), total_num_entries);
|
||||
|
||||
migrate_kv_store_data(source_store, target_store).unwrap();
|
||||
|
||||
assert_eq!(target_store.list_all_keys().unwrap().len(), total_num_entries);
|
||||
for (p, s, k) in &all_keys {
|
||||
assert_eq!(target_store.read(p, s, k).unwrap(), dummy_data);
|
||||
}
|
||||
}
|
||||
|
||||
// Integration-test the given KVStore implementation. Test relaying a few payments and check that
|
||||
// the persisted data is updated the appropriate number of times.
|
||||
pub(crate) fn do_test_store<K: KVStore>(store_0: &K, store_1: &K) {
|
||||
|
|
|
@ -163,6 +163,40 @@ pub trait KVStore {
|
|||
) -> Result<Vec<String>, io::Error>;
|
||||
}
|
||||
|
||||
/// Provides additional interface methods that are required for [`KVStore`]-to-[`KVStore`]
|
||||
/// data migration.
|
||||
pub trait MigratableKVStore: KVStore {
|
||||
/// Returns *all* known keys as a list of `primary_namespace`, `secondary_namespace`, `key` tuples.
|
||||
///
|
||||
/// This is useful for migrating data from [`KVStore`] implementation to [`KVStore`]
|
||||
/// implementation.
|
||||
///
|
||||
/// Must exhaustively return all entries known to the store to ensure no data is missed, but
|
||||
/// may return the items in arbitrary order.
|
||||
fn list_all_keys(&self) -> Result<Vec<(String, String, String)>, io::Error>;
|
||||
}
|
||||
|
||||
/// Migrates all data from one store to another.
|
||||
///
|
||||
/// This operation assumes that `target_store` is empty, i.e., any data present under copied keys
|
||||
/// might get overriden. User must ensure `source_store` is not modified during operation,
|
||||
/// otherwise no consistency guarantees can be given.
|
||||
///
|
||||
/// Will abort and return an error if any IO operation fails. Note that in this case the
|
||||
/// `target_store` might get left in an intermediate state.
|
||||
pub fn migrate_kv_store_data<S: MigratableKVStore, T: MigratableKVStore>(
|
||||
source_store: &mut S, target_store: &mut T,
|
||||
) -> Result<(), io::Error> {
|
||||
let keys_to_migrate = source_store.list_all_keys()?;
|
||||
|
||||
for (primary_namespace, secondary_namespace, key) in &keys_to_migrate {
|
||||
let data = source_store.read(primary_namespace, secondary_namespace, key)?;
|
||||
target_store.write(primary_namespace, secondary_namespace, key, &data)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Trait that handles persisting a [`ChannelManager`], [`NetworkGraph`], and [`WriteableScore`] to disk.
|
||||
///
|
||||
/// [`ChannelManager`]: crate::ln::channelmanager::ChannelManager
|
||||
|
|
Loading…
Add table
Reference in a new issue