mirrors/electrs
1
0
Fork 0
mirror of https://github.com/romanz/electrs.git synced 2024-11-19 09:54:09 +01:00
Code Issues Projects Releases Wiki Activity

Remove sstable ingestion support

Browse Source 
Currently, it's easier to use regular batch writes (without auto
compactions) during bulk indexing.
This commit is contained in:
Roman Zeyde 2018-07-16 12:03:02 +03:00
parent c7948e9972
commit d14746a36b
No known key found for this signature in database
GPG Key ID: 87CAE5FA46917CBB
1 changed files with 1 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/store.rs
View File

@ -1,7 +1,7 @@
use rocksdb;
use rocksdb::Writable;
use std::path::{Path, PathBuf};
use std::path::Path;
use util::Bytes;
@ -70,24 +70,10 @@ impl DBStore {
self
}
pub fn sstable(&self) -> SSTableWriter {
SSTableWriter::new()
}
pub fn put(&self, key: &[u8], value: &[u8]) {
self.db.put(key, value).unwrap();
}
pub fn ingest(&self, sstables: &[PathBuf]) {
let mut opts = rocksdb::IngestExternalFileOptions::new();
opts.move_files(true);
let sstables: Vec<&str> = sstables.iter().map(|path| path.to_str().unwrap()).collect();
info!("ingesting {} SSTables", sstables.len());
self.db
.ingest_external_file(&opts, &sstables)
.expect("failed to ingest SSTables")
}
pub fn compact(&self) {
info!("starting full compaction");
self.db.compact_range(None, None); // would take a while
@ -141,35 +127,3 @@ impl Drop for DBStore {
trace!("closing DB");
}
}
pub struct SSTableWriter {
writer: rocksdb::SstFileWriter,
}
impl SSTableWriter {
fn new() -> Self {
let mut cf_opts = rocksdb::ColumnFamilyOptions::new();
cf_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
cf_opts.compression(rocksdb::DBCompressionType::Snappy);
SSTableWriter {
writer: rocksdb::SstFileWriter::new(rocksdb::EnvOptions::new(), cf_opts),
}
}
pub fn build(mut self, path: &Path, mut rows: Vec<Row>) {
rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
rows.dedup_by(|a, b| a.key.eq(&b.key)); // SSTableWriter requires ascending keys.
let path = path.to_str().unwrap();
self.writer
.open(path)
.expect(&format!("failed to open SSTable {}", path));
for row in &rows {
self.writer
.put(row.key.as_slice(), row.value.as_slice())
.expect(&format!("failed to write SSTable {}", path));
}
self.writer
.finish()
.expect(&format!("failed to close SSTable {}", path));
}
}