add lmdb backend

Cargo.toml

@@ -64,6 +64,7 @@ panic_hook = { path = "util/panic-hook" }
 keccak-hash = "0.1"
 migration-rocksdb = { path = "util/migration-rocksdb" }
 kvdb = "0.1"
+kvdb-lmdb = "0.1"
 kvdb-rocksdb = "0.1.3"
 journaldb = { path = "util/journaldb" }
 
@@ -137,3 +138,7 @@ members = [
 
 [patch.crates-io]
 heapsize = { git = "https://github.com/cheme/heapsize.git", branch = "ec-macfix" }
+# TODO: remove when kvdb-lmdb 0.1 is released
+kvdb = { git = "https://github.com/paritytech/parity-common", branch = "lmdb" }
+kvdb-lmdb = { git = "https://github.com/paritytech/parity-common", branch = "lmdb" }
+kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", branch = "lmdb" }

ethcore/src/client/config.rs

@@ -27,7 +27,7 @@ pub use trace::Config as TraceConfig;
 pub use evm::VMType;
 
 /// Client state db compaction profile
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum DatabaseCompactionProfile {
 	/// Try to determine compaction profile automatically
 	Auto,
@@ -82,6 +82,20 @@ impl Display for Mode {
 	}
 }
 
+/// Which database to use.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum DatabaseBackend {
+	/// LMDB
+	Lmdb,
+	/// RocksDB
+	RocksDB {
+		/// RocksDB column cache-size if not default
+		db_cache_size: Option<usize>,
+		/// State db compaction profile
+		db_compaction: DatabaseCompactionProfile,
+	},
+}
+
 /// Client configuration. Includes configs for all sub-systems.
 #[derive(Debug, PartialEq, Clone)]
 pub struct ClientConfig {
@@ -99,10 +113,8 @@ pub struct ClientConfig {
 	pub pruning: journaldb::Algorithm,
 	/// The name of the client instance.
 	pub name: String,
-	/// RocksDB column cache-size if not default
-	pub db_cache_size: Option<usize>,
-	/// State db compaction profile
-	pub db_compaction: DatabaseCompactionProfile,
+	/// The database backend.
+	pub db_backend: DatabaseBackend,
 	/// Operating mode
 	pub mode: Mode,
 	/// The chain spec name
@@ -138,8 +150,10 @@ impl Default for ClientConfig {
 			fat_db: false,
 			pruning: journaldb::Algorithm::OverlayRecent,
 			name: "default".into(),
-			db_cache_size: None,
-			db_compaction: Default::default(),
+			db_backend: DatabaseBackend::RocksDB {
+				db_cache_size: None,
+				db_compaction: Default::default(),
+			},
 			mode: Mode::Active,
 			spec_name: "".into(),
 			verifier_type: VerifierType::Canon,

ethcore/src/client/mod.rs

@@ -28,7 +28,7 @@ mod test_client;
 mod trace;
 
 pub use self::client::*;
-pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType};
+pub use self::config::{Mode, ClientConfig, DatabaseBackend, DatabaseCompactionProfile, BlockChainConfig, VMType};
 #[cfg(any(test, feature = "test-helpers"))]
 pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactResult};
 pub use self::io_message::ClientIoMessage;

parity/cache.rs

@@ -27,7 +27,7 @@ const DEFAULT_STATE_CACHE_SIZE: u32 = 25;
 
 /// Configuration for application cache sizes.
 /// All	values are represented in MB.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub struct CacheConfig {
 	/// Size of rocksDB cache. Almost all goes to the state column.
 	db: u32,

parity/db/rocksdb/blooms.rs → parity/db/impls/blooms.rs

@@ -20,14 +20,13 @@ use std::path::Path;
 use ethereum_types::Bloom;
 use ethcore::error::Error;
 use rlp;
-use super::kvdb_rocksdb::DatabaseConfig;
-use super::open_database;
+use super::{KvdbBackend, open_database};
 
 const LOG_BLOOMS_ELEMENTS_PER_INDEX: u64 = 16;
 
-pub fn migrate_blooms<P: AsRef<Path>>(path: P, config: &DatabaseConfig) -> Result<(), Error> {
+pub fn migrate_blooms<P: AsRef<Path>>(path: P, backend: KvdbBackend) -> Result<(), Error> {
 	// init
-	let db = open_database(&path.as_ref().to_string_lossy(), config)?;
+	let db = open_database(&path.as_ref().to_string_lossy(), backend)?;
 
 	// possible optimization:
 	// pre-allocate space on disk for faster migration

parity/db/rocksdb/helpers.rs → parity/db/impls/helpers.rs

@@ -16,8 +16,9 @@
 
 use std::path::Path;
 use ethcore_db::NUM_COLUMNS;
-use ethcore::client::{ClientConfig, DatabaseCompactionProfile};
+use ethcore::client::{ClientConfig, DatabaseCompactionProfile, DatabaseBackend};
 use super::kvdb_rocksdb::{CompactionProfile, DatabaseConfig};
+use super::KvdbBackend;
 
 pub fn compaction_profile(profile: &DatabaseCompactionProfile, db_path: &Path) -> CompactionProfile {
 	match profile {
@@ -27,11 +28,16 @@ pub fn compaction_profile(profile: &DatabaseCompactionProfile, db_path: &Path) -
 	}
 }
 
-pub fn client_db_config(client_path: &Path, client_config: &ClientConfig) -> DatabaseConfig {
-	let mut client_db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
-
-	client_db_config.memory_budget = client_config.db_cache_size;
-	client_db_config.compaction = compaction_profile(&client_config.db_compaction, &client_path);
-
-	client_db_config
+pub fn client_db_config(client_path: &Path, client_config: &ClientConfig) -> KvdbBackend {
+	match client_config.db_backend {
+		DatabaseBackend::Lmdb => KvdbBackend::Lmdb,
+		DatabaseBackend::RocksDB { ref db_cache_size, ref db_compaction } => {
+			let mut config = DatabaseConfig::with_columns(NUM_COLUMNS);
+
+			config.memory_budget = db_cache_size;
+			config.compaction = compaction_profile(db_compaction, &client_path);
+
+			KvdbBackend::RocksDB { config }
+		} 
+	}
 }

parity/db/rocksdb/mod.rs → parity/db/impls/mod.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
+extern crate kvdb_lmdb;
 extern crate kvdb_rocksdb;
 extern crate migration_rocksdb;
 extern crate ethcore_blockchain;
@@ -23,18 +24,17 @@ use std::sync::Arc;
 use std::path::Path;
 use blooms_db;
 use ethcore_db::NUM_COLUMNS;
-use ethcore::client::{ClientConfig, DatabaseCompactionProfile};
+use ethcore::client::ClientConfig;
 use kvdb::KeyValueDB;
 use self::ethcore_blockchain::{BlockChainDBHandler, BlockChainDB};
-use self::kvdb_rocksdb::{Database, DatabaseConfig};
-
-use cache::CacheConfig;
+use self::kvdb_rocksdb::{DatabaseConfig, Database as RocksDB};
+use self::kvdb_lmdb::Database as Lmdb;
 
 mod blooms;
-mod migration;
+mod rocksdb_migration;
 mod helpers;
 
-pub use self::migration::migrate;
+pub use self::rocksdb_migration::migrate;
 
 struct AppDB {
 	key_value: Arc<KeyValueDB>,
@@ -58,48 +58,34 @@ impl BlockChainDB for AppDB {
 
 /// Open a secret store DB using the given secret store data path. The DB path is one level beneath the data path.
 #[cfg(feature = "secretstore")]
-pub fn open_secretstore_db(data_path: &str) -> Result<Arc<KeyValueDB>, String> {
+pub fn open_secretstore_db(data_path: &str, backend: KvdbBackend) -> Result<Arc<KeyValueDB>, String> {
 	use std::path::PathBuf;
 
 	let mut db_path = PathBuf::from(data_path);
 	db_path.push("db");
 	let db_path = db_path.to_str().ok_or_else(|| "Invalid secretstore path".to_string())?;
-	Ok(Arc::new(Database::open_default(&db_path).map_err(|e| format!("Error opening database: {:?}", e))?))
+	backend.open(&db_path).map_err(|e| format!("Error opening database: {:?}", e))
 }
 
 /// Create a restoration db handler using the config generated by `client_path` and `client_config`.
 pub fn restoration_db_handler(client_path: &Path, client_config: &ClientConfig) -> Box<BlockChainDBHandler> {
-	let client_db_config = helpers::client_db_config(client_path, client_config);
+	let db_backend = helpers::client_db_config(client_path, client_config);
 
 	struct RestorationDBHandler {
-		config: DatabaseConfig,
+		db_backend: KvdbBackend,
 	}
 
 	impl BlockChainDBHandler for RestorationDBHandler {
 		fn open(&self, db_path: &Path) -> io::Result<Arc<BlockChainDB>> {
-			open_database(&db_path.to_string_lossy(), &self.config)
+			open_database(&db_path.to_string_lossy(), self.db_backend.clone())
 		}
 	}
 
-	Box::new(RestorationDBHandler {
-		config: client_db_config,
-	})
+	Box::new(RestorationDBHandler { db_backend })
 }
 
 /// Open a new main DB.
-pub fn open_db(client_path: &str, cache_config: &CacheConfig, compaction: &DatabaseCompactionProfile) -> io::Result<Arc<BlockChainDB>> {
-	let path = Path::new(client_path);
-
-	let db_config = DatabaseConfig {
-		memory_budget: Some(cache_config.blockchain() as usize * 1024 * 1024),
-		compaction: helpers::compaction_profile(&compaction, path),
-		.. DatabaseConfig::with_columns(NUM_COLUMNS)
-	};
-
-	open_database(client_path, &db_config)
-}
-
-pub fn open_database(client_path: &str, config: &DatabaseConfig) -> io::Result<Arc<BlockChainDB>> {
+pub fn open_database(client_path: &str, backend: KvdbBackend) -> io::Result<Arc<BlockChainDB>> {
 	let path = Path::new(client_path);
 
 	let blooms_path = path.join("blooms");
@@ -108,10 +94,31 @@ pub fn open_database(client_path: &str, config: &DatabaseConfig) -> io::Result<A
 	fs::create_dir_all(&trace_blooms_path)?;
 
 	let db = AppDB {
-		key_value: Arc::new(Database::open(&config, client_path)?),
+		key_value: backend.open(client_path)?,
 		blooms: blooms_db::Database::open(blooms_path)?,
 		trace_blooms: blooms_db::Database::open(trace_blooms_path)?,
 	};
 
 	Ok(Arc::new(db))
 }
+
+/// Which database to use.
+#[derive(Debug, Clone)]
+pub enum KvdbBackend {
+	/// LMDB
+	Lmdb,
+	/// RocksDB
+	RocksDB {
+		/// RocksDB config
+		config: DatabaseConfig,
+	},
+}
+
+impl KvdbBackend {
+	fn open(&self, client_path: &str) -> io::Result<Arc<KeyValueDB>> {
+		Ok(match *self {
+			KvdbBackend::Lmdb => Arc::new(Lmdb::open(client_path, NUM_COLUMNS.unwrap_or_default())?),
+			KvdbBackend::RocksDB { ref config } => Arc::new(RocksDB::open(config, client_path)?),
+		})
+	}
+}

parity/db/mod.rs

@@ -16,10 +16,9 @@
 
 //! Database-related operations.
 
-#[path="rocksdb/mod.rs"]
 mod impls;
 
-pub use self::impls::{open_db, restoration_db_handler, migrate};
+pub use self::impls::{open_database, restoration_db_handler, migrate, KvdbBackend};
 
 #[cfg(feature = "secretstore")]
 pub use self::impls::open_secretstore_db;