[secretstore] migrate to version 4

parity/db/rocksdb/migration_secretstore.rs

@@ -0,0 +1,160 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Parity Ethereum.
+
+// Parity Ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Secret Store DB migration module.
+
+
+use std::fmt::{Display, Error as FmtError, Formatter};
+use std::fs;
+use std::io::{Error as IoError, ErrorKind as IoErrorKind, Read as _, Write as _};
+use std::path::PathBuf;
+
+use kvdb::DBTransaction;
+use super::kvdb_rocksdb::{Database, DatabaseConfig};
+
+/// We used to store the version in the database (until version 4).
+const LEGACY_DB_META_KEY_VERSION: &'static [u8; 7] = b"version";
+/// Current db version.
+const CURRENT_VERSION: u8 = 4;
+/// Database is assumed to be at the default version, when no version file is found.
+const DEFAULT_VERSION: u8 = 3;
+/// Version file name.
+const VERSION_FILE_NAME: &'static str = "db_version";
+
+/// Migration related erorrs.
+#[derive(Debug)]
+pub enum Error {
+	/// Returned when current version cannot be read or guessed.
+	UnknownDatabaseVersion,
+	/// Existing DB is newer than the known one.
+	FutureDBVersion,
+	/// Migration was completed succesfully,
+	/// but there was a problem with io.
+	Io(IoError),
+}
+
+impl Display for Error {
+	fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> {
+		let out = match *self {
+			Error::UnknownDatabaseVersion =>
+				"Current Secret Store database version cannot be read".into(),
+			Error::FutureDBVersion =>
+				"Secret Store database was created with newer client version.\
+				Upgrade your client or delete DB and resync.".into(),
+			Error::Io(ref err) =>
+				format!("Unexpected io error on Secret Store database migration: {}.", err),
+		};
+		write!(f, "{}", out)
+	}
+}
+
+impl From<IoError> for Error {
+	fn from(err: IoError) -> Self {
+		Error::Io(err)
+	}
+}
+
+// Moved "default" column to column 0 in preparation for a kvdb-rocksdb 0.3 migration.
+fn migrate_to_v4(parent_dir: &str) -> Result<(), Error> {
+	// Naïve implementation until
+	// https://github.com/facebook/rocksdb/issues/6130 is resolved
+	let old_db_config = DatabaseConfig {
+		columns: None,
+		..Default::default()
+	};
+	let new_db_config = DatabaseConfig {
+		columns: Some(1),
+		..Default::default()
+	};
+	const BATCH_SIZE: usize = 1024;
+	let new_dir = migration_dir(parent_dir);
+	let old_db = Database::open(&old_db_config, &db_dir(parent_dir))?;
+	let new_db = Database::open(&new_db_config, &new_dir)?;
+
+	// remove legacy version key
+	{
+		let mut batch = DBTransaction::with_capacity(1);
+		batch.delete(None, LEGACY_DB_META_KEY_VERSION);
+		old_db.write(batch)?;
+	}
+
+	let mut batch = DBTransaction::with_capacity(BATCH_SIZE);
+	for (i, (key, value)) in old_db.iter(None).enumerate() {
+		batch.put(Some(0), &key, &value);
+		if i % BATCH_SIZE == 0 {
+			new_db.write(batch)?;
+			batch = DBTransaction::with_capacity(BATCH_SIZE);
+		}
+	}
+	new_db.write(batch)?;
+	drop(new_db);
+	old_db.restore(&new_dir)?;
+	Ok(())
+}
+
+/// Apply all migrations if possible.
+pub fn upgrade_db(db_path: &str) -> Result<(), Error> {
+	match current_version(db_path)? {
+		old_version if old_version < CURRENT_VERSION => {
+			migrate_to_v4(db_path)?;
+			update_version(db_path)?;
+			Ok(())
+		},
+		CURRENT_VERSION => Ok(()),
+		_ => Err(Error::FutureDBVersion),
+	}
+}
+
+fn db_dir(path: &str) -> String {
+	let mut dir = PathBuf::from(path);
+	dir.push("db");
+	dir.to_string_lossy().to_string()
+}
+
+fn migration_dir(path: &str) -> String {
+	let mut dir = PathBuf::from(path);
+	dir.push("migration");
+	dir.to_string_lossy().to_string()
+}
+
+/// Returns the version file path.
+fn version_file_path(path: &str) -> PathBuf {
+	let mut file_path = PathBuf::from(path);
+	file_path.push(VERSION_FILE_NAME);
+	file_path
+}
+
+/// Reads current database version from the file at given path.
+/// If the file does not exist returns `DEFAULT_VERSION`.
+fn current_version(path: &str) -> Result<u8, Error> {
+	match fs::File::open(version_file_path(path)) {
+		Err(ref err) if err.kind() == IoErrorKind::NotFound => Ok(DEFAULT_VERSION),
+		Err(err) => Err(err.into()),
+		Ok(mut file) => {
+			let mut s = String::new();
+			file.read_to_string(&mut s)?;
+			u8::from_str_radix(&s, 10).map_err(|_| Error::UnknownDatabaseVersion)
+		},
+	}
+}
+
+/// Writes current database version to the file.
+/// Creates a new file if the version file does not exist yet.
+fn update_version(path: &str) -> Result<(), Error> {
+	let mut file = fs::File::create(version_file_path(path))?;
+	file.write_all(format!("{}", CURRENT_VERSION).as_bytes())?;
+	Ok(())
+}

parity/db/rocksdb/mod.rs

@@ -33,6 +33,8 @@ use cache::CacheConfig;
 mod blooms;
 mod migration;
 mod helpers;
+#[cfg(feature = "secretstore")]
+mod migration_secretstore;
 
 pub use self::migration::migrate;
 
@@ -61,10 +63,14 @@ impl BlockChainDB for AppDB {
 pub fn open_secretstore_db(data_path: &str) -> Result<Arc<dyn KeyValueDB>, String> {
 	use std::path::PathBuf;
 
+	migration_secretstore::upgrade_db(data_path).map_err(|e| e.to_string())?;
+
 	let mut db_path = PathBuf::from(data_path);
 	db_path.push("db");
 	let db_path = db_path.to_str().ok_or_else(|| "Invalid secretstore path".to_string())?;
-	Ok(Arc::new(Database::open_default(&db_path).map_err(|e| format!("Error opening database: {:?}", e))?))
+
+	let config = DatabaseConfig::with_columns(Some(1));
+	Ok(Arc::new(Database::open(&config, &db_path).map_err(|e| format!("Error opening database: {:?}", e))?))
 }
 
 /// Create a restoration db handler using the config generated by `client_path` and `client_config`.

parity/secretstore.rs

@@ -136,13 +136,13 @@ mod server {
 
 	/// Key server
 	pub struct KeyServer {
-		_key_server: Box<ethcore_secretstore::KeyServer>,
+		_key_server: Box<dyn ethcore_secretstore::KeyServer>,
 	}
 
 	impl KeyServer {
 		/// Create new key server
 		pub fn new(mut conf: Configuration, deps: Dependencies, executor: Executor) -> Result<Self, String> {
-			let self_secret: Arc<ethcore_secretstore::NodeKeyPair> = match conf.self_secret.take() {
+			let self_secret: Arc<dyn ethcore_secretstore::NodeKeyPair> = match conf.self_secret.take() {
 				Some(NodeSecretKey::Plain(secret)) => Arc::new(ethcore_secretstore::PlainNodeKeyPair::new(
 					KeyPair::from_secret(secret).map_err(|e| format!("invalid secret: {}", e))?)),
 				#[cfg(feature = "accounts")]

secret-store/Cargo.toml

@@ -17,6 +17,7 @@ ethcore-accounts = { path = "../accounts", optional = true}
 ethcore-call-contract = { path = "../ethcore/call-contract" }
 ethcore-sync = { path = "../ethcore/sync" }
 ethereum-types = "0.8.0"
+ethkey = { path = "../accounts/ethkey", optional = true }
 futures = "0.1"
 hyper = { version = "0.12", default-features = false }
 keccak-hash = "0.4.0"
@@ -48,4 +49,4 @@ tempdir = "0.3"
 kvdb-rocksdb = "0.2.0"
 
 [features]
-accounts = ["ethcore-accounts"]
+accounts = ["ethcore-accounts", "ethkey"]

secret-store/src/key_storage.rs

@@ -125,12 +125,12 @@ impl PersistentKeyStorage {
 }
 
 fn upgrade_db(db: Arc<dyn KeyValueDB>) -> Result<Arc<dyn KeyValueDB>, Error> {
-	let version = db.get(None, DB_META_KEY_VERSION)?;
+	let version = db.get(Some(0), DB_META_KEY_VERSION)?;
 	let version = version.and_then(|v| v.get(0).cloned());
 	match version {
 		None => {
 			let mut batch = db.transaction();
-			batch.put(None, DB_META_KEY_VERSION, &[CURRENT_VERSION]);
+			batch.put(Some(0), DB_META_KEY_VERSION, &[CURRENT_VERSION]);
 			db.write(batch)?;
 			Ok(db)
 		},
@@ -144,7 +144,7 @@ impl KeyStorage for PersistentKeyStorage {
 		let key: SerializableDocumentKeyShareV3 = key.into();
 		let key = serde_json::to_vec(&key).map_err(|e| Error::Database(e.to_string()))?;
 		let mut batch = self.db.transaction();
-		batch.put(None, document.as_bytes(), &key);
+		batch.put(Some(0), document.as_bytes(), &key);
 		self.db.write(batch).map_err(Into::into)
 	}
 
@@ -153,7 +153,7 @@ impl KeyStorage for PersistentKeyStorage {
 	}
 
 	fn get(&self, document: &ServerKeyId) -> Result<Option<DocumentKeyShare>, Error> {
-		self.db.get(None, document.as_bytes())
+		self.db.get(Some(0), document.as_bytes())
 			.map_err(|e| Error::Database(e.to_string()))
 			.and_then(|key| match key {
 				None => Ok(None),
@@ -166,28 +166,28 @@ impl KeyStorage for PersistentKeyStorage {
 
 	fn remove(&self, document: &ServerKeyId) -> Result<(), Error> {
 		let mut batch = self.db.transaction();
-		batch.delete(None, document.as_bytes());
+		batch.delete(Some(0), document.as_bytes());
 		self.db.write(batch).map_err(Into::into)
 	}
 
 	fn clear(&self) -> Result<(), Error> {
 		let mut batch = self.db.transaction();
 		for (key, _) in self.iter() {
-			batch.delete(None, key.as_bytes());
+			batch.delete(Some(0), key.as_bytes());
 		}
 		self.db.write(batch)
 			.map_err(|e| Error::Database(e.to_string()))
 	}
 
 	fn contains(&self, document: &ServerKeyId) -> bool {
-		self.db.get(None, document.as_bytes())
+		self.db.get(Some(0), document.as_bytes())
 			.map(|k| k.is_some())
 			.unwrap_or(false)
 	}
 
 	fn iter<'a>(&'a self) -> Box<dyn Iterator<Item=(ServerKeyId, DocumentKeyShare)> + 'a> {
 		Box::new(PersistentKeyStorageIterator {
-			iter: self.db.iter(None),
+			iter: self.db.iter(Some(0)),
 		})
 	}
 }

secret-store/src/lib.rs

@@ -53,7 +53,7 @@ extern crate lazy_static;
 #[macro_use]
 extern crate log;
 
-#[cfg(test)]
+#[cfg(any(test, feature = "accounts"))]
 extern crate ethkey;
 #[cfg(test)]
 extern crate env_logger;

secret-store/src/node_key_pair.rs

@@ -74,10 +74,10 @@ mod accounts {
 		pub fn new(account_provider: Arc<AccountProvider>, address: Address, password: Password) -> Result<Self, EthKeyError> {
 			let public = account_provider.account_public(address.clone(), &password).map_err(|e| EthKeyError::Custom(format!("{}", e)))?;
 			Ok(KeyStoreNodeKeyPair {
-				account_provider: account_provider,
-				address: address,
-				public: public,
-				password: password,
+				account_provider,
+				address,
+				public,
+				password,
 			})
 		}
 	}