Backport core PRs to stable (#7892)

* update back-references more aggressively after answering from cache (#7578)

* Flush keyfiles. Resolves #7632 (#7868)

* Fix wallet import (#7873)

* rpc: generate new account id for imported wallets

* ethstore: handle duplicate wallet filenames

* ethstore: simplify deduplication of wallet file names

* ethstore: do not dedup wallet filenames on update

* ethstore: fix minor grumbles

* parity-version pr reopen (#7136)

* parity-version module split from util

removed unused util deps and features

trigger buildbot again

only kvdb links rocksdb

snappy linker issues

* rm snappy

* fixed old version imports

* Move updater metadata to Cargo.toml of parity-version. (#7832)

* Update version.

* Bump parity version.

* Fix version.

* Fix compilation.

Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 description = "Parity Ethereum client"
 name = "parity"
+# NOTE Make sure to update util/version/Cargo.toml as well
 version = "1.8.10"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
@@ -60,6 +61,7 @@ parity-reactor = { path = "util/reactor" }
 parity-rpc = { path = "rpc" }
 parity-rpc-client = { path = "rpc_client" }
 parity-updater = { path = "updater" }
+parity-version = { path = "util/version" }
 parity-whisper = { path = "whisper" }
 path = { path = "util/path" }
 panic_hook = { path = "panic_hook" }

dapps/Cargo.toml

@@ -37,6 +37,7 @@ parity-hash-fetch = { path = "../hash-fetch" }
 parity-reactor = { path = "../util/reactor" }
 parity-ui = { path = "./ui" }
 hash = { path = "../util/hash" }
+parity-version = { path = "../util/version" }
 
 clippy = { version = "0.0.103", optional = true}
 

dapps/src/handlers/content.rs

@@ -19,7 +19,7 @@
 use hyper::{self, mime, header};
 use hyper::StatusCode;
 
-use util::version;
+use parity_version::version;
 
 use handlers::add_security_headers;
 use Embeddable;

dapps/src/lib.rs

@@ -44,6 +44,7 @@ extern crate parity_dapps_glue as parity_dapps;
 extern crate parity_hash_fetch as hash_fetch;
 extern crate parity_ui;
 extern crate hash;
+extern crate parity_version;
 
 #[macro_use]
 extern crate futures;

ethcore/Cargo.toml

@@ -74,7 +74,6 @@ vm = { path = "vm" }
 wasm = { path = "wasm" }
 hash = { path = "../util/hash" }
 triehash = { path = "../util/triehash" }
-semantic_version = { path = "../util/semantic_version" }
 unexpected = { path = "../util/unexpected" }
 
 [dev-dependencies]

ethcore/light/src/on_demand/mod.rs

@@ -90,7 +90,14 @@ impl Pending {
 			match self.requests[idx].respond_local(cache) {
 				Some(response) => {
 					self.requests.supply_response_unchecked(&response);
+
+					// update header and back-references after each from-cache
+					// response to ensure that the requests are left in a consistent
+					// state and increase the likelihood of being able to answer
+					// the next request from cache.
 					self.update_header_refs(idx, &response);
+					self.fill_unanswered();
+
 					self.responses.push(response);
 				}
 				None => break,

ethcore/src/account_provider/mod.rs

@@ -257,9 +257,9 @@ impl AccountProvider {
 		Ok(Address::from(account.address).into())
 	}
 
-	/// Import a new presale wallet.
-	pub fn import_wallet(&self, json: &[u8], password: &str) -> Result<Address, Error> {
-		let account = self.sstore.import_wallet(SecretVaultRef::Root, json, password)?;
+	/// Import a new wallet.
+	pub fn import_wallet(&self, json: &[u8], password: &str, gen_id: bool) -> Result<Address, Error> {
+		let account = self.sstore.import_wallet(SecretVaultRef::Root, json, password, gen_id)?;
 		if self.blacklisted_accounts.contains(&account.address) {
 			self.sstore.remove_account(&account, password)?;
 			return Err(SSError::InvalidAccount.into());

ethcore/src/engines/authority_round/mod.rs

@@ -41,7 +41,6 @@ use itertools::{self, Itertools};
 use rlp::{UntrustedRlp, encode};
 use bigint::prelude::{U256, U128};
 use bigint::hash::{H256, H520};
-use semantic_version::SemanticVersion;
 use parking_lot::{Mutex, RwLock};
 use unexpected::{Mismatch, OutOfBounds};
 use util::Address;
@@ -483,8 +482,6 @@ impl IoHandler<()> for TransitionHandler {
 impl Engine<EthereumMachine> for AuthorityRound {
 	fn name(&self) -> &str { "AuthorityRound" }
 
-	fn version(&self) -> SemanticVersion { SemanticVersion::new(1, 0, 0) }
-
 	fn machine(&self) -> &EthereumMachine { &self.machine }
 
 	/// Two fields - consensus step and the corresponding proposer signature.
@@ -919,7 +916,6 @@ mod tests {
 	fn has_valid_metadata() {
 		let engine = Spec::new_test_round().engine;
 		assert!(!engine.name().is_empty());
-		assert!(engine.version().major >= 1);
 	}
 
 	#[test]

ethcore/src/engines/basic_authority.rs

@@ -29,7 +29,6 @@ use ethjson;
 use header::Header;
 use client::EngineClient;
 use machine::{AuxiliaryData, Call, EthereumMachine};
-use semantic_version::SemanticVersion;
 use super::signer::EngineSigner;
 use super::validator_set::{ValidatorSet, SimpleList, new_validator_set};
 
@@ -95,7 +94,6 @@ impl BasicAuthority {
 
 impl Engine<EthereumMachine> for BasicAuthority {
 	fn name(&self) -> &str { "BasicAuthority" }
-	fn version(&self) -> SemanticVersion { SemanticVersion::new(1, 0, 0) }
 
 	fn machine(&self) -> &EthereumMachine { &self.machine }
 
@@ -218,7 +216,6 @@ mod tests {
 	fn has_valid_metadata() {
 		let engine = new_test_authority().engine;
 		assert!(!engine.name().is_empty());
-		assert!(engine.version().major >= 1);
 	}
 
 	#[test]

ethcore/src/engines/mod.rs

@@ -54,7 +54,6 @@ use ethkey::Signature;
 use parity_machine::{Machine, LocalizedMachine as Localized};
 use bigint::prelude::U256;
 use bigint::hash::H256;
-use semantic_version::SemanticVersion;
 use util::*;
 use unexpected::{Mismatch, OutOfBounds};
 use bytes::Bytes;
@@ -178,8 +177,6 @@ pub enum EpochChange<M: Machine> {
 pub trait Engine<M: Machine>: Sync + Send {
 	/// The name of this engine.
 	fn name(&self) -> &str;
-	/// The version of this engine. Should be of the form
-	fn version(&self) -> SemanticVersion { SemanticVersion::new(0, 0, 0) }
 
 	/// Get access to the underlying state machine.
 	// TODO: decouple.

ethcore/src/engines/tendermint/mod.rs

@@ -50,7 +50,6 @@ use super::transition::TransitionHandler;
 use super::vote_collector::VoteCollector;
 use self::message::*;
 use self::params::TendermintParams;
-use semantic_version::SemanticVersion;
 use machine::{AuxiliaryData, EthereumMachine};
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
@@ -443,8 +442,6 @@ impl Tendermint {
 impl Engine<EthereumMachine> for Tendermint {
 	fn name(&self) -> &str { "Tendermint" }
 
-	fn version(&self) -> SemanticVersion { SemanticVersion::new(1, 0, 0) }
-
 	/// (consensus view, proposal signature, authority signatures)
 	fn seal_fields(&self) -> usize { 3 }
 
@@ -857,7 +854,6 @@ mod tests {
 	fn has_valid_metadata() {
 		let engine = Spec::new_test_tendermint().engine;
 		assert!(!engine.name().is_empty());
-		assert!(engine.version().major >= 1);
 	}
 
 	#[test]

ethcore/src/ethereum/ethash.rs

@@ -31,7 +31,6 @@ use engines::{self, Engine};
 use ethjson;
 use rlp::{self, UntrustedRlp};
 use machine::EthereumMachine;
-use semantic_version::SemanticVersion;
 
 /// Number of blocks in an ethash snapshot.
 // make dependent on difficulty incrment divisor?
@@ -169,7 +168,6 @@ impl engines::EpochVerifier<EthereumMachine> for Arc<Ethash> {
 
 impl Engine<EthereumMachine> for Arc<Ethash> {
 	fn name(&self) -> &str { "Ethash" }
-	fn version(&self) -> SemanticVersion { SemanticVersion::new(1, 0, 0) }
 	fn machine(&self) -> &EthereumMachine { &self.machine }
 
 	// Two fields - nonce and mix.
@@ -577,7 +575,6 @@ mod tests {
 	fn has_valid_metadata() {
 		let engine = test_spec().engine;
 		assert!(!engine.name().is_empty());
-		assert!(engine.version().major >= 1);
 	}
 
 	#[test]

ethcore/src/ethereum/mod.rs

@@ -30,15 +30,6 @@ pub use self::denominations::*;
 use machine::EthereumMachine;
 use super::spec::*;
 
-/// Most recent fork block that we support on Mainnet.
-pub const FORK_SUPPORTED_FOUNDATION: u64 = 4370000;
-
-/// Most recent fork block that we support on Ropsten.
-pub const FORK_SUPPORTED_ROPSTEN: u64 = 10;
-
-/// Most recent fork block that we support on Kovan.
-pub const FORK_SUPPORTED_KOVAN: u64 = 0;
-
 fn load<'a, T: Into<Option<SpecParams<'a>>>>(params: T, b: &[u8]) -> Spec {
 	match params.into() {
 		Some(params) => Spec::load(params, b),

ethcore/src/lib.rs

@@ -110,7 +110,6 @@ extern crate memorydb;
 extern crate patricia_trie as trie;
 extern crate triehash;
 extern crate ansi_term;
-extern crate semantic_version;
 extern crate unexpected;
 extern crate kvdb;
 extern crate kvdb_rocksdb;

ethcore/src/state/account.rs

@@ -35,6 +35,15 @@ use std::cell::{RefCell, Cell};
 
 const STORAGE_CACHE_ITEMS: usize = 8192;
 
+/// Boolean type for clean/dirty status.
+#[derive(PartialEq, Eq, Clone, Copy, Debug)]
+pub enum Filth {
+	/// Data has not been changed.
+	Clean,
+	/// Data has been changed.
+	Dirty,
+}
+
 /// Single account in the system.
 /// Keeps track of changes to the code and storage.
 /// The changes are applied in `commit_storage` and `commit_code`

ethstore/src/dir/disk.rs

@@ -15,6 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::{fs, io};
+use std::io::Write;
 use std::path::{PathBuf, Path};
 use std::collections::HashMap;
 use time;
@@ -152,31 +153,39 @@ impl<T> DiskDirectory<T> where T: KeyFileManager {
 		)
 	}
 
-	/// insert account with given file name
-	pub fn insert_with_filename(&self, account: SafeAccount, filename: String) -> Result<SafeAccount, Error> {
+
+	/// insert account with given filename. if the filename is a duplicate of any stored account and dedup is set to
+	/// true, a random suffix is appended to the filename.
+	pub fn insert_with_filename(&self, account: SafeAccount, mut filename: String, dedup: bool) -> Result<SafeAccount, Error> {
+		// path to keyfile
+		let mut keyfile_path = self.path.join(filename.as_str());
+
+		// check for duplicate filename and append random suffix
+		if dedup && keyfile_path.exists() {
+			let suffix = ::random::random_string(4);
+			filename.push_str(&format!("-{}", suffix));
+			keyfile_path.set_file_name(&filename);
+		}
+
 		// update account filename
 		let original_account = account.clone();
 		let mut account = account;
-		account.filename = Some(filename.clone());
+		account.filename = Some(filename);
 
 		{
-			// Path to keyfile
-			let mut keyfile_path = self.path.clone();
-			keyfile_path.push(filename.as_str());
-
 			// save the file
 			let mut file = fs::File::create(&keyfile_path)?;
-			if let Err(err) = self.key_manager.write(original_account, &mut file).map_err(|e| Error::Custom(format!("{:?}", e))) {
-				drop(file);
-				fs::remove_file(keyfile_path).expect("Expected to remove recently created file");
-				return Err(err);
-			}
+
+			// write key content
+			self.key_manager.write(original_account, &mut file).map_err(|e| Error::Custom(format!("{:?}", e)))?;
+
+			file.flush()?;
 
 			if let Err(_) = restrict_permissions_to_owner(keyfile_path.as_path()) {
-				drop(file);
-				fs::remove_file(keyfile_path).expect("Expected to remove recently created file");
 				return Err(Error::Io(io::Error::last_os_error()));
 			}
+
+			file.sync_all()?;
 		}
 
 		Ok(account)
@@ -199,17 +208,13 @@ impl<T> KeyDirectory for DiskDirectory<T> where T: KeyFileManager {
 
 	fn update(&self, account: SafeAccount) -> Result<SafeAccount, Error> {
 		// Disk store handles updates correctly iff filename is the same
-		self.insert(account)
+		let filename = account_filename(&account);
+		self.insert_with_filename(account, filename, false)
 	}
 
 	fn insert(&self, account: SafeAccount) -> Result<SafeAccount, Error> {
-		// build file path
-		let filename = account.filename.as_ref().cloned().unwrap_or_else(|| {
-			let timestamp = time::strftime("%Y-%m-%dT%H-%M-%S", &time::now_utc()).expect("Time-format string is valid.");
-			format!("UTC--{}Z--{}", timestamp, Uuid::from(account.id))
-		});
-
-		self.insert_with_filename(account, filename)
+		let filename = account_filename(&account);
+		self.insert_with_filename(account, filename, true)
 	}
 
 	fn remove(&self, account: &SafeAccount) -> Result<(), Error> {
@@ -285,6 +290,14 @@ impl KeyFileManager for DiskKeyFileManager {
 	}
 }
 
+fn account_filename(account: &SafeAccount) -> String {
+	// build file path
+	account.filename.clone().unwrap_or_else(|| {
+		let timestamp = time::strftime("%Y-%m-%dT%H-%M-%S", &time::now_utc()).expect("Time-format string is valid.");
+		format!("UTC--{}Z--{}", timestamp, Uuid::from(account.id))
+	})
+}
+
 #[cfg(test)]
 mod test {
 	extern crate tempdir;
@@ -317,6 +330,38 @@ mod test {
 		let _ = fs::remove_dir_all(dir);
 	}
 
+	#[test]
+	fn should_handle_duplicate_filenames() {
+		// given
+		let mut dir = env::temp_dir();
+		dir.push("ethstore_should_handle_duplicate_filenames");
+		let keypair = Random.generate().unwrap();
+		let password = "hello world";
+		let directory = RootDiskDirectory::create(dir.clone()).unwrap();
+
+		// when
+		let account = SafeAccount::create(&keypair, [0u8; 16], password, 1024, "Test".to_owned(), "{}".to_owned());
+		let filename = "test".to_string();
+		let dedup = true;
+
+		directory.insert_with_filename(account.clone(), "foo".to_string(), dedup).unwrap();
+		let file1 = directory.insert_with_filename(account.clone(), filename.clone(), dedup).unwrap().filename.unwrap();
+		let file2 = directory.insert_with_filename(account.clone(), filename.clone(), dedup).unwrap().filename.unwrap();
+		let file3 = directory.insert_with_filename(account.clone(), filename.clone(), dedup).unwrap().filename.unwrap();
+
+		// then
+		// the first file should have the original names
+		assert_eq!(file1, filename);
+
+		// the following duplicate files should have a suffix appended
+		assert!(file2 != file3);
+		assert_eq!(file2.len(), filename.len() + 5);
+		assert_eq!(file3.len(), filename.len() + 5);
+
+		// cleanup
+		let _ = fs::remove_dir_all(dir);
+	}
+
 	#[test]
 	fn should_manage_vaults() {
 		// given

ethstore/src/dir/vault.rs

@@ -106,7 +106,7 @@ impl VaultDiskDirectory {
 	fn copy_to_vault(&self, vault: &VaultDiskDirectory) -> Result<(), Error> {
 		for account in self.load()? {
 			let filename = account.filename.clone().expect("self is instance of DiskDirectory; DiskDirectory fills filename in load; qed");
-			vault.insert_with_filename(account, filename)?;
+			vault.insert_with_filename(account, filename, true)?;
 		}
 
 		Ok(())

ethstore/src/ethstore.rs

@@ -158,9 +158,14 @@ impl SecretStore for EthStore {
 		self.insert_account(vault, keypair.secret().clone(), password)
 	}
 
-	fn import_wallet(&self, vault: SecretVaultRef, json: &[u8], password: &str) -> Result<StoreAccountRef, Error> {
+	fn import_wallet(&self, vault: SecretVaultRef, json: &[u8], password: &str, gen_id: bool) -> Result<StoreAccountRef, Error> {
 		let json_keyfile = json::KeyFile::load(json).map_err(|_| Error::InvalidKeyFile("Invalid JSON format".to_owned()))?;
 		let mut safe_account = SafeAccount::from_file(json_keyfile, None);
+
+		if gen_id {
+			safe_account.id = Random::random();
+		}
+
 		let secret = safe_account.crypto.secret(password).map_err(|_| Error::InvalidPassword)?;
 		safe_account.address = KeyPair::from_secret(secret)?.address();
 		self.store.import(vault, safe_account)