simplify compression and move it out of rlp crate

Cargo.toml

@@ -131,4 +131,5 @@ members = [
 	"miner",
 	"transaction-pool",
 	"whisper",
+	"util/rlp_compress"
 ]

ethcore/Cargo.toml

@@ -47,6 +47,7 @@ price-info = { path = "../price-info" }
 rayon = "0.8"
 rand = "0.4"
 rlp = { path = "../util/rlp" }
+rlp_compress = { path = "../util/rlp_compress" }
 rlp_derive = { path = "../util/rlp_derive" }
 kvdb = { path = "../util/kvdb" }
 kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }

ethcore/migrations/Cargo.toml

@@ -13,6 +13,7 @@ log = "0.3"
 macros = { path = "../../util/macros" }
 migration = { path = "../../util/migration" }
 rlp = { path = "../../util/rlp" }
+rlp_compress = { path = "../../util/rlp_compress" }
 patricia-trie = { path = "../../util/patricia_trie" }
 journaldb = { path = "../../util/journaldb" }
 ethcore-bloom-journal = { path = "../../util/bloom" }

ethcore/migrations/src/blocks/v8.rs

@@ -17,7 +17,7 @@
 //! This migration compresses the state db.
 
 use migration::{SimpleMigration, Progress};
-use rlp::{Compressible, UntrustedRlp, RlpType};
+use rlp_compress::{compress, blocks_swapper};
 
 /// Compressing migration.
 #[derive(Default)]
@@ -32,6 +32,7 @@ impl SimpleMigration for V8 {
 
 	fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
 		self.0.tick();
-		Some((key,UntrustedRlp::new(&value).compress(RlpType::Blocks).into_vec()))
+		let compressed = compress(&value, blocks_swapper()).into_vec();
+		Some((key, compressed))
 	}
 }

ethcore/migrations/src/lib.rs

@@ -22,6 +22,7 @@ extern crate log;
 extern crate macros;
 extern crate migration;
 extern crate rlp;
+extern crate rlp_compress;
 extern crate ethereum_types;
 extern crate ethcore_bytes as bytes;
 extern crate kvdb;

ethcore/src/blockchain/blockchain.rs

@@ -26,6 +26,7 @@ use ethereum_types::{H256, Bloom, U256};
 use parking_lot::{Mutex, RwLock};
 use bytes::Bytes;
 use rlp::*;
+use rlp_compress::{compress, decompress, blocks_swapper};
 use header::*;
 use transaction::*;
 use views::*;
@@ -272,7 +273,7 @@ impl BlockProvider for BlockChain {
 
 		let result = match opt {
 			Some(b) => {
-				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).into_vec();
+				let bytes = decompress(&b, blocks_swapper()).into_vec();
 				let mut write = self.block_headers.write();
 				write.insert(*hash, bytes.clone());
 				Some(encoded::Header::new(bytes))
@@ -308,7 +309,7 @@ impl BlockProvider for BlockChain {
 
 		let result = match opt {
 			Some(b) => {
-				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).into_vec();
+				let bytes = decompress(&b, blocks_swapper()).into_vec();
 				let mut write = self.block_bodies.write();
 				write.insert(*hash, bytes.clone());
 				Some(encoded::Body::new(bytes))
@@ -727,9 +728,8 @@ impl BlockChain {
 
 		assert!(self.pending_best_block.read().is_none());
 
-		let block_rlp = UntrustedRlp::new(bytes);
-		let compressed_header = block_rlp.at(0).unwrap().compress(RlpType::Blocks);
-		let compressed_body = UntrustedRlp::new(&Self::block_to_body(bytes)).compress(RlpType::Blocks);
+		let compressed_header = compress(block.header_rlp().as_raw(), blocks_swapper());
+		let compressed_body = compress(&Self::block_to_body(bytes), blocks_swapper());
 
 		// store block in db
 		batch.put(db::COL_HEADERS, &hash, &compressed_header);
@@ -928,9 +928,12 @@ impl BlockChain {
 
 		assert!(self.pending_best_block.read().is_none());
 
+		let compressed_header = compress(block.header_rlp().as_raw(), blocks_swapper());
+		let compressed_body = compress(&Self::block_to_body(bytes), blocks_swapper());
+
 		// store block in db
-		batch.put_compressed(db::COL_HEADERS, &hash, block.header_rlp().as_raw().to_vec());
-		batch.put_compressed(db::COL_BODIES, &hash, Self::block_to_body(bytes));
+		batch.put(db::COL_HEADERS, &hash, &compressed_header);
+		batch.put(db::COL_BODIES, &hash, &compressed_body);
 
 		let info = self.block_info(&header);
 

ethcore/src/lib.rs

@@ -83,6 +83,7 @@ extern crate price_info;
 extern crate rand;
 extern crate rayon;
 extern crate rlp;
+extern crate rlp_compress;
 extern crate keccak_hash as hash;
 extern crate heapsize;
 extern crate memorydb;

ethcore/src/state/account.rs

@@ -485,7 +485,7 @@ impl fmt::Debug for Account {
 
 #[cfg(test)]
 mod tests {
-	use rlp::{UntrustedRlp, RlpType, Compressible};
+	use rlp_compress::{compress, decompress, snapshot_swapper};
 	use ethereum_types::{H256, Address};
 	use memorydb::MemoryDB;
 	use bytes::Bytes;
@@ -495,10 +495,9 @@ mod tests {
 	#[test]
 	fn account_compress() {
 		let raw = Account::new_basic(2.into(), 4.into()).rlp();
-		let rlp = UntrustedRlp::new(&raw);
-		let compact_vec = rlp.compress(RlpType::Snapshot).into_vec();
+		let compact_vec = compress(&raw, snapshot_swapper());
 		assert!(raw.len() > compact_vec.len());
-		let again_raw = UntrustedRlp::new(&compact_vec).decompress(RlpType::Snapshot);
+		let again_raw = decompress(&compact_vec, snapshot_swapper());
 		assert_eq!(raw, again_raw.into_vec());
     }
 

util/kvdb-memorydb/Cargo.toml

@@ -5,5 +5,4 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 parking_lot = "0.5"
-rlp = { path = "../rlp" }
 kvdb = { path = "../kvdb" }

util/kvdb-memorydb/src/lib.rs

@@ -16,12 +16,10 @@
 
 extern crate parking_lot;
 extern crate kvdb;
-extern crate rlp;
 
 use std::collections::{BTreeMap, HashMap};
 use parking_lot::RwLock;
 use kvdb::{DBValue, DBTransaction, KeyValueDB, DBOp, Result};
-use rlp::{RlpType, UntrustedRlp, Compressible};
 
 /// A key-value database fulfilling the `KeyValueDB` trait, living in memory.
 /// This is generally intended for tests and is not particularly optimized.
@@ -75,14 +73,6 @@ impl KeyValueDB for InMemory {
 						col.insert(key.into_vec(), value);
 					}
 				},
-				DBOp::InsertCompressed { col, key, value } => {
-					if let Some(col) = columns.get_mut(&col) {
-						let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-						let mut value = DBValue::new();
-						value.append_slice(&compressed);
-						col.insert(key.into_vec(), value);
-					}
-				},
 				DBOp::Delete { col, key } => {
 					if let Some(col) = columns.get_mut(&col) {
 						col.remove(&*key);

util/kvdb-rocksdb/Cargo.toml

@@ -11,7 +11,6 @@ log = "0.3"
 num_cpus = "1.0"
 parking_lot = "0.5"
 regex = "0.2"
-rlp = { path = "../rlp" }
 rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" }
 interleaved-ordered = "0.1.0"
 

util/kvdb-rocksdb/src/lib.rs

@@ -26,7 +26,6 @@ extern crate rocksdb;
 
 extern crate ethereum_types;
 extern crate kvdb;
-extern crate rlp;
 
 use std::cmp;
 use std::collections::HashMap;
@@ -42,7 +41,6 @@ use rocksdb::{
 use interleaved_ordered::{interleave_ordered, InterleaveOrdered};
 
 use elastic_array::ElasticArray32;
-use rlp::{UntrustedRlp, RlpType, Compressible};
 use kvdb::{KeyValueDB, DBTransaction, DBValue, DBOp, Result};
 
 #[cfg(target_os = "linux")]
@@ -56,7 +54,6 @@ const DB_DEFAULT_MEMORY_BUDGET_MB: usize = 128;
 
 enum KeyState {
 	Insert(DBValue),
-	InsertCompressed(DBValue),
 	Delete,
 }
 
@@ -406,10 +403,6 @@ impl Database {
 					let c = Self::to_overlay_column(col);
 					overlay[c].insert(key, KeyState::Insert(value));
 				},
-				DBOp::InsertCompressed { col, key, value } => {
-					let c = Self::to_overlay_column(col);
-					overlay[c].insert(key, KeyState::InsertCompressed(value));
-				},
 				DBOp::Delete { col, key } => {
 					let c = Self::to_overlay_column(col);
 					overlay[c].insert(key, KeyState::Delete);
@@ -442,14 +435,6 @@ impl Database {
 										batch.put(&key, &value)?;
 									}
 								},
-								KeyState::InsertCompressed(ref value) => {
-									let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-									if c > 0 {
-										batch.put_cf(cfs[c - 1], &key, &compressed)?;
-									} else {
-										batch.put(&key, &value)?;
-									}
-								}
 							}
 						}
 					}
@@ -498,10 +483,6 @@ impl Database {
 						DBOp::Insert { col, key, value } => {
 							col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(cfs[c as usize], &key, &value))?
 						},
-						DBOp::InsertCompressed { col, key, value } => {
-							let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-							col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(cfs[c as usize], &key, &compressed))?
-						},
 						DBOp::Delete { col, key } => {
 							col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(cfs[c as usize], &key))?
 						},
@@ -522,12 +503,12 @@ impl Database {
 			Some(DBAndColumns { ref db, ref cfs }) => {
 				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
 				match overlay.get(key) {
-					Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
+					Some(&KeyState::Insert(ref value)) => Ok(Some(value.clone())),
 					Some(&KeyState::Delete) => Ok(None),
 					None => {
 						let flushing = &self.flushing.read()[Self::to_overlay_column(col)];
 						match flushing.get(key) {
-							Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
+							Some(&KeyState::Insert(ref value)) => Ok(Some(value.clone())),
 							Some(&KeyState::Delete) => Ok(None),
 							None => {
 								col.map_or_else(
@@ -562,8 +543,7 @@ impl Database {
 				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
 				let mut overlay_data = overlay.iter()
 					.filter_map(|(k, v)| match *v {
-						KeyState::Insert(ref value) |
-						KeyState::InsertCompressed(ref value) =>
+						KeyState::Insert(ref value) =>
 							Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_vec().into_boxed_slice())),
 						KeyState::Delete => None,
 					}).collect::<Vec<_>>();

util/kvdb/src/lib.rs

@@ -56,11 +56,6 @@ pub enum DBOp {
 		key: ElasticArray32<u8>,
 		value: DBValue,
 	},
-	InsertCompressed {
-		col: Option<u32>,
-		key: ElasticArray32<u8>,
-		value: DBValue,
-	},
 	Delete {
 		col: Option<u32>,
 		key: ElasticArray32<u8>,
@@ -72,7 +67,6 @@ impl DBOp {
 	pub fn key(&self) -> &[u8] {
 		match *self {
 			DBOp::Insert { ref key, .. } => key,
-			DBOp::InsertCompressed { ref key, .. } => key,
 			DBOp::Delete { ref key, .. } => key,
 		}
 	}
@@ -81,7 +75,6 @@ impl DBOp {
 	pub fn col(&self) -> Option<u32> {
 		match *self {
 			DBOp::Insert { col, .. } => col,
-			DBOp::InsertCompressed { col, .. } => col,
 			DBOp::Delete { col, .. } => col,
 		}
 	}
@@ -122,18 +115,6 @@ impl DBTransaction {
 		});
 	}
 
-	/// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write.
-	/// Value will be RLP-compressed on flush
-	pub fn put_compressed(&mut self, col: Option<u32>, key: &[u8], value: Bytes) {
-		let mut ekey = ElasticArray32::new();
-		ekey.append_slice(key);
-		self.ops.push(DBOp::InsertCompressed {
-			col: col,
-			key: ekey,
-			value: DBValue::from_vec(value),
-		});
-	}
-
 	/// Delete value by key.
 	pub fn delete(&mut self, col: Option<u32>, key: &[u8]) {
 		let mut ekey = ElasticArray32::new();

util/rlp/Cargo.toml

@@ -9,6 +9,5 @@ authors = ["Parity Technologies <admin@parity.io>"]
 [dependencies]
 elastic-array = "0.9"
 ethereum-types = "0.2"
-lazy_static = "1.0"
 rustc-hex = "1.0"
 byteorder = "1.0"