Replace legacy Rlp with UntrustedRlp and use in ethcore rlp views (op…

…enethereum#8316)

* WIP

* Replace Rlp with UntrustedRlp in views, explicity unwrap with expect

First pass to get it to compile. Need to figure out whether to do this or to propogate Errors upstream, which would require many more changes to dependent code. If we do this way we are assuming that the views are always used in a context where the rlp is trusted to be valid e.g. when reading from our own DB. So need to fid out whether views are used with data received from an untrusted (e.g. extrernal peer).

* Remove original Rlp impl, rename UntrustedRlp -> Rlp

* Create rlp views with view! macro to record debug info

Views are assumed to be over valid rlp, so if there is a decoding error we record where the view was created in the first place and report it in the expect

* Use $crate in view! macro to avoid import, fix tests

* Expect valid rlp in decode functions for now

* Replace spaces with tabs in new file

* Add doc tests for creating views with macro

* Update rlp docs to reflect removing of UntrustedRlp

* Replace UntrustedRlp usages in private-tx merge

ethcore/light/src/cht.rs

@@ -26,7 +26,7 @@ use hashdb::HashDB;
 use memorydb::MemoryDB;
 use bytes::Bytes;
 use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder};
-use rlp::{RlpStream, UntrustedRlp};
+use rlp::{RlpStream, Rlp};
 
 // encode a key.
 macro_rules! key {
@@ -150,7 +150,7 @@ pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)
 	let res = match TrieDB::new(&db, &root) {
 		Err(_) => return None,
 		Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| {
-			let rlp = UntrustedRlp::new(val);
+			let rlp = Rlp::new(val);
 			rlp.val_at::<H256>(0)
 				.and_then(|h| rlp.val_at::<U256>(1).map(|td| (h, td)))
 				.ok()

ethcore/light/src/client/header_chain.rs

@@ -41,7 +41,7 @@ use ethcore::engines::epoch::{
 	PendingTransition as PendingEpochTransition
 };
 
-use rlp::{Encodable, Decodable, DecoderError, RlpStream, UntrustedRlp};
+use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp};
 use heapsize::HeapSizeOf;
 use ethereum_types::{H256, H264, U256};
 use plain_hasher::H256FastMap;
@@ -125,7 +125,7 @@ impl Encodable for Entry {
 }
 
 impl Decodable for Entry {
-	fn decode(rlp: &UntrustedRlp) -> Result<Self, DecoderError> {
+	fn decode(rlp: &Rlp) -> Result<Self, DecoderError> {
 		let mut candidates = SmallVec::<[Candidate; 3]>::new();
 
 		for item in rlp.iter() {
@@ -186,7 +186,7 @@ fn encode_canonical_transition(header: &Header, proof: &[u8]) -> Vec<u8> {
 
 // decode last canonical transition entry.
 fn decode_canonical_transition(t: &[u8]) -> Result<(Header, &[u8]), DecoderError> {
-	let rlp = UntrustedRlp::new(t);
+	let rlp = Rlp::new(t);
 
 	Ok((rlp.val_at(0)?, rlp.at(1)?.data()?))
 }

ethcore/light/src/net/mod.rs

@@ -22,7 +22,7 @@ use transaction::UnverifiedTransaction;
 
 use io::TimerToken;
 use network::{HostInfo, NetworkProtocolHandler, NetworkContext, PeerId};
-use rlp::{RlpStream, UntrustedRlp};
+use rlp::{RlpStream, Rlp};
 use ethereum_types::{H256, U256};
 use kvdb::DBValue;
 use parking_lot::{Mutex, RwLock};
@@ -528,7 +528,7 @@ impl LightProtocol {
 	//   - check whether peer exists
 	//   - check whether request was made
 	//   - check whether request kinds match
-	fn pre_verify_response(&self, peer: &PeerId, raw: &UntrustedRlp) -> Result<IdGuard, Error> {
+	fn pre_verify_response(&self, peer: &PeerId, raw: &Rlp) -> Result<IdGuard, Error> {
 		let req_id = ReqId(raw.val_at(0)?);
 		let cur_credits: U256 = raw.val_at(1)?;
 
@@ -572,7 +572,7 @@ impl LightProtocol {
 	/// Packet data is _untrusted_, which means that invalid data won't lead to
 	/// issues.
 	pub fn handle_packet(&self, io: &IoContext, peer: &PeerId, packet_id: u8, data: &[u8]) {
-		let rlp = UntrustedRlp::new(data);
+		let rlp = Rlp::new(data);
 
 		trace!(target: "pip", "Incoming packet {} from peer {}", packet_id, peer);
 
@@ -794,7 +794,7 @@ impl LightProtocol {
 
 impl LightProtocol {
 	// Handle status message from peer.
-	fn status(&self, peer: &PeerId, io: &IoContext, data: UntrustedRlp) -> Result<(), Error> {
+	fn status(&self, peer: &PeerId, io: &IoContext, data: Rlp) -> Result<(), Error> {
 		let pending = match self.pending_peers.write().remove(peer) {
 			Some(pending) => pending,
 			None => {
@@ -855,7 +855,7 @@ impl LightProtocol {
 	}
 
 	// Handle an announcement.
-	fn announcement(&self, peer: &PeerId, io: &IoContext, data: UntrustedRlp) -> Result<(), Error> {
+	fn announcement(&self, peer: &PeerId, io: &IoContext, data: Rlp) -> Result<(), Error> {
 		if !self.peers.read().contains_key(peer) {
 			debug!(target: "pip", "Ignoring announcement from unknown peer");
 			return Ok(())
@@ -900,7 +900,7 @@ impl LightProtocol {
 	}
 
 	// Receive requests from a peer.
-	fn request(&self, peer_id: &PeerId, io: &IoContext, raw: UntrustedRlp) -> Result<(), Error> {
+	fn request(&self, peer_id: &PeerId, io: &IoContext, raw: Rlp) -> Result<(), Error> {
 		// the maximum amount of requests we'll fill in a single packet.
 		const MAX_REQUESTS: usize = 256;
 
@@ -968,7 +968,7 @@ impl LightProtocol {
 	}
 
 	// handle a packet with responses.
-	fn response(&self, peer: &PeerId, io: &IoContext, raw: UntrustedRlp) -> Result<(), Error> {
+	fn response(&self, peer: &PeerId, io: &IoContext, raw: Rlp) -> Result<(), Error> {
 		let (req_id, responses) = {
 			let id_guard = self.pre_verify_response(peer, &raw)?;
 			let responses: Vec<Response> = raw.list_at(2)?;
@@ -987,7 +987,7 @@ impl LightProtocol {
 	}
 
 	// handle an update of request credits parameters.
-	fn update_credits(&self, peer_id: &PeerId, io: &IoContext, raw: UntrustedRlp) -> Result<(), Error> {
+	fn update_credits(&self, peer_id: &PeerId, io: &IoContext, raw: Rlp) -> Result<(), Error> {
 		let peers = self.peers.read();
 
 		let peer = peers.get(peer_id).ok_or(Error::UnknownPeer)?;
@@ -1022,7 +1022,7 @@ impl LightProtocol {
 	}
 
 	// handle an acknowledgement of request credits update.
-	fn acknowledge_update(&self, peer_id: &PeerId, _io: &IoContext, _raw: UntrustedRlp) -> Result<(), Error> {
+	fn acknowledge_update(&self, peer_id: &PeerId, _io: &IoContext, _raw: Rlp) -> Result<(), Error> {
 		let peers = self.peers.read();
 		let peer = peers.get(peer_id).ok_or(Error::UnknownPeer)?;
 		let mut peer = peer.lock();
@@ -1041,7 +1041,7 @@ impl LightProtocol {
 	}
 
 	// Receive a set of transactions to relay.
-	fn relay_transactions(&self, peer: &PeerId, io: &IoContext, data: UntrustedRlp) -> Result<(), Error> {
+	fn relay_transactions(&self, peer: &PeerId, io: &IoContext, data: Rlp) -> Result<(), Error> {
 		const MAX_TRANSACTIONS: usize = 256;
 
 		let txs: Vec<_> = data.iter()

ethcore/light/src/net/request_credits.rs

@@ -29,7 +29,7 @@
 use request::{self, Request};
 use super::error::Error;
 
-use rlp::{UntrustedRlp, RlpStream, Decodable, Encodable, DecoderError};
+use rlp::{Rlp, RlpStream, Decodable, Encodable, DecoderError};
 use ethereum_types::U256;
 use std::time::{Duration, Instant};
 
@@ -162,7 +162,7 @@ impl Encodable for CostTable {
 }
 
 impl Decodable for CostTable {
-	fn decode(rlp: &UntrustedRlp) -> Result<Self, DecoderError> {
+	fn decode(rlp: &Rlp) -> Result<Self, DecoderError> {
 		let base = rlp.val_at(0)?;
 
 		let mut headers = None;

ethcore/light/src/net/status.rs

@@ -16,7 +16,7 @@
 
 //! Peer status and capabilities.
 
-use rlp::{DecoderError, Encodable, Decodable, RlpStream, UntrustedRlp};
+use rlp::{DecoderError, Encodable, Decodable, RlpStream, Rlp};
 use ethereum_types::{H256, U256};
 
 use super::request_credits::FlowParams;
@@ -85,7 +85,7 @@ impl Key {
 // helper for decoding key-value pairs in the handshake or an announcement.
 struct Parser<'a> {
 	pos: usize,
-	rlp: UntrustedRlp<'a>,
+	rlp: Rlp<'a>,
 }
 
 impl<'a> Parser<'a> {
@@ -97,7 +97,7 @@ impl<'a> Parser<'a> {
 
 	// expect a specific next key, and get the value's RLP.
 	// if the key isn't found, the position isn't advanced.
-	fn expect_raw(&mut self, key: Key) -> Result<UntrustedRlp<'a>, DecoderError> {
+	fn expect_raw(&mut self, key: Key) -> Result<Rlp<'a>, DecoderError> {
 		trace!(target: "les", "Expecting key {}", key.as_str());
 		let pre_pos = self.pos;
 		if let Some((k, val)) = self.get_next()? {
@@ -109,7 +109,7 @@ impl<'a> Parser<'a> {
 	}
 
 	// get the next key and value RLP.
-	fn get_next(&mut self) -> Result<Option<(Key, UntrustedRlp<'a>)>, DecoderError> {
+	fn get_next(&mut self) -> Result<Option<(Key, Rlp<'a>)>, DecoderError> {
 		while self.pos < self.rlp.item_count()? {
 			let pair = self.rlp.at(self.pos)?;
 			let k: String = pair.val_at(0)?;
@@ -208,7 +208,7 @@ impl Capabilities {
 ///   - chain status
 ///   - serving capabilities
 ///   - request credit parameters
-pub fn parse_handshake(rlp: UntrustedRlp) -> Result<(Status, Capabilities, Option<FlowParams>), DecoderError> {
+pub fn parse_handshake(rlp: Rlp) -> Result<(Status, Capabilities, Option<FlowParams>), DecoderError> {
 	let mut parser = Parser {
 		pos: 0,
 		rlp: rlp,
@@ -304,7 +304,7 @@ pub struct Announcement {
 }
 
 /// Parse an announcement.
-pub fn parse_announcement(rlp: UntrustedRlp) -> Result<Announcement, DecoderError> {
+pub fn parse_announcement(rlp: Rlp) -> Result<Announcement, DecoderError> {
 	let mut last_key = None;
 
 	let mut announcement = Announcement {
@@ -374,7 +374,7 @@ mod tests {
 	use super::*;
 	use super::super::request_credits::FlowParams;
 	use ethereum_types::{U256, H256};
-	use rlp::{RlpStream, UntrustedRlp};
+	use rlp::{RlpStream, Rlp};
 
 	#[test]
 	fn full_handshake() {
@@ -404,7 +404,7 @@ mod tests {
 		let handshake = write_handshake(&status, &capabilities, Some(&flow_params));
 
 		let (read_status, read_capabilities, read_flow)
-			= parse_handshake(UntrustedRlp::new(&handshake)).unwrap();
+			= parse_handshake(Rlp::new(&handshake)).unwrap();
 
 		assert_eq!(read_status, status);
 		assert_eq!(read_capabilities, capabilities);
@@ -439,7 +439,7 @@ mod tests {
 		let handshake = write_handshake(&status, &capabilities, Some(&flow_params));
 
 		let (read_status, read_capabilities, read_flow)
-			= parse_handshake(UntrustedRlp::new(&handshake)).unwrap();
+			= parse_handshake(Rlp::new(&handshake)).unwrap();
 
 		assert_eq!(read_status, status);
 		assert_eq!(read_capabilities, capabilities);
@@ -473,7 +473,7 @@ mod tests {
 
 		let handshake = write_handshake(&status, &capabilities, Some(&flow_params));
 		let interleaved = {
-			let handshake = UntrustedRlp::new(&handshake);
+			let handshake = Rlp::new(&handshake);
 			let mut stream = RlpStream::new_list(handshake.item_count().unwrap_or(0) * 3);
 
 			for item in handshake.iter() {
@@ -489,7 +489,7 @@ mod tests {
 		};
 
 		let (read_status, read_capabilities, read_flow)
-			= parse_handshake(UntrustedRlp::new(&interleaved)).unwrap();
+			= parse_handshake(Rlp::new(&interleaved)).unwrap();
 
 		assert_eq!(read_status, status);
 		assert_eq!(read_capabilities, capabilities);
@@ -510,7 +510,7 @@ mod tests {
 		};
 
 		let serialized = write_announcement(&announcement);
-		let read = parse_announcement(UntrustedRlp::new(&serialized)).unwrap();
+		let read = parse_announcement(Rlp::new(&serialized)).unwrap();
 
 		assert_eq!(read, announcement);
 	}
@@ -529,7 +529,7 @@ mod tests {
 			.append_raw(&encode_flag(Key::ServeHeaders), 1);
 
 		let out = stream.drain();
-		assert!(parse_announcement(UntrustedRlp::new(&out)).is_err());
+		assert!(parse_announcement(Rlp::new(&out)).is_err());
 
 		let mut stream = RlpStream::new_list(6);
 		stream
@@ -541,7 +541,7 @@ mod tests {
 			.append_raw(&encode_pair(Key::ServeStateSince, &44u64), 1);
 
 		let out = stream.drain();
-		assert!(parse_announcement(UntrustedRlp::new(&out)).is_ok());
+		assert!(parse_announcement(Rlp::new(&out)).is_ok());
 	}
 
 	#[test]
@@ -566,7 +566,7 @@ mod tests {
 		let handshake = write_handshake(&status, &capabilities, None);
 
 		let (read_status, read_capabilities, read_flow)
-			= parse_handshake(UntrustedRlp::new(&handshake)).unwrap();
+			= parse_handshake(Rlp::new(&handshake)).unwrap();
 
 		assert_eq!(read_status, status);
 		assert_eq!(read_capabilities, capabilities);

ethcore/light/src/net/tests/mod.rs

@@ -31,7 +31,7 @@ use provider::Provider;
 use request;
 use request::*;
 
-use rlp::{UntrustedRlp, RlpStream};
+use rlp::{Rlp, RlpStream};
 use ethereum_types::{H256, U256, Address};
 
 use std::sync::Arc;
@@ -688,7 +688,7 @@ fn id_guard() {
 		stream.begin_list(2).append(&125usize).append(&3usize);
 
 		let packet = stream.out();
-		assert!(proto.response(&peer_id, &Expect::Nothing, UntrustedRlp::new(&packet)).is_err());
+		assert!(proto.response(&peer_id, &Expect::Nothing, Rlp::new(&packet)).is_err());
 	}
 
 	// next, do an unexpected response.
@@ -699,7 +699,7 @@ fn id_guard() {
 		stream.begin_list(0);
 
 		let packet = stream.out();
-		assert!(proto.response(&peer_id, &Expect::Nothing, UntrustedRlp::new(&packet)).is_err());
+		assert!(proto.response(&peer_id, &Expect::Nothing, Rlp::new(&packet)).is_err());
 	}
 
 	// lastly, do a valid (but empty) response.
@@ -710,7 +710,7 @@ fn id_guard() {
 		stream.begin_list(0);
 
 		let packet = stream.out();
-		assert!(proto.response(&peer_id, &Expect::Nothing, UntrustedRlp::new(&packet)).is_ok());
+		assert!(proto.response(&peer_id, &Expect::Nothing, Rlp::new(&packet)).is_ok());
 	}
 
 	let peers = proto.peers.read();

ethcore/light/src/on_demand/request.rs

@@ -30,7 +30,7 @@ use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY, KECCAK_EMPTY_LIST_RLP, keccak};
 
 use request::{self as net_request, IncompleteRequest, CompleteRequest, Output, OutputKind, Field};
 
-use rlp::{RlpStream, UntrustedRlp};
+use rlp::{RlpStream, Rlp};
 use ethereum_types::{H256, U256, Address};
 use parking_lot::Mutex;
 use hashdb::HashDB;
@@ -831,7 +831,7 @@ impl Account {
 
 		match TrieDB::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? {
 			Some(val) => {
-				let rlp = UntrustedRlp::new(&val);
+				let rlp = Rlp::new(&val);
 				Ok(Some(BasicAccount {
 					nonce: rlp.val_at(0)?,
 					balance: rlp.val_at(1)?,