TokenStream improvements

src/libsyntax/attr/mod.rs

@@ -483,7 +483,7 @@ impl MetaItem {
             last_pos = segment.ident.span.hi();
         }
         idents.push(self.node.tokens(self.span));
-        TokenStream::concat(idents)
+        TokenStream::new(idents)
     }
 
     fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
@@ -539,7 +539,7 @@ impl MetaItemKind {
         match *self {
             MetaItemKind::Word => TokenStream::empty(),
             MetaItemKind::NameValue(ref lit) => {
-                TokenStream::concat(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()])
+                TokenStream::new(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()])
             }
             MetaItemKind::List(ref list) => {
                 let mut tokens = Vec::new();
@@ -552,7 +552,7 @@ impl MetaItemKind {
                 TokenTree::Delimited(
                     DelimSpan::from_single(span),
                     token::Paren,
-                    TokenStream::concat(tokens).into(),
+                    TokenStream::new(tokens).into(),
                 ).into()
             }
         }

src/libsyntax/ext/quote.rs

@@ -247,7 +247,7 @@ pub mod rt {
 
             let delim_span = DelimSpan::from_single(self.span);
             r.push(TokenTree::Delimited(
-                delim_span, token::Bracket, TokenStream::concat(inner).into()
+                delim_span, token::Bracket, TokenStream::new(inner).into()
             ));
             r
         }

src/libsyntax/ext/tt/transcribe.rs

@@ -103,12 +103,12 @@ pub fn transcribe(cx: &ExtCtxt,
                 }
                 Frame::Delimited { forest, span, .. } => {
                     if result_stack.is_empty() {
-                        return TokenStream::concat(result);
+                        return TokenStream::new(result);
                     }
                     let tree = TokenTree::Delimited(
                         span,
                         forest.delim,
-                        TokenStream::concat(result).into(),
+                        TokenStream::new(result).into(),
                     );
                     result = result_stack.pop().unwrap();
                     result.push(tree.into());

src/libsyntax/lib.rs

@@ -145,12 +145,6 @@ pub mod util {
     #[cfg(test)]
     pub mod parser_testing;
     pub mod move_map;
-
-    mod rc_slice;
-    pub use self::rc_slice::RcSlice;
-
-    mod rc_vec;
-    pub use self::rc_vec::RcVec;
 }
 
 pub mod json;

src/libsyntax/parse/attr.rs

@@ -170,7 +170,7 @@ impl<'a> Parser<'a> {
                     token::CloseDelim(_) | token::Eof => self.unexpected()?,
                     _ => self.parse_token_tree(),
                 };
-                TokenStream::concat(vec![eq.into(), tree.into()])
+                TokenStream::new(vec![eq.into(), tree.into()])
             } else {
                 TokenStream::empty()
             };

src/libsyntax/parse/lexer/tokentrees.rs

@@ -22,22 +22,22 @@ impl<'a> StringReader<'a> {
             tts.push(self.parse_token_tree()?);
         }
 
-        Ok(TokenStream::concat(tts))
+        Ok(TokenStream::new(tts))
     }
 
     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
         let mut tts = vec![];
         loop {
             if let token::CloseDelim(..) = self.token {
-                return TokenStream::concat(tts);
+                return TokenStream::new(tts);
             }
 
             match self.parse_token_tree() {
                 Ok(tree) => tts.push(tree),
                 Err(mut e) => {
                     e.emit();
-                    return TokenStream::concat(tts);
+                    return TokenStream::new(tts);
                 }
             }
         }

src/libsyntax/parse/mod.rs

@@ -842,13 +842,13 @@ mod tests {
         with_globals(|| {
             let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
 
-            let expected = TokenStream::concat(vec![
+            let expected = TokenStream::new(vec![
                 TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(),
                 TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
                     token::DelimToken::Paren,
-                    TokenStream::concat(vec![
+                    TokenStream::new(vec![
                         TokenTree::Token(sp(6, 7),
                                          token::Ident(Ident::from_str("b"), false)).into(),
                         TokenTree::Token(sp(8, 9), token::Colon).into(),
@@ -859,7 +859,7 @@ mod tests {
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
                     token::DelimToken::Brace,
-                    TokenStream::concat(vec![
+                    TokenStream::new(vec![
                         TokenTree::Token(sp(17, 18),
                                          token::Ident(Ident::from_str("b"), false)).into(),
                         TokenTree::Token(sp(18, 19), token::Semi).into(),

src/libsyntax/parse/parser.rs

@@ -2928,7 +2928,7 @@ impl<'a> Parser<'a> {
                 _ => result.push(self.parse_token_tree().into()),
             }
         }
-        TokenStream::concat(result)
+        TokenStream::new(result)
     }
 
     /// Parse a prefix-unary-operator expr
@@ -4624,7 +4624,7 @@ impl<'a> Parser<'a> {
                         self.unexpected()?;
                         unreachable!()
                     };
-                    TokenStream::concat(vec![
+                    TokenStream::new(vec![
                         args.into(),
                         TokenTree::Token(token_lo.to(self.prev_span), token::FatArrow).into(),
                         body.into(),

src/libsyntax/parse/token.rs

@@ -207,6 +207,10 @@ pub enum Token {
     Eof,
 }
 
+// `Token` is used a lot. Make sure it doesn't unintentionally get bigger.
+#[cfg(target_arch = "x86_64")]
+static_assert!(MEM_SIZE_OF_STATEMENT: mem::size_of::<Token>() == 16);
+
 impl Token {
     pub fn interpolated(nt: Nonterminal) -> Token {
         Token::Interpolated(Lrc::new((nt, LazyTokenStream::new())))