From ccd956aca6d5b108a998fa13d4570f76dbff98a4 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jun 2022 09:14:18 +1000 Subject: [PATCH 1/5] Remove `Cursor::append`. It's a weird function: it lets you modify the token stream in the middle of iteration. There is only one call site, and it is only used for the rare `ProceduralMasquerade` legacy case. --- compiler/rustc_ast/src/tokenstream.rs | 12 +----------- compiler/rustc_expand/src/proc_macro_server.rs | 6 +++--- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 84d8829c398e6..70d5e53d969d0 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -25,7 +25,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_span::{Span, DUMMY_SP}; use smallvec::{smallvec, SmallVec}; -use std::{fmt, iter, mem}; +use std::{fmt, iter}; /// When the main Rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token tree. This is a very @@ -683,16 +683,6 @@ impl Cursor { self.index } - pub fn append(&mut self, new_stream: TokenStream) { - if new_stream.is_empty() { - return; - } - let index = self.index; - let stream = mem::take(&mut self.stream); - *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees(); - self.index = index; - } - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { self.stream.0[self.index..].get(n).map(|(tree, _)| tree) } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 093896c339d29..7cb587f6fc49e 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -522,10 +522,10 @@ impl server::TokenStream for Rustc<'_, '_> { // FIXME: It needs to be removed, but there are some // compatibility issues (see #73345). if group.flatten { - cursor.append(group.stream); - continue; + tts.append(&mut self.into_trees(group.stream)); + } else { + tts.push(TokenTree::Group(group)); } - tts.push(TokenTree::Group(group)); } Some(tt) => tts.push(tt), None => return tts, From 178b746d0416a6daa18f70423d25392f0c2e03e1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jun 2022 09:27:58 +1000 Subject: [PATCH 2/5] Remove `Cursor::index`. It's unused. --- compiler/rustc_ast/src/tokenstream.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 70d5e53d969d0..11826825e04e6 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -679,10 +679,6 @@ impl Cursor { }) } - pub fn index(&self) -> usize { - self.index - } - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { self.stream.0[self.index..].get(n).map(|(tree, _)| tree) } From f6b57883e0cd4ba5f1bfd8d4484778c5852ce585 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jun 2022 09:33:08 +1000 Subject: [PATCH 3/5] Remove `TokenStream::from_streams`. By inlining it into the only non-test call site. The one test call site is changed to use `TokenStreamBuilder`. --- compiler/rustc_ast/src/tokenstream.rs | 77 +++++++++---------- .../rustc_expand/src/tokenstream/tests.rs | 6 +- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 11826825e04e6..66b9e9c7eaf52 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -399,45 +399,6 @@ impl TokenStream { self.0.len() } - pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { - match streams.len() { - 0 => TokenStream::default(), - 1 => streams.pop().unwrap(), - _ => { - // We are going to extend the first stream in `streams` with - // the elements from the subsequent streams. This requires - // using `make_mut()` on the first stream, and in practice this - // doesn't cause cloning 99.9% of the time. - // - // One very common use case is when `streams` has two elements, - // where the first stream has any number of elements within - // (often 1, but sometimes many more) and the second stream has - // a single element within. - - // Determine how much the first stream will be extended. - // Needed to avoid quadratic blow up from on-the-fly - // reallocations (#57735). - let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum(); - - // Get the first stream. If it's `None`, create an empty - // stream. - let mut iter = streams.drain(..); - let mut first_stream_lrc = iter.next().unwrap().0; - - // Append the elements to the first stream, after reserving - // space for them. - let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); - first_vec_mut.reserve(num_appends); - for stream in iter { - first_vec_mut.extend(stream.0.iter().cloned()); - } - - // Create the final `TokenStream`. - TokenStream(first_stream_lrc) - } - } - } - pub fn trees(&self) -> CursorRef<'_> { CursorRef::new(self) } @@ -605,7 +566,43 @@ impl TokenStreamBuilder { } pub fn build(self) -> TokenStream { - TokenStream::from_streams(self.0) + let mut streams = self.0; + match streams.len() { + 0 => TokenStream::default(), + 1 => streams.pop().unwrap(), + _ => { + // We are going to extend the first stream in `streams` with + // the elements from the subsequent streams. This requires + // using `make_mut()` on the first stream, and in practice this + // doesn't cause cloning 99.9% of the time. + // + // One very common use case is when `streams` has two elements, + // where the first stream has any number of elements within + // (often 1, but sometimes many more) and the second stream has + // a single element within. + + // Determine how much the first stream will be extended. + // Needed to avoid quadratic blow up from on-the-fly + // reallocations (#57735). + let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum(); + + // Get the first stream. If it's `None`, create an empty + // stream. + let mut iter = streams.drain(..); + let mut first_stream_lrc = iter.next().unwrap().0; + + // Append the elements to the first stream, after reserving + // space for them. + let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); + first_vec_mut.reserve(num_appends); + for stream in iter { + first_vec_mut.extend(stream.0.iter().cloned()); + } + + // Create the final `TokenStream`. + TokenStream(first_stream_lrc) + } + } } } diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs index 270532f8edeec..e4a4db204d922 100644 --- a/compiler/rustc_expand/src/tokenstream/tests.rs +++ b/compiler/rustc_expand/src/tokenstream/tests.rs @@ -4,7 +4,6 @@ use rustc_ast::token; use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree}; use rustc_span::create_default_session_globals_then; use rustc_span::{BytePos, Span, Symbol}; -use smallvec::smallvec; fn string_to_ts(string: &str) -> TokenStream { string_to_stream(string.to_owned()) @@ -24,7 +23,10 @@ fn test_concat() { let test_res = string_to_ts("foo::bar::baz"); let test_fst = string_to_ts("foo::bar"); let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); + let mut builder = TokenStreamBuilder::new(); + builder.push(test_fst); + builder.push(test_snd); + let eq_res = builder.build(); assert_eq!(test_res.trees().count(), 5); assert_eq!(eq_res.trees().count(), 5); assert_eq!(test_res.eq_unspanned(&eq_res), true); From 2a5487afb47aa01c7ee1446bae616bd2e34ec6ad Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jun 2022 13:00:43 +1000 Subject: [PATCH 4/5] Merge `TokenStreamBuilder::push` into `TokenStreamBuilder::build`. Both functions do some modifying of streams using `make_mut`: - `push` sometimes glues the first token of the next stream to the last token of the first stream. - `build` appends tokens to the first stream. By doing all of this in the one place, things are simpler. The first stream can be modified in both ways (if necessary) in the one place, and any next stream with the first token removed doesn't need to be stored. --- compiler/rustc_ast/src/tokenstream.rs | 85 ++++++++++----------------- 1 file changed, 32 insertions(+), 53 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 66b9e9c7eaf52..37de90d64c774 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -523,46 +523,7 @@ impl TokenStreamBuilder { } pub fn push>(&mut self, stream: T) { - let mut stream = stream.into(); - - // If `self` is not empty and the last tree within the last stream is a - // token tree marked with `Joint`... - if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() - && let Some((TokenTree::Token(last_token), Spacing::Joint)) = last_stream_lrc.last() - // ...and `stream` is not empty and the first tree within it is - // a token tree... - && let TokenStream(ref mut stream_lrc) = stream - && let Some((TokenTree::Token(token), spacing)) = stream_lrc.first() - // ...and the two tokens can be glued together... - && let Some(glued_tok) = last_token.glue(&token) - { - // ...then do so, by overwriting the last token - // tree in `self` and removing the first token tree - // from `stream`. This requires using `make_mut()` - // on the last stream in `self` and on `stream`, - // and in practice this doesn't cause cloning 99.9% - // of the time. - - // Overwrite the last token tree with the merged - // token. - let last_vec_mut = Lrc::make_mut(last_stream_lrc); - *last_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); - - // Remove the first token tree from `stream`. (This - // is almost always the only tree in `stream`.) - let stream_vec_mut = Lrc::make_mut(stream_lrc); - stream_vec_mut.remove(0); - - // Don't push `stream` if it's empty -- that could - // block subsequent token gluing, by getting - // between two token trees that should be glued - // together. - if !stream.is_empty() { - self.0.push(stream); - } - return; - } - self.0.push(stream); + self.0.push(stream.into()); } pub fn build(self) -> TokenStream { @@ -571,9 +532,9 @@ impl TokenStreamBuilder { 0 => TokenStream::default(), 1 => streams.pop().unwrap(), _ => { - // We are going to extend the first stream in `streams` with - // the elements from the subsequent streams. This requires - // using `make_mut()` on the first stream, and in practice this + // We will extend the first stream in `streams` with the + // elements from the subsequent streams. This requires using + // `make_mut()` on the first stream, and in practice this // doesn't cause cloning 99.9% of the time. // // One very common use case is when `streams` has two elements, @@ -586,21 +547,39 @@ impl TokenStreamBuilder { // reallocations (#57735). let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum(); - // Get the first stream. If it's `None`, create an empty - // stream. + // Get the first stream, which will become the result stream. + // If it's `None`, create an empty stream. let mut iter = streams.drain(..); - let mut first_stream_lrc = iter.next().unwrap().0; + let mut res_stream_lrc = iter.next().unwrap().0; - // Append the elements to the first stream, after reserving - // space for them. - let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); - first_vec_mut.reserve(num_appends); + // Append the subsequent elements to the result stream, after + // reserving space for them. + let res_vec_mut = Lrc::make_mut(&mut res_stream_lrc); + res_vec_mut.reserve(num_appends); for stream in iter { - first_vec_mut.extend(stream.0.iter().cloned()); + let stream_iter = stream.0.iter().cloned(); + + // If (a) `res_mut_vec` is not empty and the last tree + // within it is a token tree marked with `Joint`, and (b) + // `stream` is not empty and the first tree within it is a + // token tree, and (c) the two tokens can be glued + // together... + if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last() + && let Some((TokenTree::Token(tok), spacing)) = stream.0.first() + && let Some(glued_tok) = last_tok.glue(&tok) + { + // ...then overwrite the last token tree in + // `res_vec_mut` with the glued token, and skip the + // first token tree from `stream`. + *res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); + res_vec_mut.extend(stream_iter.skip(1)); + } else { + // Append all of `stream`. + res_vec_mut.extend(stream_iter); + } } - // Create the final `TokenStream`. - TokenStream(first_stream_lrc) + TokenStream(res_stream_lrc) } } } From 69f45b78608c1ddef9994b013461dcb7584d7072 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jun 2022 13:52:48 +1000 Subject: [PATCH 5/5] Add blank lines between methods in `proc_macro_server.rs`. Because that's the standard way of doing it. --- .../rustc_expand/src/proc_macro_server.rs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 7cb587f6fc49e..1e4193a5a16cc 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -329,6 +329,7 @@ impl Ident { sess.symbol_gallery.insert(sym, span); Ident { sym, is_raw, span } } + fn dollar_crate(span: Span) -> Ident { // `$crate` is accepted as an ident only if it comes from the compiler. Ident { sym: kw::DollarCrate, is_raw: false, span } @@ -403,6 +404,7 @@ impl server::TokenStream for Rustc<'_, '_> { fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { stream.is_empty() } + fn from_str(&mut self, src: &str) -> Self::TokenStream { parse_stream_from_source_str( FileName::proc_macro_source_code(src), @@ -411,9 +413,11 @@ impl server::TokenStream for Rustc<'_, '_> { Some(self.call_site), ) } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { pprust::tts_to_string(stream) } + fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result { // Parse the expression from our tokenstream. let expr: PResult<'_, _> = try { @@ -464,12 +468,14 @@ impl server::TokenStream for Rustc<'_, '_> { _ => Err(()), } } + fn from_token_tree( &mut self, tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } + fn concat_trees( &mut self, base: Option, @@ -484,6 +490,7 @@ impl server::TokenStream for Rustc<'_, '_> { } builder.build() } + fn concat_streams( &mut self, base: Option, @@ -498,6 +505,7 @@ impl server::TokenStream for Rustc<'_, '_> { } builder.build() } + fn into_trees( &mut self, stream: Self::TokenStream, @@ -543,21 +551,27 @@ impl server::Group for Rustc<'_, '_> { flatten: false, } } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { group.delimiter } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { group.stream.clone() } + fn span(&mut self, group: &Self::Group) -> Self::Span { group.span.entire() } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { group.span.open } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { group.span.close } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { group.span = DelimSpan::from_single(span); } @@ -567,15 +581,19 @@ impl server::Punct for Rustc<'_, '_> { fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) } + fn as_char(&mut self, punct: Self::Punct) -> char { punct.ch } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { if punct.joint { Spacing::Joint } else { Spacing::Alone } } + fn span(&mut self, punct: Self::Punct) -> Self::Span { punct.span } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { Punct { span, ..punct } } @@ -585,9 +603,11 @@ impl server::Ident for Rustc<'_, '_> { fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { Ident::new(self.sess(), Symbol::intern(string), is_raw, span) } + fn span(&mut self, ident: Self::Ident) -> Self::Span { ident.span } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { Ident { span, ..ident } } @@ -639,45 +659,57 @@ impl server::Literal for Rustc<'_, '_> { Ok(Literal { lit, span: self.call_site }) } + fn to_string(&mut self, literal: &Self::Literal) -> String { literal.lit.to_string() } + fn debug_kind(&mut self, literal: &Self::Literal) -> String { format!("{:?}", literal.lit.kind) } + fn symbol(&mut self, literal: &Self::Literal) -> String { literal.lit.symbol.to_string() } + fn suffix(&mut self, literal: &Self::Literal) -> Option { literal.lit.suffix.as_ref().map(Symbol::to_string) } + fn integer(&mut self, n: &str) -> Self::Literal { self.lit(token::Integer, Symbol::intern(n), None) } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) } + fn float(&mut self, n: &str) -> Self::Literal { self.lit(token::Float, Symbol::intern(n), None) } + fn f32(&mut self, n: &str) -> Self::Literal { self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) } + fn f64(&mut self, n: &str) -> Self::Literal { self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) } + fn string(&mut self, string: &str) -> Self::Literal { let quoted = format!("{:?}", string); assert!(quoted.starts_with('"') && quoted.ends_with('"')); let symbol = "ed[1..quoted.len() - 1]; self.lit(token::Str, Symbol::intern(symbol), None) } + fn character(&mut self, ch: char) -> Self::Literal { let quoted = format!("{:?}", ch); assert!(quoted.starts_with('\'') && quoted.ends_with('\'')); let symbol = "ed[1..quoted.len() - 1]; self.lit(token::Char, Symbol::intern(symbol), None) } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { let string = bytes .iter() @@ -687,12 +719,15 @@ impl server::Literal for Rustc<'_, '_> { .collect::(); self.lit(token::ByteStr, Symbol::intern(&string), None) } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { literal.span } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { literal.span = span; } + fn subspan( &mut self, literal: &Self::Literal, @@ -735,6 +770,7 @@ impl server::SourceFile for Rustc<'_, '_> { fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { Lrc::ptr_eq(file1, file2) } + fn path(&mut self, file: &Self::SourceFile) -> String { match file.name { FileName::Real(ref name) => name @@ -746,6 +782,7 @@ impl server::SourceFile for Rustc<'_, '_> { _ => file.name.prefer_local().to_string(), } } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { file.is_real_file() } @@ -755,6 +792,7 @@ impl server::MultiSpan for Rustc<'_, '_> { fn new(&mut self) -> Self::MultiSpan { vec![] } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { spans.push(span) } @@ -766,6 +804,7 @@ impl server::Diagnostic for Rustc<'_, '_> { diag.set_span(MultiSpan::from_spans(spans)); diag } + fn sub( &mut self, diag: &mut Self::Diagnostic, @@ -775,6 +814,7 @@ impl server::Diagnostic for Rustc<'_, '_> { ) { diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); } + fn emit(&mut self, mut diag: Self::Diagnostic) { self.sess().span_diagnostic.emit_diagnostic(&mut diag); } @@ -788,38 +828,49 @@ impl server::Span for Rustc<'_, '_> { format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) } } + fn def_site(&mut self) -> Self::Span { self.def_site } + fn call_site(&mut self) -> Self::Span { self.call_site } + fn mixed_site(&mut self) -> Self::Span { self.mixed_site } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { self.sess().source_map().lookup_char_pos(span.lo()).file } + fn parent(&mut self, span: Self::Span) -> Option { span.parent_callsite() } + fn source(&mut self, span: Self::Span) -> Self::Span { span.source_callsite() } + fn start(&mut self, span: Self::Span) -> LineColumn { let loc = self.sess().source_map().lookup_char_pos(span.lo()); LineColumn { line: loc.line, column: loc.col.to_usize() } } + fn end(&mut self, span: Self::Span) -> LineColumn { let loc = self.sess().source_map().lookup_char_pos(span.hi()); LineColumn { line: loc.line, column: loc.col.to_usize() } } + fn before(&mut self, span: Self::Span) -> Self::Span { span.shrink_to_lo() } + fn after(&mut self, span: Self::Span) -> Self::Span { span.shrink_to_hi() } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { let self_loc = self.sess().source_map().lookup_char_pos(first.lo()); let other_loc = self.sess().source_map().lookup_char_pos(second.lo()); @@ -830,9 +881,11 @@ impl server::Span for Rustc<'_, '_> { Some(first.to(second)) } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { span.with_ctxt(at.ctxt()) } + fn source_text(&mut self, span: Self::Span) -> Option { self.sess().source_map().span_to_snippet(span).ok() } @@ -863,6 +916,7 @@ impl server::Span for Rustc<'_, '_> { fn save_span(&mut self, span: Self::Span) -> usize { self.sess().save_proc_macro_span(span) } + fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span { let (resolver, krate, def_site) = (&*self.ecx.resolver, self.krate, self.def_site); *self.rebased_spans.entry(id).or_insert_with(|| {