Skip to content

Commit

Permalink
Rollup merge of rust-lang#98278 - nnethercote:some-token-stream-clean…
Browse files Browse the repository at this point in the history 
…ups, r=petrochenkov

Some token stream cleanups

Best reviewed one commit at a time.

r? ``@petrochenkov``
  • Loading branch information
@JohnTitor
JohnTitor authored Jun 21, 2022
2 parents cfed2ae + 69f45b7 commit 0827359
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 100 deletions.
152 changes: 57 additions & 95 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
use rustc_span::{Span, DUMMY_SP};
use smallvec::{smallvec, SmallVec};

use std::{fmt, iter, mem};
use std::{fmt, iter};

/// When the main Rust parser encounters a syntax-extension invocation, it
/// parses the arguments to the invocation as a token tree. This is a very
Expand Down Expand Up @@ -399,45 +399,6 @@ impl TokenStream {
self.0.len()
}

pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream {
match streams.len() {
0 => TokenStream::default(),
1 => streams.pop().unwrap(),
_ => {
// We are going to extend the first stream in `streams` with
// the elements from the subsequent streams. This requires
// using `make_mut()` on the first stream, and in practice this
// doesn't cause cloning 99.9% of the time.
//
// One very common use case is when `streams` has two elements,
// where the first stream has any number of elements within
// (often 1, but sometimes many more) and the second stream has
// a single element within.

// Determine how much the first stream will be extended.
// Needed to avoid quadratic blow up from on-the-fly
// reallocations (#57735).
let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum();

// Get the first stream. If it's `None`, create an empty
// stream.
let mut iter = streams.drain(..);
let mut first_stream_lrc = iter.next().unwrap().0;

// Append the elements to the first stream, after reserving
// space for them.
let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc);
first_vec_mut.reserve(num_appends);
for stream in iter {
first_vec_mut.extend(stream.0.iter().cloned());
}

// Create the final `TokenStream`.
TokenStream(first_stream_lrc)
}
}
}

pub fn trees(&self) -> CursorRef<'_> {
CursorRef::new(self)
}
Expand Down Expand Up @@ -562,50 +523,65 @@ impl TokenStreamBuilder {
}

pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
let mut stream = stream.into();

// If `self` is not empty and the last tree within the last stream is a
// token tree marked with `Joint`...
if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut()
&& let Some((TokenTree::Token(last_token), Spacing::Joint)) = last_stream_lrc.last()
// ...and `stream` is not empty and the first tree within it is
// a token tree...
&& let TokenStream(ref mut stream_lrc) = stream
&& let Some((TokenTree::Token(token), spacing)) = stream_lrc.first()
// ...and the two tokens can be glued together...
&& let Some(glued_tok) = last_token.glue(&token)
{
// ...then do so, by overwriting the last token
// tree in `self` and removing the first token tree
// from `stream`. This requires using `make_mut()`
// on the last stream in `self` and on `stream`,
// and in practice this doesn't cause cloning 99.9%
// of the time.

// Overwrite the last token tree with the merged
// token.
let last_vec_mut = Lrc::make_mut(last_stream_lrc);
*last_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing);

// Remove the first token tree from `stream`. (This
// is almost always the only tree in `stream`.)
let stream_vec_mut = Lrc::make_mut(stream_lrc);
stream_vec_mut.remove(0);

// Don't push `stream` if it's empty -- that could
// block subsequent token gluing, by getting
// between two token trees that should be glued
// together.
if !stream.is_empty() {
self.0.push(stream);
}
return;
}
self.0.push(stream);
self.0.push(stream.into());
}

pub fn build(self) -> TokenStream {
TokenStream::from_streams(self.0)
let mut streams = self.0;
match streams.len() {
0 => TokenStream::default(),
1 => streams.pop().unwrap(),
_ => {
// We will extend the first stream in `streams` with the
// elements from the subsequent streams. This requires using
// `make_mut()` on the first stream, and in practice this
// doesn't cause cloning 99.9% of the time.
//
// One very common use case is when `streams` has two elements,
// where the first stream has any number of elements within
// (often 1, but sometimes many more) and the second stream has
// a single element within.

// Determine how much the first stream will be extended.
// Needed to avoid quadratic blow up from on-the-fly
// reallocations (#57735).
let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum();

// Get the first stream, which will become the result stream.
// If it's `None`, create an empty stream.
let mut iter = streams.drain(..);
let mut res_stream_lrc = iter.next().unwrap().0;

// Append the subsequent elements to the result stream, after
// reserving space for them.
let res_vec_mut = Lrc::make_mut(&mut res_stream_lrc);
res_vec_mut.reserve(num_appends);
for stream in iter {
let stream_iter = stream.0.iter().cloned();

// If (a) `res_mut_vec` is not empty and the last tree
// within it is a token tree marked with `Joint`, and (b)
// `stream` is not empty and the first tree within it is a
// token tree, and (c) the two tokens can be glued
// together...
if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last()
&& let Some((TokenTree::Token(tok), spacing)) = stream.0.first()
&& let Some(glued_tok) = last_tok.glue(&tok)
{
// ...then overwrite the last token tree in
// `res_vec_mut` with the glued token, and skip the
// first token tree from `stream`.
*res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing);
res_vec_mut.extend(stream_iter.skip(1));
} else {
// Append all of `stream`.
res_vec_mut.extend(stream_iter);
}
}

TokenStream(res_stream_lrc)
}
}
}
}

Expand Down Expand Up @@ -679,20 +655,6 @@ impl Cursor {
})
}

pub fn index(&self) -> usize {
self.index
}

pub fn append(&mut self, new_stream: TokenStream) {
if new_stream.is_empty() {
return;
}
let index = self.index;
let stream = mem::take(&mut self.stream);
*self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees();
self.index = index;
}

pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
}
Expand Down
Loading

0 comments on commit 0827359

Please sign in to comment.