From d950c1b2000371253e026281cbfac30857b04470 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sat, 8 Jan 2022 12:39:47 -0800 Subject: [PATCH 1/8] Extend rustdoc macro-generated-macro test --- src/test/rustdoc/macro-generated-macro.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/test/rustdoc/macro-generated-macro.rs b/src/test/rustdoc/macro-generated-macro.rs index 25d8bc3ec6281..942d8afaa5040 100644 --- a/src/test/rustdoc/macro-generated-macro.rs +++ b/src/test/rustdoc/macro-generated-macro.rs @@ -1,14 +1,21 @@ -macro_rules! outer { - ($($matcher:tt)*) => { +macro_rules! make_macro { + ($macro_name:ident $($matcher:tt)*) => { #[macro_export] - macro_rules! inner { + macro_rules! $macro_name { (<= $($matcher)* =>) => {}; } } } -// @has macro_generated_macro/macro.inner.html //pre 'macro_rules! inner {' +// @has macro_generated_macro/macro.interpolations.html //pre 'macro_rules! interpolations {' // @has - //pre '(<= type $($i : ident) :: * + $e : expr =>) => { ... };' -outer!(type $($i:ident)::* + $e:expr); +make_macro!(interpolations type $($i:ident)::* + $e:expr); +interpolations!(<= type foo::bar + x.sort() =>); -inner!(<= type foo::bar + x.sort() =>); +// @has macro_generated_macro/macro.attributes.html //pre 'macro_rules! attributes {' +// @has - //pre '(<= #! [no_std] #[inline] =>) => { ... };' +make_macro!(attributes #![no_std] #[inline]); + +// @has macro_generated_macro/macro.groups.html //pre 'macro_rules! groups {' +// @has - //pre '(<= fn {} () { foo [0] } =>) => { ... };' +make_macro!(groups fn {}() {foo[0]}); From 090461475102cda8c42a7b4c65e25d0d38b27db8 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sat, 8 Jan 2022 12:43:22 -0800 Subject: [PATCH 2/8] Render more readable macro matchers in rustdoc --- compiler/rustc_ast_pretty/src/pp.rs | 2 +- src/librustdoc/clean/utils.rs | 143 +++++++++++++++++++++- src/test/rustdoc/macro-generated-macro.rs | 16 ++- 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_ast_pretty/src/pp.rs b/compiler/rustc_ast_pretty/src/pp.rs index ad9d15f1ce345..7fba80a98dc69 100644 --- a/compiler/rustc_ast_pretty/src/pp.rs +++ b/compiler/rustc_ast_pretty/src/pp.rs @@ -599,7 +599,7 @@ impl Printer { self.break_offset(n, 0) } - crate fn zerobreak(&mut self) { + pub fn zerobreak(&mut self) { self.spaces(0) } diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs index 38da9a4635db8..a88e11b667af9 100644 --- a/src/librustdoc/clean/utils.rs +++ b/src/librustdoc/clean/utils.rs @@ -9,7 +9,10 @@ use crate::formats::item_type::ItemType; use crate::visit_lib::LibEmbargoVisitor; use rustc_ast as ast; -use rustc_ast::tokenstream::TokenTree; +use rustc_ast::token::{self, BinOpToken, DelimToken}; +use rustc_ast::tokenstream::{TokenStream, TokenTree}; +use rustc_ast_pretty::pprust::state::State as Printer; +use rustc_ast_pretty::pprust::PrintState; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; @@ -504,10 +507,44 @@ pub(super) fn render_macro_arms<'a>( /// as part of an item declaration. pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> String { if let Some(snippet) = snippet_equal_to_token(tcx, matcher) { - snippet - } else { - rustc_ast_pretty::pprust::tt_to_string(matcher) + // If the original source code is known, we display the matcher exactly + // as present in the source code. + return snippet; + } + + // If the matcher is macro-generated or some other reason the source code + // snippet is not available, we attempt to nicely render the token tree. + let mut printer = Printer::new(); + + // If the inner ibox fits on one line, we get: + // + // macro_rules! macroname { + // (the matcher) => {...}; + // } + // + // If the inner ibox gets wrapped, the cbox will break and get indented: + // + // macro_rules! macroname { + // ( + // the matcher ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~! + // ) => {...}; + // } + printer.cbox(8); + printer.word("("); + printer.zerobreak(); + printer.ibox(0); + match matcher { + TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts), + // Matcher which is not a Delimited is unexpected and should've failed + // to compile, but we render whatever it is wrapped in parens. + TokenTree::Token(_) => print_tt(&mut printer, matcher), } + printer.end(); + printer.break_offset_if_not_bol(0, -4); + printer.word(")"); + printer.end(); + printer.s.eof() } /// Find the source snippet for this token's Span, reparse it, and return the @@ -551,6 +588,104 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option, tt: &TokenTree) { + match tt { + TokenTree::Token(token) => { + let token_str = printer.token_to_string(token); + printer.word(token_str); + if let token::DocComment(..) = token.kind { + printer.hardbreak() + } + } + TokenTree::Delimited(_span, delim, tts) => { + let open_delim = printer.token_kind_to_string(&token::OpenDelim(*delim)); + printer.word(open_delim); + if !tts.is_empty() { + if *delim == DelimToken::Brace { + printer.space(); + } + print_tts(printer, tts); + if *delim == DelimToken::Brace { + printer.space(); + } + } + let close_delim = printer.token_kind_to_string(&token::CloseDelim(*delim)); + printer.word(close_delim); + } + } +} + +fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { + #[derive(Copy, Clone, PartialEq)] + enum State { + Start, + Dollar, + DollarIdent, + DollarIdentColon, + DollarParen, + DollarParenSep, + Pound, + PoundBang, + Ident, + Other, + } + + use State::*; + + let mut state = Start; + for tt in tts.trees() { + let (needs_space, next_state) = match &tt { + TokenTree::Token(tt) => match (state, &tt.kind) { + (Dollar, token::Ident(..)) => (false, DollarIdent), + (DollarIdent, token::Colon) => (false, DollarIdentColon), + (DollarIdentColon, token::Ident(..)) => (false, Other), + ( + DollarParen, + token::BinOp(BinOpToken::Plus | BinOpToken::Star) | token::Question, + ) => (false, Other), + (DollarParen, _) => (false, DollarParenSep), + (DollarParenSep, token::BinOp(BinOpToken::Plus | BinOpToken::Star)) => { + (false, Other) + } + (Pound, token::Not) => (false, PoundBang), + (_, token::Ident(symbol, /* is_raw */ false)) + if !usually_needs_space_between_keyword_and_open_delim(*symbol) => + { + (true, Ident) + } + (_, token::Comma | token::Semi) => (false, Other), + (_, token::Dollar) => (true, Dollar), + (_, token::Pound) => (true, Pound), + (_, _) => (true, Other), + }, + TokenTree::Delimited(_, delim, _) => match (state, delim) { + (Dollar, DelimToken::Paren) => (false, DollarParen), + (Pound | PoundBang, DelimToken::Bracket) => (false, Other), + (Ident, DelimToken::Paren | DelimToken::Bracket) => (false, Other), + (_, _) => (true, Other), + }, + }; + if state != Start && needs_space { + printer.space(); + } + print_tt(printer, &tt); + state = next_state; + } +} + +// This rough subset of keywords is listed here to distinguish tokens resembling +// `f(0)` (no space between ident and paren) from tokens resembling `if let (0, +// 0) = x` (space between ident and paren). +fn usually_needs_space_between_keyword_and_open_delim(symbol: Symbol) -> bool { + match symbol.as_str() { + "as" | "box" | "break" | "const" | "continue" | "crate" | "else" | "enum" | "extern" + | "for" | "if" | "impl" | "in" | "let" | "loop" | "macro" | "match" | "mod" | "move" + | "mut" | "ref" | "return" | "static" | "struct" | "trait" | "type" | "unsafe" | "use" + | "where" | "while" | "yield" => true, + _ => false, + } +} + pub(super) fn display_macro_source( cx: &mut DocContext<'_>, name: Symbol, diff --git a/src/test/rustdoc/macro-generated-macro.rs b/src/test/rustdoc/macro-generated-macro.rs index 942d8afaa5040..9f155439b7eef 100644 --- a/src/test/rustdoc/macro-generated-macro.rs +++ b/src/test/rustdoc/macro-generated-macro.rs @@ -8,14 +8,22 @@ macro_rules! make_macro { } // @has macro_generated_macro/macro.interpolations.html //pre 'macro_rules! interpolations {' -// @has - //pre '(<= type $($i : ident) :: * + $e : expr =>) => { ... };' +// @has - //pre '(<= type $($i:ident)::* + $e:expr =>) => { ... };' make_macro!(interpolations type $($i:ident)::* + $e:expr); interpolations!(<= type foo::bar + x.sort() =>); // @has macro_generated_macro/macro.attributes.html //pre 'macro_rules! attributes {' -// @has - //pre '(<= #! [no_std] #[inline] =>) => { ... };' -make_macro!(attributes #![no_std] #[inline]); +// @has - //pre '(<= #![no_std] #[cfg(feature = "alloc")] =>) => { ... };' +make_macro!(attributes #![no_std] #[cfg(feature = "alloc")]); // @has macro_generated_macro/macro.groups.html //pre 'macro_rules! groups {' -// @has - //pre '(<= fn {} () { foo [0] } =>) => { ... };' +// @has - //pre '(<= fn {} () { foo[0] } =>) => { ... };' make_macro!(groups fn {}() {foo[0]}); + +// @has macro_generated_macro/macro.linebreak.html //pre 'macro_rules! linebreak {' +// @has - //pre ' (' +// @has - //pre ' <= 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25' +// @has - //pre ' 26 27 28 =>' +// @has - //pre ' ) => { ... };' +// @has - //pre '};' +make_macro!(linebreak 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28); From 81b1e327909832045fdcfc6fd9ab74c1e43aeb4b Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 14 Jan 2022 14:28:20 -0800 Subject: [PATCH 3/8] Move render_macro_matcher to own module --- src/librustdoc/clean/mod.rs | 1 + src/librustdoc/clean/render_macro_matchers.rs | 191 ++++++++++++++++++ src/librustdoc/clean/utils.rs | 191 +----------------- 3 files changed, 194 insertions(+), 189 deletions(-) create mode 100644 src/librustdoc/clean/render_macro_matchers.rs diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index e759baa045892..dfa83ba995947 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -5,6 +5,7 @@ mod auto_trait; mod blanket_impl; crate mod cfg; crate mod inline; +mod render_macro_matchers; mod simplify; crate mod types; crate mod utils; diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs new file mode 100644 index 0000000000000..14990c2e9e6e9 --- /dev/null +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -0,0 +1,191 @@ +use rustc_ast::token::{self, BinOpToken, DelimToken}; +use rustc_ast::tokenstream::{TokenStream, TokenTree}; +use rustc_ast_pretty::pprust::state::State as Printer; +use rustc_ast_pretty::pprust::PrintState; +use rustc_middle::ty::TyCtxt; +use rustc_session::parse::ParseSess; +use rustc_span::source_map::FilePathMapping; +use rustc_span::symbol::Symbol; + +/// Render a macro matcher in a format suitable for displaying to the user +/// as part of an item declaration. +pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> String { + if let Some(snippet) = snippet_equal_to_token(tcx, matcher) { + // If the original source code is known, we display the matcher exactly + // as present in the source code. + return snippet; + } + + // If the matcher is macro-generated or some other reason the source code + // snippet is not available, we attempt to nicely render the token tree. + let mut printer = Printer::new(); + + // If the inner ibox fits on one line, we get: + // + // macro_rules! macroname { + // (the matcher) => {...}; + // } + // + // If the inner ibox gets wrapped, the cbox will break and get indented: + // + // macro_rules! macroname { + // ( + // the matcher ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~! + // ) => {...}; + // } + printer.cbox(8); + printer.word("("); + printer.zerobreak(); + printer.ibox(0); + match matcher { + TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts), + // Matcher which is not a Delimited is unexpected and should've failed + // to compile, but we render whatever it is wrapped in parens. + TokenTree::Token(_) => print_tt(&mut printer, matcher), + } + printer.end(); + printer.break_offset_if_not_bol(0, -4); + printer.word(")"); + printer.end(); + printer.s.eof() +} + +/// Find the source snippet for this token's Span, reparse it, and return the +/// snippet if the reparsed TokenTree matches the argument TokenTree. +fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option { + // Find what rustc thinks is the source snippet. + // This may not actually be anything meaningful if this matcher was itself + // generated by a macro. + let source_map = tcx.sess.source_map(); + let span = matcher.span(); + let snippet = source_map.span_to_snippet(span).ok()?; + + // Create a Parser. + let sess = ParseSess::new(FilePathMapping::empty()); + let file_name = source_map.span_to_filename(span); + let mut parser = + match rustc_parse::maybe_new_parser_from_source_str(&sess, file_name, snippet.clone()) { + Ok(parser) => parser, + Err(diagnostics) => { + for mut diagnostic in diagnostics { + diagnostic.cancel(); + } + return None; + } + }; + + // Reparse a single token tree. + let mut reparsed_trees = match parser.parse_all_token_trees() { + Ok(reparsed_trees) => reparsed_trees, + Err(mut diagnostic) => { + diagnostic.cancel(); + return None; + } + }; + if reparsed_trees.len() != 1 { + return None; + } + let reparsed_tree = reparsed_trees.pop().unwrap(); + + // Compare against the original tree. + if reparsed_tree.eq_unspanned(matcher) { Some(snippet) } else { None } +} + +fn print_tt(printer: &mut Printer<'_>, tt: &TokenTree) { + match tt { + TokenTree::Token(token) => { + let token_str = printer.token_to_string(token); + printer.word(token_str); + if let token::DocComment(..) = token.kind { + printer.hardbreak() + } + } + TokenTree::Delimited(_span, delim, tts) => { + let open_delim = printer.token_kind_to_string(&token::OpenDelim(*delim)); + printer.word(open_delim); + if !tts.is_empty() { + if *delim == DelimToken::Brace { + printer.space(); + } + print_tts(printer, tts); + if *delim == DelimToken::Brace { + printer.space(); + } + } + let close_delim = printer.token_kind_to_string(&token::CloseDelim(*delim)); + printer.word(close_delim); + } + } +} + +fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { + #[derive(Copy, Clone, PartialEq)] + enum State { + Start, + Dollar, + DollarIdent, + DollarIdentColon, + DollarParen, + DollarParenSep, + Pound, + PoundBang, + Ident, + Other, + } + + use State::*; + + let mut state = Start; + for tt in tts.trees() { + let (needs_space, next_state) = match &tt { + TokenTree::Token(tt) => match (state, &tt.kind) { + (Dollar, token::Ident(..)) => (false, DollarIdent), + (DollarIdent, token::Colon) => (false, DollarIdentColon), + (DollarIdentColon, token::Ident(..)) => (false, Other), + ( + DollarParen, + token::BinOp(BinOpToken::Plus | BinOpToken::Star) | token::Question, + ) => (false, Other), + (DollarParen, _) => (false, DollarParenSep), + (DollarParenSep, token::BinOp(BinOpToken::Plus | BinOpToken::Star)) => { + (false, Other) + } + (Pound, token::Not) => (false, PoundBang), + (_, token::Ident(symbol, /* is_raw */ false)) + if !usually_needs_space_between_keyword_and_open_delim(*symbol) => + { + (true, Ident) + } + (_, token::Comma | token::Semi) => (false, Other), + (_, token::Dollar) => (true, Dollar), + (_, token::Pound) => (true, Pound), + (_, _) => (true, Other), + }, + TokenTree::Delimited(_, delim, _) => match (state, delim) { + (Dollar, DelimToken::Paren) => (false, DollarParen), + (Pound | PoundBang, DelimToken::Bracket) => (false, Other), + (Ident, DelimToken::Paren | DelimToken::Bracket) => (false, Other), + (_, _) => (true, Other), + }, + }; + if state != Start && needs_space { + printer.space(); + } + print_tt(printer, &tt); + state = next_state; + } +} + +// This rough subset of keywords is listed here to distinguish tokens resembling +// `f(0)` (no space between ident and paren) from tokens resembling `if let (0, +// 0) = x` (space between ident and paren). +fn usually_needs_space_between_keyword_and_open_delim(symbol: Symbol) -> bool { + match symbol.as_str() { + "as" | "box" | "break" | "const" | "continue" | "crate" | "else" | "enum" | "extern" + | "for" | "if" | "impl" | "in" | "let" | "loop" | "macro" | "match" | "mod" | "move" + | "mut" | "ref" | "return" | "static" | "struct" | "trait" | "type" | "unsafe" | "use" + | "where" | "while" | "yield" => true, + _ => false, + } +} diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs index a88e11b667af9..dabf1e878c9fb 100644 --- a/src/librustdoc/clean/utils.rs +++ b/src/librustdoc/clean/utils.rs @@ -1,5 +1,6 @@ use crate::clean::auto_trait::AutoTraitFinder; use crate::clean::blanket_impl::BlanketImplFinder; +use crate::clean::render_macro_matchers::render_macro_matcher; use crate::clean::{ inline, Clean, Crate, ExternalCrate, Generic, GenericArg, GenericArgs, ImportSource, Item, ItemKind, Lifetime, Path, PathSegment, Primitive, PrimitiveType, Type, TypeBinding, Visibility, @@ -9,10 +10,7 @@ use crate::formats::item_type::ItemType; use crate::visit_lib::LibEmbargoVisitor; use rustc_ast as ast; -use rustc_ast::token::{self, BinOpToken, DelimToken}; -use rustc_ast::tokenstream::{TokenStream, TokenTree}; -use rustc_ast_pretty::pprust::state::State as Printer; -use rustc_ast_pretty::pprust::PrintState; +use rustc_ast::tokenstream::TokenTree; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; @@ -20,8 +18,6 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::mir::interpret::ConstValue; use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; use rustc_middle::ty::{self, DefIdTree, TyCtxt}; -use rustc_session::parse::ParseSess; -use rustc_span::source_map::FilePathMapping; use rustc_span::symbol::{kw, sym, Symbol}; use std::fmt::Write as _; use std::mem; @@ -503,189 +499,6 @@ pub(super) fn render_macro_arms<'a>( out } -/// Render a macro matcher in a format suitable for displaying to the user -/// as part of an item declaration. -pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> String { - if let Some(snippet) = snippet_equal_to_token(tcx, matcher) { - // If the original source code is known, we display the matcher exactly - // as present in the source code. - return snippet; - } - - // If the matcher is macro-generated or some other reason the source code - // snippet is not available, we attempt to nicely render the token tree. - let mut printer = Printer::new(); - - // If the inner ibox fits on one line, we get: - // - // macro_rules! macroname { - // (the matcher) => {...}; - // } - // - // If the inner ibox gets wrapped, the cbox will break and get indented: - // - // macro_rules! macroname { - // ( - // the matcher ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~! - // ) => {...}; - // } - printer.cbox(8); - printer.word("("); - printer.zerobreak(); - printer.ibox(0); - match matcher { - TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts), - // Matcher which is not a Delimited is unexpected and should've failed - // to compile, but we render whatever it is wrapped in parens. - TokenTree::Token(_) => print_tt(&mut printer, matcher), - } - printer.end(); - printer.break_offset_if_not_bol(0, -4); - printer.word(")"); - printer.end(); - printer.s.eof() -} - -/// Find the source snippet for this token's Span, reparse it, and return the -/// snippet if the reparsed TokenTree matches the argument TokenTree. -fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option { - // Find what rustc thinks is the source snippet. - // This may not actually be anything meaningful if this matcher was itself - // generated by a macro. - let source_map = tcx.sess.source_map(); - let span = matcher.span(); - let snippet = source_map.span_to_snippet(span).ok()?; - - // Create a Parser. - let sess = ParseSess::new(FilePathMapping::empty()); - let file_name = source_map.span_to_filename(span); - let mut parser = - match rustc_parse::maybe_new_parser_from_source_str(&sess, file_name, snippet.clone()) { - Ok(parser) => parser, - Err(diagnostics) => { - for mut diagnostic in diagnostics { - diagnostic.cancel(); - } - return None; - } - }; - - // Reparse a single token tree. - let mut reparsed_trees = match parser.parse_all_token_trees() { - Ok(reparsed_trees) => reparsed_trees, - Err(mut diagnostic) => { - diagnostic.cancel(); - return None; - } - }; - if reparsed_trees.len() != 1 { - return None; - } - let reparsed_tree = reparsed_trees.pop().unwrap(); - - // Compare against the original tree. - if reparsed_tree.eq_unspanned(matcher) { Some(snippet) } else { None } -} - -fn print_tt(printer: &mut Printer<'_>, tt: &TokenTree) { - match tt { - TokenTree::Token(token) => { - let token_str = printer.token_to_string(token); - printer.word(token_str); - if let token::DocComment(..) = token.kind { - printer.hardbreak() - } - } - TokenTree::Delimited(_span, delim, tts) => { - let open_delim = printer.token_kind_to_string(&token::OpenDelim(*delim)); - printer.word(open_delim); - if !tts.is_empty() { - if *delim == DelimToken::Brace { - printer.space(); - } - print_tts(printer, tts); - if *delim == DelimToken::Brace { - printer.space(); - } - } - let close_delim = printer.token_kind_to_string(&token::CloseDelim(*delim)); - printer.word(close_delim); - } - } -} - -fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { - #[derive(Copy, Clone, PartialEq)] - enum State { - Start, - Dollar, - DollarIdent, - DollarIdentColon, - DollarParen, - DollarParenSep, - Pound, - PoundBang, - Ident, - Other, - } - - use State::*; - - let mut state = Start; - for tt in tts.trees() { - let (needs_space, next_state) = match &tt { - TokenTree::Token(tt) => match (state, &tt.kind) { - (Dollar, token::Ident(..)) => (false, DollarIdent), - (DollarIdent, token::Colon) => (false, DollarIdentColon), - (DollarIdentColon, token::Ident(..)) => (false, Other), - ( - DollarParen, - token::BinOp(BinOpToken::Plus | BinOpToken::Star) | token::Question, - ) => (false, Other), - (DollarParen, _) => (false, DollarParenSep), - (DollarParenSep, token::BinOp(BinOpToken::Plus | BinOpToken::Star)) => { - (false, Other) - } - (Pound, token::Not) => (false, PoundBang), - (_, token::Ident(symbol, /* is_raw */ false)) - if !usually_needs_space_between_keyword_and_open_delim(*symbol) => - { - (true, Ident) - } - (_, token::Comma | token::Semi) => (false, Other), - (_, token::Dollar) => (true, Dollar), - (_, token::Pound) => (true, Pound), - (_, _) => (true, Other), - }, - TokenTree::Delimited(_, delim, _) => match (state, delim) { - (Dollar, DelimToken::Paren) => (false, DollarParen), - (Pound | PoundBang, DelimToken::Bracket) => (false, Other), - (Ident, DelimToken::Paren | DelimToken::Bracket) => (false, Other), - (_, _) => (true, Other), - }, - }; - if state != Start && needs_space { - printer.space(); - } - print_tt(printer, &tt); - state = next_state; - } -} - -// This rough subset of keywords is listed here to distinguish tokens resembling -// `f(0)` (no space between ident and paren) from tokens resembling `if let (0, -// 0) = x` (space between ident and paren). -fn usually_needs_space_between_keyword_and_open_delim(symbol: Symbol) -> bool { - match symbol.as_str() { - "as" | "box" | "break" | "const" | "continue" | "crate" | "else" | "enum" | "extern" - | "for" | "if" | "impl" | "in" | "let" | "loop" | "macro" | "match" | "mod" | "move" - | "mut" | "ref" | "return" | "static" | "struct" | "trait" | "type" | "unsafe" | "use" - | "where" | "while" | "yield" => true, - _ => false, - } -} - pub(super) fn display_macro_source( cx: &mut DocContext<'_>, name: Symbol, From aff5296c78438759f9ea2b6d6f24ddab25d05407 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 14 Jan 2022 14:33:58 -0800 Subject: [PATCH 4/8] Move test of macro-generated macro with linebreak to snapshot --- .../rustdoc/macro-generated-macro.macro_linebreak_pre.html | 6 ++++++ src/test/rustdoc/macro-generated-macro.rs | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html diff --git a/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html b/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html new file mode 100644 index 0000000000000..fd0955f3056b8 --- /dev/null +++ b/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html @@ -0,0 +1,6 @@ +
macro_rules! linebreak {
+    (
+        <= 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
+        26 27 28 =>
+    ) => { ... };
+}
\ No newline at end of file diff --git a/src/test/rustdoc/macro-generated-macro.rs b/src/test/rustdoc/macro-generated-macro.rs index 9f155439b7eef..51e35b852a11d 100644 --- a/src/test/rustdoc/macro-generated-macro.rs +++ b/src/test/rustdoc/macro-generated-macro.rs @@ -20,10 +20,5 @@ make_macro!(attributes #![no_std] #[cfg(feature = "alloc")]); // @has - //pre '(<= fn {} () { foo[0] } =>) => { ... };' make_macro!(groups fn {}() {foo[0]}); -// @has macro_generated_macro/macro.linebreak.html //pre 'macro_rules! linebreak {' -// @has - //pre ' (' -// @has - //pre ' <= 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25' -// @has - //pre ' 26 27 28 =>' -// @has - //pre ' ) => { ... };' -// @has - //pre '};' +// @snapshot macro_linebreak_pre macro_generated_macro/macro.linebreak.html //pre make_macro!(linebreak 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28); From c70c646a0f7d7e9a1a481d5ae5b26b140cfd4cf2 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 14 Jan 2022 14:38:15 -0800 Subject: [PATCH 5/8] Eliminate string comparison from rustdoc keyword spacing logic --- src/librustdoc/clean/render_macro_matchers.rs | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index 14990c2e9e6e9..a6faa0d685755 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -5,7 +5,7 @@ use rustc_ast_pretty::pprust::PrintState; use rustc_middle::ty::TyCtxt; use rustc_session::parse::ParseSess; use rustc_span::source_map::FilePathMapping; -use rustc_span::symbol::Symbol; +use rustc_span::symbol::{kw, Symbol}; /// Render a macro matcher in a format suitable for displaying to the user /// as part of an item declaration. @@ -181,11 +181,38 @@ fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { // `f(0)` (no space between ident and paren) from tokens resembling `if let (0, // 0) = x` (space between ident and paren). fn usually_needs_space_between_keyword_and_open_delim(symbol: Symbol) -> bool { - match symbol.as_str() { - "as" | "box" | "break" | "const" | "continue" | "crate" | "else" | "enum" | "extern" - | "for" | "if" | "impl" | "in" | "let" | "loop" | "macro" | "match" | "mod" | "move" - | "mut" | "ref" | "return" | "static" | "struct" | "trait" | "type" | "unsafe" | "use" - | "where" | "while" | "yield" => true, + match symbol { + kw::As + | kw::Box + | kw::Break + | kw::Const + | kw::Continue + | kw::Crate + | kw::Else + | kw::Enum + | kw::Extern + | kw::For + | kw::If + | kw::Impl + | kw::In + | kw::Let + | kw::Loop + | kw::Macro + | kw::Match + | kw::Mod + | kw::Move + | kw::Mut + | kw::Ref + | kw::Return + | kw::Static + | kw::Struct + | kw::Trait + | kw::Type + | kw::Unsafe + | kw::Use + | kw::Where + | kw::While + | kw::Yield => true, _ => false, } } From a6c9d8f524c035b71c15058f3682dc684c268d82 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 14 Jan 2022 14:52:04 -0800 Subject: [PATCH 6/8] Invert the keyword list to list only *no* space before delim --- src/librustdoc/clean/render_macro_matchers.rs | 98 ++++++++++++------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index a6faa0d685755..dff370ab75025 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -5,7 +5,8 @@ use rustc_ast_pretty::pprust::PrintState; use rustc_middle::ty::TyCtxt; use rustc_session::parse::ParseSess; use rustc_span::source_map::FilePathMapping; -use rustc_span::symbol::{kw, Symbol}; +use rustc_span::symbol::{kw, Ident, Symbol}; +use rustc_span::Span; /// Render a macro matcher in a format suitable for displaying to the user /// as part of an item declaration. @@ -153,7 +154,7 @@ fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { } (Pound, token::Not) => (false, PoundBang), (_, token::Ident(symbol, /* is_raw */ false)) - if !usually_needs_space_between_keyword_and_open_delim(*symbol) => + if !usually_needs_space_between_keyword_and_open_delim(*symbol, tt.span) => { (true, Ident) } @@ -177,42 +178,63 @@ fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { } } -// This rough subset of keywords is listed here to distinguish tokens resembling -// `f(0)` (no space between ident and paren) from tokens resembling `if let (0, -// 0) = x` (space between ident and paren). -fn usually_needs_space_between_keyword_and_open_delim(symbol: Symbol) -> bool { +fn usually_needs_space_between_keyword_and_open_delim(symbol: Symbol, span: Span) -> bool { + let ident = Ident { name: symbol, span }; + let is_keyword = ident.is_used_keyword() || ident.is_unused_keyword(); + if !is_keyword { + // An identifier that is not a keyword usually does not need a space + // before an open delim. For example: `f(0)` or `f[0]`. + return false; + } + match symbol { - kw::As - | kw::Box - | kw::Break - | kw::Const - | kw::Continue - | kw::Crate - | kw::Else - | kw::Enum - | kw::Extern - | kw::For - | kw::If - | kw::Impl - | kw::In - | kw::Let - | kw::Loop - | kw::Macro - | kw::Match - | kw::Mod - | kw::Move - | kw::Mut - | kw::Ref - | kw::Return - | kw::Static - | kw::Struct - | kw::Trait - | kw::Type - | kw::Unsafe - | kw::Use - | kw::Where - | kw::While - | kw::Yield => true, - _ => false, + // No space after keywords that are syntactically an expression. For + // example: a tuple struct created with `let _ = Self(0, 0)`, or if + // someone has `impl Index for bool` then `true[MyStruct]`. + kw::False | kw::SelfLower | kw::SelfUpper | kw::True => false, + + // No space, as in `let _: fn();` + kw::Fn => false, + + // No space, as in `pub(crate) type T;` + kw::Pub => false, + + // No space for keywords that can end an expression, as in `fut.await()` + // where fut's Output type is `fn()`. + kw::Await => false, + + // Otherwise space after keyword. Some examples: + // + // `expr as [T; 2]` + // ^ + // `box (tuple,)` + // ^ + // `break (tuple,)` + // ^ + // `type T = dyn (Fn() -> dyn Trait) + Send;` + // ^ + // `for (tuple,) in iter {}` + // ^ + // `if (tuple,) == v {}` + // ^ + // `impl [T] {}` + // ^ + // `for x in [..] {}` + // ^ + // `let () = unit;` + // ^ + // `match [x, y] {...}` + // ^ + // `&mut (x as T)` + // ^ + // `return [];` + // ^ + // `fn f() where (): Into` + // ^ + // `while (a + b).what() {}` + // ^ + // `yield [];` + // ^ + _ => true, } } From f562b248306adbb6b95d2e6a5ce5c7ef30db144a Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Tue, 18 Jan 2022 11:11:06 -0800 Subject: [PATCH 7/8] Convert macro-generated-macro snapshot to /text() using PR 92914 --- .../macro-generated-macro.macro_linebreak_pre.html | 10 +++++----- src/test/rustdoc/macro-generated-macro.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html b/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html index fd0955f3056b8..ce5d3a8461b71 100644 --- a/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html +++ b/src/test/rustdoc/macro-generated-macro.macro_linebreak_pre.html @@ -1,6 +1,6 @@ -
macro_rules! linebreak {
+macro_rules! linebreak {
     (
-        <= 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
-        26 27 28 =>
-    ) => { ... };
-}
\ No newline at end of file + <= 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 + 26 27 28 => + ) => { ... }; +} \ No newline at end of file diff --git a/src/test/rustdoc/macro-generated-macro.rs b/src/test/rustdoc/macro-generated-macro.rs index 51e35b852a11d..3a13439915ea5 100644 --- a/src/test/rustdoc/macro-generated-macro.rs +++ b/src/test/rustdoc/macro-generated-macro.rs @@ -20,5 +20,5 @@ make_macro!(attributes #![no_std] #[cfg(feature = "alloc")]); // @has - //pre '(<= fn {} () { foo[0] } =>) => { ... };' make_macro!(groups fn {}() {foo[0]}); -// @snapshot macro_linebreak_pre macro_generated_macro/macro.linebreak.html //pre +// @snapshot macro_linebreak_pre macro_generated_macro/macro.linebreak.html //pre/text() make_macro!(linebreak 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28); From 039a058306b04b85a518db317304d1c25d516784 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Tue, 18 Jan 2022 11:19:43 -0800 Subject: [PATCH 8/8] Add a more complete test of rustdoc macro_rules matcher rendering --- ...macro-generated-macro.macro_morestuff_pre.html | 15 +++++++++++++++ src/test/rustdoc/macro-generated-macro.rs | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 src/test/rustdoc/macro-generated-macro.macro_morestuff_pre.html diff --git a/src/test/rustdoc/macro-generated-macro.macro_morestuff_pre.html b/src/test/rustdoc/macro-generated-macro.macro_morestuff_pre.html new file mode 100644 index 0000000000000..28f15522a82f7 --- /dev/null +++ b/src/test/rustdoc/macro-generated-macro.macro_morestuff_pre.html @@ -0,0 +1,15 @@ +macro_rules! morestuff { + ( + <= "space between most kinds of tokens" : 1 $x + @ :: >>= 'static + "no space inside paren or bracket" : (2 a) [2 a] $(2 $a:tt)* + "space inside curly brace" : { 2 a } + "no space inside empty delimiters" : () [] {} + "no space before comma or semicolon" : a, (a), { a }, a; [T; 0]; + "the three repetition specifiers" : $(@)*, $(@)+, $(@)? + "repetition separators" : $(@)|*, $(@)|+, $(@)==*, $(@)static* + "plus or star cannot be a repetition separator" : $(@)+ * $(@)* + + "no space between ident and paren" : let _ = f(0) + f[0] + Struct {}; + "space between keyword and paren" : return (a,) & for x in (..) + "some special case keywords" : pub(crate), fn() -> u8, Self(0, 0) => + ) => { ... }; +} \ No newline at end of file diff --git a/src/test/rustdoc/macro-generated-macro.rs b/src/test/rustdoc/macro-generated-macro.rs index 3a13439915ea5..1a423cac1b5ca 100644 --- a/src/test/rustdoc/macro-generated-macro.rs +++ b/src/test/rustdoc/macro-generated-macro.rs @@ -22,3 +22,18 @@ make_macro!(groups fn {}() {foo[0]}); // @snapshot macro_linebreak_pre macro_generated_macro/macro.linebreak.html //pre/text() make_macro!(linebreak 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28); + +// @snapshot macro_morestuff_pre macro_generated_macro/macro.morestuff.html //pre/text() +make_macro!(morestuff + "space between most kinds of tokens": 1 $x + @ :: >>= 'static + "no space inside paren or bracket": (2 a) [2 a] $(2 $a:tt)* + "space inside curly brace": { 2 a } + "no space inside empty delimiters": () [] {} + "no space before comma or semicolon": a, (a), { a }, a; [T; 0]; + "the three repetition specifiers": $(@)*, $(@)+, $(@)? + "repetition separators": $(@)|*, $(@)|+, $(@)==*, $(@)static* + "plus or star cannot be a repetition separator": $(@)+ * $(@)* + + "no space between ident and paren": let _ = f(0) + f[0] + Struct {}; + "space between keyword and paren": return (a,) & for x in (..) + "some special case keywords": pub(crate), fn() -> u8, Self(0, 0) +);