Auto merge of #22061 - pczarn:quote_matcher_and_attr, r=kmcallister

Fixes #19674
Fixes #17396 (already closed, yeah)

cc @kmcallister , @cmr

src/doc/reference.md

@@ -764,7 +764,15 @@ usually in [procedural macros](book/plugins.html#syntax-extensions):
 * `quote_pat!`
 * `quote_stmt!`
 * `quote_tokens!`
+* `quote_matcher!`
 * `quote_ty!`
+* `quote_attr!`
+
+Keep in mind that when `$name : ident` appears in the input to
+`quote_tokens!`, the result contains unquoted `name` followed by two tokens.
+However, input of the same form passed to `quote_matcher!` becomes a
+quasiquoted MBE-matcher of a nonterminal. No unquotation happens. Otherwise
+the result of `quote_matcher!` is identical to that of `quote_tokens!`.
 
 Documentation is very limited at the moment.
 

src/libsyntax/ast.rs

@@ -58,9 +58,12 @@ pub use self::PathParameters::*;
 use codemap::{Span, Spanned, DUMMY_SP, ExpnId};
 use abi::Abi;
 use ast_util;
+use ext::base;
+use ext::tt::macro_parser;
 use owned_slice::OwnedSlice;
 use parse::token::{InternedString, str_to_ident};
 use parse::token;
+use parse::lexer;
 use ptr::P;
 
 use std::fmt;
@@ -960,6 +963,18 @@ impl TokenTree {
             TtSequence(span, _)  => span,
         }
     }
+
+    /// Use this token tree as a matcher to parse given tts.
+    pub fn parse(cx: &base::ExtCtxt, mtch: &[TokenTree], tts: &[TokenTree])
+                 -> macro_parser::NamedParseResult {
+        // `None` is because we're not interpolating
+        let arg_rdr = lexer::new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic,
+                                                         None,
+                                                         None,
+                                                         tts.iter().cloned().collect(),
+                                                         true);
+        macro_parser::parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtch)
+    }
 }
 
 pub type Mac = Spanned<Mac_>;

src/libsyntax/ext/base.rs

@@ -1,4 +1,4 @@
-// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
 // http://rust-lang.org/COPYRIGHT.
 //
@@ -519,6 +519,12 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
         syntax_expanders.insert(intern("quote_stmt"),
                            builtin_normal_expander(
                                 ext::quote::expand_quote_stmt));
+        syntax_expanders.insert(intern("quote_matcher"),
+                           builtin_normal_expander(
+                                ext::quote::expand_quote_matcher));
+        syntax_expanders.insert(intern("quote_attr"),
+                           builtin_normal_expander(
+                                ext::quote::expand_quote_attr));
     }
 
     syntax_expanders.insert(intern("line"),

src/libsyntax/ext/quote.rs

@@ -1,4 +1,4 @@
-// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
 // http://rust-lang.org/COPYRIGHT.
 //
@@ -401,7 +401,7 @@ pub fn expand_quote_tokens<'cx>(cx: &'cx mut ExtCtxt,
                                 tts: &[ast::TokenTree])
                                 -> Box<base::MacResult+'cx> {
     let (cx_expr, expr) = expand_tts(cx, sp, tts);
-    let expanded = expand_wrapper(cx, sp, cx_expr, expr);
+    let expanded = expand_wrapper(cx, sp, cx_expr, expr, &[&["syntax", "ext", "quote", "rt"]]);
     base::MacEager::expr(expanded)
 }
 
@@ -465,6 +465,32 @@ pub fn expand_quote_stmt(cx: &mut ExtCtxt,
     base::MacEager::expr(expanded)
 }
 
+pub fn expand_quote_attr(cx: &mut ExtCtxt,
+                         sp: Span,
+                         tts: &[ast::TokenTree])
+                         -> Box<base::MacResult+'static> {
+    let expanded = expand_parse_call(cx, sp, "parse_attribute",
+                                    vec!(cx.expr_bool(sp, true)), tts);
+
+    base::MacEager::expr(expanded)
+}
+
+pub fn expand_quote_matcher(cx: &mut ExtCtxt,
+                            sp: Span,
+                            tts: &[ast::TokenTree])
+                            -> Box<base::MacResult+'static> {
+    let (cx_expr, tts) = parse_arguments_to_quote(cx, tts);
+    let mut vector = mk_stmts_let(cx, sp);
+    vector.extend(statements_mk_tts(cx, &tts[..], true).into_iter());
+    let block = cx.expr_block(
+        cx.block_all(sp,
+                     vector,
+                     Some(cx.expr_ident(sp, id_ext("tt")))));
+
+    let expanded = expand_wrapper(cx, sp, cx_expr, block, &[&["syntax", "ext", "quote", "rt"]]);
+    base::MacEager::expr(expanded)
+}
+
 fn ids_ext(strs: Vec<String> ) -> Vec<ast::Ident> {
     strs.iter().map(|str| str_to_ident(&(*str))).collect()
 }
@@ -527,7 +553,7 @@ fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
 }
 
 #[allow(non_upper_case_globals)]
-fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
+fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
     macro_rules! mk_lit {
         ($name: expr, $suffix: expr, $($args: expr),*) => {{
             let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![$($args),*]);
@@ -606,6 +632,21 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
                                 vec!(mk_name(cx, sp, ident.ident())));
         }
 
+        token::MatchNt(name, kind, namep, kindp) => {
+            return cx.expr_call(sp,
+                                mk_token_path(cx, sp, "MatchNt"),
+                                vec!(mk_ident(cx, sp, name),
+                                     mk_ident(cx, sp, kind),
+                                     match namep {
+                                        ModName => mk_token_path(cx, sp, "ModName"),
+                                        Plain   => mk_token_path(cx, sp, "Plain"),
+                                     },
+                                     match kindp {
+                                        ModName => mk_token_path(cx, sp, "ModName"),
+                                        Plain   => mk_token_path(cx, sp, "Plain"),
+                                     }));
+        }
+
         token::Interpolated(_) => panic!("quote! with interpolated token"),
 
         _ => ()
@@ -642,7 +683,7 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
     mk_token_path(cx, sp, name)
 }
 
-fn mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
+fn statements_mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree, matcher: bool) -> Vec<P<ast::Stmt>> {
     match *tt {
         ast::TtToken(sp, SubstNt(ident, _)) => {
             // tt.extend($ident.to_tokens(ext_cx).into_iter())
@@ -663,18 +704,18 @@ fn mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
 
             vec!(cx.stmt_expr(e_push))
         }
-        ref tt @ ast::TtToken(_, MatchNt(..)) => {
+        ref tt @ ast::TtToken(_, MatchNt(..)) if !matcher => {
             let mut seq = vec![];
             for i in 0..tt.len() {
                 seq.push(tt.get_tt(i));
             }
-            mk_tts(cx, &seq[..])
+            statements_mk_tts(cx, &seq[..], matcher)
         }
         ast::TtToken(sp, ref tok) => {
             let e_sp = cx.expr_ident(sp, id_ext("_sp"));
             let e_tok = cx.expr_call(sp,
                                      mk_ast_path(cx, sp, "TtToken"),
-                                     vec!(e_sp, mk_token(cx, sp, tok)));
+                                     vec!(e_sp, expr_mk_token(cx, sp, tok)));
             let e_push =
                 cx.expr_method_call(sp,
                                     cx.expr_ident(sp, id_ext("tt")),
@@ -683,27 +724,61 @@ fn mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
             vec!(cx.stmt_expr(e_push))
         },
         ast::TtDelimited(_, ref delimed) => {
-            mk_tt(cx, &delimed.open_tt()).into_iter()
-                .chain(delimed.tts.iter().flat_map(|tt| mk_tt(cx, tt).into_iter()))
-                .chain(mk_tt(cx, &delimed.close_tt()).into_iter())
+            statements_mk_tt(cx, &delimed.open_tt(), matcher).into_iter()
+                .chain(delimed.tts.iter()
+                                  .flat_map(|tt| statements_mk_tt(cx, tt, matcher).into_iter()))
+                .chain(statements_mk_tt(cx, &delimed.close_tt(), matcher).into_iter())
                 .collect()
         },
-        ast::TtSequence(..) => panic!("TtSequence in quote!"),
-    }
-}
+        ast::TtSequence(sp, ref seq) => {
+            if !matcher {
+                panic!("TtSequence in quote!");
+            }
 
-fn mk_tts(cx: &ExtCtxt, tts: &[ast::TokenTree]) -> Vec<P<ast::Stmt>> {
-    let mut ss = Vec::new();
-    for tt in tts {
-        ss.extend(mk_tt(cx, tt).into_iter());
+            let e_sp = cx.expr_ident(sp, id_ext("_sp"));
+
+            let stmt_let_tt = cx.stmt_let(sp, true, id_ext("tt"), cx.expr_vec_ng(sp));
+            let mut tts_stmts = vec![stmt_let_tt];
+            tts_stmts.extend(statements_mk_tts(cx, &seq.tts[..], matcher).into_iter());
+            let e_tts = cx.expr_block(cx.block(sp, tts_stmts,
+                                                   Some(cx.expr_ident(sp, id_ext("tt")))));
+            let e_separator = match seq.separator {
+                Some(ref sep) => cx.expr_some(sp, expr_mk_token(cx, sp, sep)),
+                None => cx.expr_none(sp),
+            };
+            let e_op = match seq.op {
+                ast::ZeroOrMore => mk_ast_path(cx, sp, "ZeroOrMore"),
+                ast::OneOrMore => mk_ast_path(cx, sp, "OneOrMore"),
+            };
+            let fields = vec![cx.field_imm(sp, id_ext("tts"), e_tts),
+                              cx.field_imm(sp, id_ext("separator"), e_separator),
+                              cx.field_imm(sp, id_ext("op"), e_op),
+                              cx.field_imm(sp, id_ext("num_captures"),
+                                               cx.expr_usize(sp, seq.num_captures))];
+            let seq_path = vec![id_ext("syntax"), id_ext("ast"), id_ext("SequenceRepetition")];
+            let e_seq_struct = cx.expr_struct(sp, cx.path_global(sp, seq_path), fields);
+            let e_rc_new = cx.expr_call_global(sp, vec![id_ext("std"),
+                                                        id_ext("rc"),
+                                                        id_ext("Rc"),
+                                                        id_ext("new")],
+                                                   vec![e_seq_struct]);
+            let e_tok = cx.expr_call(sp,
+                                     mk_ast_path(cx, sp, "TtSequence"),
+                                     vec!(e_sp, e_rc_new));
+            let e_push =
+                cx.expr_method_call(sp,
+                                    cx.expr_ident(sp, id_ext("tt")),
+                                    id_ext("push"),
+                                    vec!(e_tok));
+            vec!(cx.stmt_expr(e_push))
+        }
     }
-    ss
 }
 
-fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
-              -> (P<ast::Expr>, P<ast::Expr>) {
+fn parse_arguments_to_quote(cx: &ExtCtxt, tts: &[ast::TokenTree])
+                            -> (P<ast::Expr>, Vec<ast::TokenTree>) {
     // NB: It appears that the main parser loses its mind if we consider
-    // $foo as a TtNonterminal during the main parse, so we have to re-parse
+    // $foo as a SubstNt during the main parse, so we have to re-parse
     // under quote_depth > 0. This is silly and should go away; the _guess_ is
     // it has to do with transition away from supporting old-style macros, so
     // try removing it when enough of them are gone.
@@ -719,6 +794,10 @@ fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
     let tts = p.parse_all_token_trees();
     p.abort_if_errors();
 
+    (cx_expr, tts)
+}
+
+fn mk_stmts_let(cx: &ExtCtxt, sp: Span) -> Vec<P<ast::Stmt>> {
     // We also bind a single value, sp, to ext_cx.call_site()
     //
     // This causes every span in a token-tree quote to be attributed to the
@@ -756,8 +835,23 @@ fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 
     let stmt_let_tt = cx.stmt_let(sp, true, id_ext("tt"), cx.expr_vec_ng(sp));
 
-    let mut vector = vec!(stmt_let_sp, stmt_let_tt);
-    vector.extend(mk_tts(cx, &tts[..]).into_iter());
+    vec!(stmt_let_sp, stmt_let_tt)
+}
+
+fn statements_mk_tts(cx: &ExtCtxt, tts: &[ast::TokenTree], matcher: bool) -> Vec<P<ast::Stmt>> {
+    let mut ss = Vec::new();
+    for tt in tts {
+        ss.extend(statements_mk_tt(cx, tt, matcher).into_iter());
+    }
+    ss
+}
+
+fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+              -> (P<ast::Expr>, P<ast::Expr>) {
+    let (cx_expr, tts) = parse_arguments_to_quote(cx, tts);
+
+    let mut vector = mk_stmts_let(cx, sp);
+    vector.extend(statements_mk_tts(cx, &tts[..], false).into_iter());
     let block = cx.expr_block(
         cx.block_all(sp,
                      vector,
@@ -769,14 +863,14 @@ fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 fn expand_wrapper(cx: &ExtCtxt,
                   sp: Span,
                   cx_expr: P<ast::Expr>,
-                  expr: P<ast::Expr>) -> P<ast::Expr> {
+                  expr: P<ast::Expr>,
+                  imports: &[&[&str]]) -> P<ast::Expr> {
     // Explicitly borrow to avoid moving from the invoker (#16992)
     let cx_expr_borrow = cx.expr_addr_of(sp, cx.expr_deref(sp, cx_expr));
     let stmt_let_ext_cx = cx.stmt_let(sp, false, id_ext("ext_cx"), cx_expr_borrow);
 
-    let stmts = [
-        &["syntax", "ext", "quote", "rt"],
-    ].iter().map(|path| {
+    let stmts = imports.iter().map(|path| {
+        // make item: `use ...;`
         let path = path.iter().map(|s| s.to_string()).collect();
         cx.stmt_item(sp, cx.item_use_glob(sp, ast::Inherited, ids_ext(path)))
     }).chain(Some(stmt_let_ext_cx).into_iter()).collect();
@@ -807,5 +901,10 @@ fn expand_parse_call(cx: &ExtCtxt,
     let expr = cx.expr_method_call(sp, new_parser_call, id_ext(parse_method),
                                    arg_exprs);
 
-    expand_wrapper(cx, sp, cx_expr, expr)
+    if parse_method == "parse_attribute" {
+        expand_wrapper(cx, sp, cx_expr, expr, &[&["syntax", "ext", "quote", "rt"],
+                                                &["syntax", "parse", "attr"]])
+    } else {
+        expand_wrapper(cx, sp, cx_expr, expr, &[&["syntax", "ext", "quote", "rt"]])
+    }
 }

src/libsyntax/ext/tt/macro_parser.rs

@@ -243,12 +243,15 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
     ret_val
 }
 
-pub enum ParseResult {
-    Success(HashMap<Ident, Rc<NamedMatch>>),
+pub enum ParseResult<T> {
+    Success(T),
     Failure(codemap::Span, String),
     Error(codemap::Span, String)
 }
 
+pub type NamedParseResult = ParseResult<HashMap<Ident, Rc<NamedMatch>>>;
+pub type PositionalParseResult = ParseResult<Vec<Rc<NamedMatch>>>;
+
 pub fn parse_or_else(sess: &ParseSess,
                      cfg: ast::CrateConfig,
                      rdr: TtReader,
@@ -280,7 +283,7 @@ pub fn parse(sess: &ParseSess,
              cfg: ast::CrateConfig,
              mut rdr: TtReader,
              ms: &[TokenTree])
-             -> ParseResult {
+             -> NamedParseResult {
     let mut cur_eis = Vec::new();
     cur_eis.push(initial_matcher_pos(Rc::new(ms.iter()
                                                 .cloned()

src/libsyntax/ext/tt/macro_rules.rs

@@ -15,7 +15,7 @@ use ext::base::{NormalTT, TTMacroExpander};
 use ext::tt::macro_parser::{Success, Error, Failure};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use ext::tt::macro_parser::{parse, parse_or_else};
-use parse::lexer::{new_tt_reader, new_tt_reader_with_doc_flag};
+use parse::lexer::new_tt_reader;
 use parse::parser::Parser;
 use parse::attr::ParserAttr;
 use parse::token::{self, special_idents, gensym_ident, NtTT, Token};
@@ -154,15 +154,8 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                 TtDelimited(_, ref delim) => &delim.tts[..],
                 _ => cx.span_fatal(sp, "malformed macro lhs")
             };
-            // `None` is because we're not interpolating
-            let arg_rdr = new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic,
-                                                      None,
-                                                      None,
-                                                      arg.iter()
-                                                         .cloned()
-                                                         .collect(),
-                                                      true);
-            match parse(cx.parse_sess(), cx.cfg(), arg_rdr, lhs_tt) {
+
+            match TokenTree::parse(cx, lhs_tt, arg) {
               Success(named_matches) => {
                 let rhs = match *rhses[i] {
                     // okay, what's your transcriber?