From a2566301e12364e227f0b72b51bb09698e1cb9dc Mon Sep 17 00:00:00 2001 From: Nick Cameron Date: Thu, 18 May 2017 10:37:24 +1200 Subject: [PATCH] Add an option to the parser to avoid parsing out of line modules This is useful if parsing from stdin or a String and don't want to try and read in a module from another file. Instead we just leave a stub in the AST. --- src/libsyntax/attr.rs | 2 +- src/libsyntax/ext/tt/macro_parser.rs | 8 ++++++-- src/libsyntax/ext/tt/macro_rules.rs | 4 ++-- src/libsyntax/parse/mod.rs | 25 +++++++++++++++++++++++-- src/libsyntax/parse/parser.rs | 15 +++++++++++++-- src/libsyntax/tokenstream.rs | 2 +- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/libsyntax/attr.rs b/src/libsyntax/attr.rs index 45f891d8dc56d..8e63e219c42c1 100644 --- a/src/libsyntax/attr.rs +++ b/src/libsyntax/attr.rs @@ -320,7 +320,7 @@ impl Attribute { pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, { - let mut parser = Parser::new(sess, self.tokens.clone(), None, false); + let mut parser = Parser::new(sess, self.tokens.clone(), None, false, false); let result = f(&mut parser)?; if parser.token != token::Eof { parser.unexpected()?; diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index bf66aa0f00bed..0b6a2eb536a20 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -418,9 +418,13 @@ fn inner_parse_loop(sess: &ParseSess, Success(()) } -pub fn parse(sess: &ParseSess, tts: TokenStream, ms: &[TokenTree], directory: Option) +pub fn parse(sess: &ParseSess, + tts: TokenStream, + ms: &[TokenTree], + directory: Option, + recurse_into_modules: bool) -> NamedParseResult { - let mut parser = Parser::new(sess, tts, directory, true); + let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true); let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo)); let mut next_eis = Vec::new(); // or proceed normally diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index a208f530602a5..73494d47fee9b 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -121,7 +121,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt, path: cx.current_expansion.module.directory.clone(), ownership: cx.current_expansion.directory_ownership, }; - let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), false); + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false); p.root_module_name = cx.current_expansion.module.mod_path.last() .map(|id| id.name.as_str().to_string()); @@ -192,7 +192,7 @@ pub fn compile(sess: &ParseSess, features: &RefCell, def: &ast::Item) ast::ItemKind::MacroDef(ref body) => body.clone().into(), _ => unreachable!(), }; - let argument_map = match parse(sess, body, &argument_gram, None) { + let argument_map = match parse(sess, body, &argument_gram, None, true) { Success(m) => m, Failure(sp, tok) => { let s = parse_failure_msg(tok); diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 1eff819d75549..3a68a6ba7646c 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -149,7 +149,9 @@ pub fn parse_stream_from_source_str(name: String, source: String, sess: &ParseSe // Create a new parser from a source string pub fn new_parser_from_source_str(sess: &ParseSess, name: String, source: String) -> Parser { - filemap_to_parser(sess, sess.codemap().new_filemap(name, source)) + let mut parser = filemap_to_parser(sess, sess.codemap().new_filemap(name, source)); + parser.recurse_into_file_modules = false; + parser } /// Create a new parser, handling errors as appropriate @@ -218,7 +220,7 @@ pub fn filemap_to_stream(sess: &ParseSess, filemap: Rc) -> TokenStream /// Given stream and the `ParseSess`, produce a parser pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser { - Parser::new(sess, stream, None, false) + Parser::new(sess, stream, None, true, false) } /// Parse a string representing a character literal into its final form. @@ -1032,4 +1034,23 @@ mod tests { Err(_) => panic!("could not get snippet"), } } + + // This tests that when parsing a string (rather than a file) we don't try + // and read in a file for a module declaration and just parse a stub. + // See `recurse_into_file_modules` in the parser. + #[test] + fn out_of_line_mod() { + let sess = ParseSess::new(FilePathMapping::empty()); + let item = parse_item_from_source_str( + "foo".to_owned(), + "mod foo { struct S; mod this_does_not_exist; }".to_owned(), + &sess, + ).unwrap().unwrap(); + + if let ast::ItemKind::Mod(ref m) = item.node { + assert!(m.items.len() == 2); + } else { + panic!(); + } + } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 4741f896d3cc0..c28f678cb5197 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -179,6 +179,8 @@ pub struct Parser<'a> { pub obsolete_set: HashSet, /// Used to determine the path to externally loaded source files pub directory: Directory, + /// Whether to parse sub-modules in other files. + pub recurse_into_file_modules: bool, /// Name of the root module this parser originated from. If `None`, then the /// name is not known. This does not change while the parser is descending /// into modules, and sub-parsers have new values for this name. @@ -190,6 +192,7 @@ pub struct Parser<'a> { pub cfg_mods: bool, } + struct TokenCursor { frame: TokenCursorFrame, stack: Vec, @@ -439,6 +442,7 @@ impl<'a> Parser<'a> { pub fn new(sess: &'a ParseSess, tokens: TokenStream, directory: Option, + recurse_into_file_modules: bool, desugar_doc_comments: bool) -> Self { let mut parser = Parser { @@ -450,6 +454,7 @@ impl<'a> Parser<'a> { prev_token_kind: PrevTokenKind::Other, restrictions: Restrictions::empty(), obsolete_set: HashSet::new(), + recurse_into_file_modules: recurse_into_file_modules, directory: Directory { path: PathBuf::new(), ownership: DirectoryOwnership::Owned }, root_module_name: None, expected_tokens: Vec::new(), @@ -467,12 +472,14 @@ impl<'a> Parser<'a> { let tok = parser.next_tok(); parser.token = tok.tok; parser.span = tok.sp; + if let Some(directory) = directory { parser.directory = directory; } else if parser.span != syntax_pos::DUMMY_SP { parser.directory.path = PathBuf::from(sess.codemap().span_to_filename(parser.span)); parser.directory.path.pop(); } + parser.process_potential_macro_variable(); parser } @@ -3921,6 +3928,7 @@ impl<'a> Parser<'a> { mem::replace(&mut self.directory.ownership, DirectoryOwnership::UnownedViaBlock); let item = self.parse_item_(attrs.clone(), false, true)?; self.directory.ownership = old_directory_ownership; + match item { Some(i) => Stmt { id: ast::DUMMY_NODE_ID, @@ -5254,7 +5262,7 @@ impl<'a> Parser<'a> { let id = self.parse_ident()?; if self.check(&token::Semi) { self.bump(); - if in_cfg { + if in_cfg && self.recurse_into_file_modules { // This mod is in an external file. Let's go get it! let ModulePathSuccess { path, directory_ownership, warn } = self.submod_path(id, &outer_attrs, id_span)?; @@ -5281,10 +5289,12 @@ impl<'a> Parser<'a> { } else { let old_directory = self.directory.clone(); self.push_directory(id, &outer_attrs); + self.expect(&token::OpenDelim(token::Brace))?; let mod_inner_lo = self.span; let attrs = self.parse_inner_attributes()?; let module = self.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo)?; + self.directory = old_directory; Ok((id, ItemKind::Mod(module), Some(attrs))) } @@ -5347,7 +5357,8 @@ impl<'a> Parser<'a> { fn submod_path(&mut self, id: ast::Ident, outer_attrs: &[ast::Attribute], - id_sp: Span) -> PResult<'a, ModulePathSuccess> { + id_sp: Span) + -> PResult<'a, ModulePathSuccess> { if let Some(path) = Parser::submod_path_from_attr(outer_attrs, &self.directory.path) { return Ok(ModulePathSuccess { directory_ownership: match path.file_name().and_then(|s| s.to_str()) { diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 9c1371a31fec7..339e7c0b628ad 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -109,7 +109,7 @@ impl TokenTree { path: cx.current_expansion.module.directory.clone(), ownership: cx.current_expansion.directory_ownership, }; - macro_parser::parse(cx.parse_sess(), tts, mtch, Some(directory)) + macro_parser::parse(cx.parse_sess(), tts, mtch, Some(directory), true) } /// Check if this TokenTree is equal to the other, regardless of span information.