Rollup merge of rust-lang#82238 - petrochenkov:nocratemod, r=Aaron1011

ast: Keep expansion status for out-of-line module items

I.e. whether a module `mod foo;` is already loaded from a file or not.
This is a pre-requisite to correctly treating inner attributes on such modules (rust-lang#81661).

With this change AST structures for `mod` items diverge even more for AST structure for the crate root, which previously used `ast::Mod`.
Therefore this PR removes `ast::Mod` from `ast::Crate` in the first commit, these two things are sufficiently different from each other, at least at syntactic level.
Customization points for visiting a "`mod` item or crate root" were also removed from AST visitors (`fn visit_mod`).
`ast::Mod` itself was refactored away in the second commit in favor of `ItemKind::Mod(Unsafe, ModKind)`.

compiler/rustc_ast/src/ast.rs

@@ -486,8 +486,8 @@ pub struct WhereEqPredicate {
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct Crate {
-    pub module: Mod,
     pub attrs: Vec<Attribute>,
+    pub items: Vec<P<Item>>,
     pub span: Span,
     /// The order of items in the HIR is unrelated to the order of
     /// items in the AST. However, we generate proc macro harnesses
@@ -2299,21 +2299,22 @@ impl FnRetTy {
     }
 }
 
-/// Module declaration.
-///
-/// E.g., `mod foo;` or `mod foo { .. }`.
+#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
+pub enum Inline {
+    Yes,
+    No,
+}
+
+/// Module item kind.
 #[derive(Clone, Encodable, Decodable, Debug)]
-pub struct Mod {
-    /// A span from the first token past `{` to the last token until `}`.
-    /// For `mod foo;`, the inner span ranges from the first token
-    /// to the last token in the external file.
-    pub inner: Span,
-    /// `unsafe` keyword accepted syntactically for macro DSLs, but not
-    /// semantically by Rust.
-    pub unsafety: Unsafe,
-    pub items: Vec<P<Item>>,
-    /// `true` for `mod foo { .. }`; `false` for `mod foo;`.
-    pub inline: bool,
+pub enum ModKind {
+    /// Module with inlined definition `mod foo { ... }`,
+    /// or with definition outlined to a separate file `mod foo;` and already loaded from it.
+    /// The inner span is from the first token past `{` to the last token until `}`,
+    /// or from the first to the last token in the loaded file.
+    Loaded(Vec<P<Item>>, Inline, Span),
+    /// Module with definition outlined to a separate file `mod foo;` but not yet loaded from it.
+    Unloaded,
 }
 
 /// Foreign module declaration.
@@ -2710,7 +2711,9 @@ pub enum ItemKind {
     /// A module declaration (`mod`).
     ///
     /// E.g., `mod foo;` or `mod foo { .. }`.
-    Mod(Mod),
+    /// `unsafe` keyword on modules is accepted syntactically for macro DSLs, but not
+    /// semantically by Rust.
+    Mod(Unsafe, ModKind),
     /// An external module (`extern`).
     ///
     /// E.g., `extern {}` or `extern "C" {}`.

compiler/rustc_ast/src/mut_visit.rs

@@ -170,10 +170,6 @@ pub trait MutVisitor: Sized {
         noop_visit_ty_constraint(t, self);
     }
 
-    fn visit_mod(&mut self, m: &mut Mod) {
-        noop_visit_mod(m, self);
-    }
-
     fn visit_foreign_mod(&mut self, nm: &mut ForeignMod) {
         noop_visit_foreign_mod(nm, self);
     }
@@ -917,7 +913,13 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
             vis.visit_generics(generics);
             visit_opt(body, |body| vis.visit_block(body));
         }
-        ItemKind::Mod(m) => vis.visit_mod(m),
+        ItemKind::Mod(_unsafety, mod_kind) => match mod_kind {
+            ModKind::Loaded(items, _inline, inner_span) => {
+                vis.visit_span(inner_span);
+                items.flat_map_in_place(|item| vis.flat_map_item(item));
+            }
+            ModKind::Unloaded => {}
+        },
         ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
         ItemKind::GlobalAsm(_ga) => {}
         ItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
@@ -998,14 +1000,10 @@ pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
     vis.visit_asyncness(asyncness);
 }
 
-pub fn noop_visit_mod<T: MutVisitor>(module: &mut Mod, vis: &mut T) {
-    let Mod { inner, unsafety: _, items, inline: _ } = module;
-    vis.visit_span(inner);
-    items.flat_map_in_place(|item| vis.flat_map_item(item));
-}
-
+// FIXME: Avoid visiting the crate as a `Mod` item, flat map only the inner items if possible,
+// or make crate visiting first class if necessary.
 pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) {
-    visit_clobber(krate, |Crate { module, attrs, span, proc_macros }| {
+    visit_clobber(krate, |Crate { attrs, items, span, proc_macros }| {
         let item_vis =
             Visibility { kind: VisibilityKind::Public, span: span.shrink_to_lo(), tokens: None };
         let item = P(Item {
@@ -1014,19 +1012,20 @@ pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) {
             id: DUMMY_NODE_ID,
             vis: item_vis,
             span,
-            kind: ItemKind::Mod(module),
+            kind: ItemKind::Mod(Unsafe::No, ModKind::Loaded(items, Inline::Yes, span)),
             tokens: None,
         });
         let items = vis.flat_map_item(item);
 
         let len = items.len();
         if len == 0 {
-            let module = Mod { inner: span, unsafety: Unsafe::No, items: vec![], inline: true };
-            Crate { module, attrs: vec![], span, proc_macros }
+            Crate { attrs: vec![], items: vec![], span, proc_macros }
         } else if len == 1 {
             let Item { attrs, span, kind, .. } = items.into_iter().next().unwrap().into_inner();
             match kind {
-                ItemKind::Mod(module) => Crate { module, attrs, span, proc_macros },
+                ItemKind::Mod(_, ModKind::Loaded(items, ..)) => {
+                    Crate { attrs, items, span, proc_macros }
+                }
                 _ => panic!("visitor converted a module to not a module"),
             }
         } else {

compiler/rustc_ast/src/visit.rs

@@ -74,7 +74,7 @@ impl<'a> FnKind<'a> {
 /// Each method of the `Visitor` trait is a hook to be potentially
 /// overridden. Each method's default implementation recursively visits
 /// the substructure of the input via the corresponding `walk` method;
-/// e.g., the `visit_mod` method by default calls `visit::walk_mod`.
+/// e.g., the `visit_item` method by default calls `visit::walk_item`.
 ///
 /// If you want to ensure that your code handles every variant
 /// explicitly, you need to override each method. (And you also need
@@ -87,9 +87,6 @@ pub trait Visitor<'ast>: Sized {
     fn visit_ident(&mut self, ident: Ident) {
         walk_ident(self, ident);
     }
-    fn visit_mod(&mut self, m: &'ast Mod, _s: Span, _attrs: &[Attribute], _n: NodeId) {
-        walk_mod(self, m);
-    }
     fn visit_foreign_item(&mut self, i: &'ast ForeignItem) {
         walk_foreign_item(self, i)
     }
@@ -238,14 +235,10 @@ pub fn walk_ident<'a, V: Visitor<'a>>(visitor: &mut V, ident: Ident) {
 }
 
 pub fn walk_crate<'a, V: Visitor<'a>>(visitor: &mut V, krate: &'a Crate) {
-    visitor.visit_mod(&krate.module, krate.span, &krate.attrs, CRATE_NODE_ID);
+    walk_list!(visitor, visit_item, &krate.items);
     walk_list!(visitor, visit_attribute, &krate.attrs);
 }
 
-pub fn walk_mod<'a, V: Visitor<'a>>(visitor: &mut V, module: &'a Mod) {
-    walk_list!(visitor, visit_item, &module.items);
-}
-
 pub fn walk_local<'a, V: Visitor<'a>>(visitor: &mut V, local: &'a Local) {
     for attr in local.attrs.iter() {
         visitor.visit_attribute(attr);
@@ -297,7 +290,12 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
             let kind = FnKind::Fn(FnCtxt::Free, item.ident, sig, &item.vis, body.as_deref());
             visitor.visit_fn(kind, item.span, item.id)
         }
-        ItemKind::Mod(ref module) => visitor.visit_mod(module, item.span, &item.attrs, item.id),
+        ItemKind::Mod(_unsafety, ref mod_kind) => match mod_kind {
+            ModKind::Loaded(items, _inline, _inner_span) => {
+                walk_list!(visitor, visit_item, items)
+            }
+            ModKind::Unloaded => {}
+        },
         ItemKind::ForeignMod(ref foreign_module) => {
             walk_list!(visitor, visit_foreign_item, &foreign_module.items);
         }

compiler/rustc_ast_lowering/src/item.rs

@@ -15,11 +15,11 @@ use rustc_span::source_map::{respan, DesugaringKind};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Span;
 use rustc_target::spec::abi;
-
 use smallvec::{smallvec, SmallVec};
-use std::collections::BTreeSet;
 use tracing::debug;
 
+use std::mem;
+
 pub(super) struct ItemLowerer<'a, 'lowering, 'hir> {
     pub(super) lctx: &'a mut LoweringContext<'lowering, 'hir>,
 }
@@ -34,25 +34,6 @@ impl ItemLowerer<'_, '_, '_> {
 }
 
 impl<'a> Visitor<'a> for ItemLowerer<'a, '_, '_> {
-    fn visit_mod(&mut self, m: &'a Mod, _s: Span, _attrs: &[Attribute], n: NodeId) {
-        let def_id = self.lctx.lower_node_id(n).expect_owner();
-
-        self.lctx.modules.insert(
-            def_id,
-            hir::ModuleItems {
-                items: BTreeSet::new(),
-                trait_items: BTreeSet::new(),
-                impl_items: BTreeSet::new(),
-                foreign_items: BTreeSet::new(),
-            },
-        );
-
-        let old = self.lctx.current_module;
-        self.lctx.current_module = def_id;
-        visit::walk_mod(self, m);
-        self.lctx.current_module = old;
-    }
-
     fn visit_item(&mut self, item: &'a Item) {
         let mut item_hir_id = None;
         self.lctx.with_hir_id_owner(item.id, |lctx| {
@@ -67,10 +48,18 @@ impl<'a> Visitor<'a> for ItemLowerer<'a, '_, '_> {
         if let Some(hir_id) = item_hir_id {
             self.lctx.with_parent_item_lifetime_defs(hir_id, |this| {
                 let this = &mut ItemLowerer { lctx: this };
-                if let ItemKind::Impl(box ImplKind { ref of_trait, .. }) = item.kind {
-                    this.with_trait_impl_ref(of_trait, |this| visit::walk_item(this, item));
-                } else {
-                    visit::walk_item(this, item);
+                match item.kind {
+                    ItemKind::Mod(..) => {
+                        let def_id = this.lctx.lower_node_id(item.id).expect_owner();
+                        let old_current_module =
+                            mem::replace(&mut this.lctx.current_module, def_id);
+                        visit::walk_item(this, item);
+                        this.lctx.current_module = old_current_module;
+                    }
+                    ItemKind::Impl(box ImplKind { ref of_trait, .. }) => {
+                        this.with_trait_impl_ref(of_trait, |this| visit::walk_item(this, item));
+                    }
+                    _ => visit::walk_item(this, item),
                 }
             });
         }
@@ -94,13 +83,13 @@ impl<'a> Visitor<'a> for ItemLowerer<'a, '_, '_> {
                 let hir_item = lctx.lower_trait_item(item);
                 let id = hir_item.trait_item_id();
                 lctx.trait_items.insert(id, hir_item);
-                lctx.modules.get_mut(&lctx.current_module).unwrap().trait_items.insert(id);
+                lctx.modules.entry(lctx.current_module).or_default().trait_items.insert(id);
             }
             AssocCtxt::Impl => {
                 let hir_item = lctx.lower_impl_item(item);
                 let id = hir_item.impl_item_id();
                 lctx.impl_items.insert(id, hir_item);
-                lctx.modules.get_mut(&lctx.current_module).unwrap().impl_items.insert(id);
+                lctx.modules.entry(lctx.current_module).or_default().impl_items.insert(id);
             }
         });
 
@@ -113,7 +102,7 @@ impl<'a> Visitor<'a> for ItemLowerer<'a, '_, '_> {
             let hir_item = lctx.lower_foreign_item(item);
             let id = hir_item.foreign_item_id();
             lctx.foreign_items.insert(id, hir_item);
-            lctx.modules.get_mut(&lctx.current_module).unwrap().foreign_items.insert(id);
+            lctx.modules.entry(lctx.current_module).or_default().foreign_items.insert(id);
         });
 
         visit::walk_foreign_item(self, item);
@@ -157,7 +146,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         &mut self,
         f: impl FnOnce(&mut LoweringContext<'_, '_>) -> T,
     ) -> T {
-        let old_in_scope_lifetimes = std::mem::replace(&mut self.in_scope_lifetimes, vec![]);
+        let old_in_scope_lifetimes = mem::replace(&mut self.in_scope_lifetimes, vec![]);
 
         // this vector is only used when walking over impl headers,
         // input types, and the like, and should not be non-empty in
@@ -172,12 +161,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
         res
     }
 
-    pub(super) fn lower_mod(&mut self, m: &Mod) -> hir::Mod<'hir> {
+    pub(super) fn lower_mod(&mut self, items: &[P<Item>], inner: Span) -> hir::Mod<'hir> {
         hir::Mod {
-            inner: m.inner,
-            item_ids: self
-                .arena
-                .alloc_from_iter(m.items.iter().flat_map(|x| self.lower_item_id(x))),
+            inner,
+            item_ids: self.arena.alloc_from_iter(items.iter().flat_map(|x| self.lower_item_id(x))),
         }
     }
 
@@ -327,7 +314,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     hir::ItemKind::Fn(sig, generics, body_id)
                 })
             }
-            ItemKind::Mod(ref m) => hir::ItemKind::Mod(self.lower_mod(m)),
+            ItemKind::Mod(_, ref mod_kind) => match mod_kind {
+                ModKind::Loaded(items, _, inner_span) => {
+                    hir::ItemKind::Mod(self.lower_mod(items, *inner_span))
+                }
+                ModKind::Unloaded => panic!("`mod` items should have been loaded by now"),
+            },
             ItemKind::ForeignMod(ref fm) => {
                 if fm.abi.is_none() {
                     self.maybe_lint_missing_abi(span, id, abi::Abi::C);

compiler/rustc_ast_lowering/src/lib.rs

@@ -560,7 +560,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         visit::walk_crate(&mut MiscCollector { lctx: &mut self }, c);
         visit::walk_crate(&mut item::ItemLowerer { lctx: &mut self }, c);
 
-        let module = self.lower_mod(&c.module);
+        let module = self.lower_mod(&c.items, c.span);
         let attrs = self.lower_attrs(&c.attrs);
         let body_ids = body_ids(&self.bodies);
         let proc_macros =
@@ -608,7 +608,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn insert_item(&mut self, item: hir::Item<'hir>) -> hir::ItemId {
         let id = hir::ItemId { def_id: item.def_id };
         self.items.insert(id, item);
-        self.modules.get_mut(&self.current_module).unwrap().items.insert(id);
+        self.modules.entry(self.current_module).or_default().items.insert(id);
         id
     }
 

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -1054,12 +1054,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 walk_list!(self, visit_attribute, &item.attrs);
                 return;
             }
-            ItemKind::Mod(Mod { inline, unsafety, .. }) => {
+            ItemKind::Mod(unsafety, ref mod_kind) => {
                 if let Unsafe::Yes(span) = unsafety {
                     self.err_handler().span_err(span, "module cannot be declared unsafe");
                 }
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
-                if !inline && !self.session.contains_name(&item.attrs, sym::path) {
+                if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _))
+                    && !self.session.contains_name(&item.attrs, sym::path)
+                {
                     self.check_mod_file_item_asciionly(item.ident);
                 }
             }

compiler/rustc_ast_passes/src/node_count.rs

@@ -20,10 +20,6 @@ impl<'ast> Visitor<'ast> for NodeCounter {
         self.count += 1;
         walk_ident(self, ident);
     }
-    fn visit_mod(&mut self, m: &Mod, _s: Span, _a: &[Attribute], _n: NodeId) {
-        self.count += 1;
-        walk_mod(self, m)
-    }
     fn visit_foreign_item(&mut self, i: &ForeignItem) {
         self.count += 1;
         walk_foreign_item(self, i)