Resolve documentation links in rustc and store the results in metadata

This commit implements MCP rust-lang/compiler-team#584

It also removes code that is no longer used, and that includes code cloning resolver, so issue rust-lang#83761 is fixed.

Cargo.lock

@@ -4613,6 +4613,7 @@ name = "rustc_resolve"
 version = "0.0.0"
 dependencies = [
  "bitflags",
+ "pulldown-cmark 0.9.2",
  "rustc_arena",
  "rustc_ast",
  "rustc_ast_pretty",
@@ -4878,7 +4879,6 @@ dependencies = [
  "itertools",
  "minifier",
  "once_cell",
- "pulldown-cmark 0.9.2",
  "rayon",
  "regex",
  "rustdoc-json-types",

compiler/rustc_data_structures/src/stable_hasher.rs

@@ -486,6 +486,14 @@ impl<HCX> ToStableHashKey<HCX> for String {
     }
 }
 
+impl<HCX, T1: ToStableHashKey<HCX>, T2: ToStableHashKey<HCX>> ToStableHashKey<HCX> for (T1, T2) {
+    type KeyType = (T1::KeyType, T2::KeyType);
+    #[inline]
+    fn to_stable_hash_key(&self, hcx: &HCX) -> Self::KeyType {
+        (self.0.to_stable_hash_key(hcx), self.1.to_stable_hash_key(hcx))
+    }
+}
+
 impl<CTX> HashStable<CTX> for bool {
     #[inline]
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {

compiler/rustc_hir/src/def.rs

@@ -2,6 +2,8 @@ use crate::hir;
 
 use rustc_ast as ast;
 use rustc_ast::NodeId;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::ToStableHashKey;
 use rustc_macros::HashStable_Generic;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::hygiene::MacroKind;
@@ -472,7 +474,8 @@ impl PartialRes {
 
 /// Different kinds of symbols can coexist even if they share the same textual name.
 /// Therefore, they each have a separate universe (known as a "namespace").
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Encodable, Decodable)]
+#[derive(HashStable_Generic)]
 pub enum Namespace {
     /// The type namespace includes `struct`s, `enum`s, `union`s, `trait`s, and `mod`s
     /// (and, by extension, crates).
@@ -499,6 +502,15 @@ impl Namespace {
     }
 }
 
+impl<CTX: crate::HashStableContext> ToStableHashKey<CTX> for Namespace {
+    type KeyType = Namespace;
+
+    #[inline]
+    fn to_stable_hash_key(&self, _: &CTX) -> Namespace {
+        *self
+    }
+}
+
 /// Just a helper ‒ separate structure for each namespace.
 #[derive(Copy, Clone, Default, Debug)]
 pub struct PerNS<T> {
@@ -760,3 +772,5 @@ pub enum LifetimeRes {
     /// HACK: This is used to recover the NodeId of an elided lifetime.
     ElidedAnchor { start: NodeId, end: NodeId },
 }
+
+pub type DocLinkResMap = FxHashMap<(Symbol, Namespace), Option<Res<NodeId>>>;

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -11,7 +11,7 @@ use rustc_data_structures::sync::{Lock, LockGuard, Lrc, OnceCell};
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::proc_macro::{AttrProcMacro, BangProcMacro, DeriveProcMacro};
-use rustc_hir::def::{CtorKind, DefKind, Res};
+use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
 use rustc_hir::diagnostic_items::DiagnosticItems;
@@ -1163,20 +1163,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         )
     }
 
-    /// Decodes all inherent impls in the crate (for rustdoc).
-    fn get_inherent_impls(self) -> impl Iterator<Item = (DefId, DefId)> + 'a {
-        (0..self.root.tables.inherent_impls.size()).flat_map(move |i| {
-            let ty_index = DefIndex::from_usize(i);
-            let ty_def_id = self.local_def_id(ty_index);
-            self.root
-                .tables
-                .inherent_impls
-                .get(self, ty_index)
-                .decode(self)
-                .map(move |impl_index| (ty_def_id, self.local_def_id(impl_index)))
-        })
-    }
-
     /// Decodes all traits in the crate (for rustdoc and rustc diagnostics).
     fn get_traits(self) -> impl Iterator<Item = DefId> + 'a {
         self.root.traits.decode(self).map(move |index| self.local_def_id(index))
@@ -1195,13 +1181,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         })
     }
 
-    fn get_all_incoherent_impls(self) -> impl Iterator<Item = DefId> + 'a {
-        self.cdata
-            .incoherent_impls
-            .values()
-            .flat_map(move |impls| impls.decode(self).map(move |idx| self.local_def_id(idx)))
-    }
-
     fn get_incoherent_impls(self, tcx: TyCtxt<'tcx>, simp: SimplifiedType) -> &'tcx [DefId] {
         if let Some(impls) = self.cdata.incoherent_impls.get(&simp) {
             tcx.arena.alloc_from_iter(impls.decode(self).map(|idx| self.local_def_id(idx)))
@@ -1598,6 +1577,24 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     fn get_is_intrinsic(self, index: DefIndex) -> bool {
         self.root.tables.is_intrinsic.get(self, index)
     }
+
+    fn get_doc_link_resolutions(self, index: DefIndex) -> DocLinkResMap {
+        self.root
+            .tables
+            .doc_link_resolutions
+            .get(self, index)
+            .expect("no resolutions for a doc link")
+            .decode(self)
+    }
+
+    fn get_doc_link_traits_in_scope(self, index: DefIndex) -> impl Iterator<Item = DefId> + 'a {
+        self.root
+            .tables
+            .doc_link_traits_in_scope
+            .get(self, index)
+            .expect("no traits in scope for a doc link")
+            .decode(self)
+    }
 }
 
 impl CrateMetadata {

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -345,6 +345,10 @@ provide! { tcx, def_id, other, cdata,
     expn_that_defined => { cdata.get_expn_that_defined(def_id.index, tcx.sess) }
     generator_diagnostic_data => { cdata.get_generator_diagnostic_data(tcx, def_id.index) }
     is_doc_hidden => { cdata.get_attr_flags(def_id.index).contains(AttrFlags::IS_DOC_HIDDEN) }
+    doc_link_resolutions => { tcx.arena.alloc(cdata.get_doc_link_resolutions(def_id.index)) }
+    doc_link_traits_in_scope => {
+        tcx.arena.alloc_from_iter(cdata.get_doc_link_traits_in_scope(def_id.index))
+    }
 }
 
 pub(in crate::rmeta) fn provide(providers: &mut Providers) {
@@ -613,36 +617,6 @@ impl CStore {
         self.get_crate_data(cnum).get_trait_impls()
     }
 
-    /// Decodes all inherent impls in the crate (for rustdoc).
-    pub fn inherent_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = (DefId, DefId)> + '_ {
-        self.get_crate_data(cnum).get_inherent_impls()
-    }
-
-    /// Decodes all incoherent inherent impls in the crate (for rustdoc).
-    pub fn incoherent_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = DefId> + '_ {
-        self.get_crate_data(cnum).get_all_incoherent_impls()
-    }
-
-    pub fn associated_item_def_ids_untracked<'a>(
-        &'a self,
-        def_id: DefId,
-        sess: &'a Session,
-    ) -> impl Iterator<Item = DefId> + 'a {
-        self.get_crate_data(def_id.krate).get_associated_item_def_ids(def_id.index, sess)
-    }
-
-    pub fn may_have_doc_links_untracked(&self, def_id: DefId) -> bool {
-        self.get_crate_data(def_id.krate)
-            .get_attr_flags(def_id.index)
-            .contains(AttrFlags::MAY_HAVE_DOC_LINKS)
-    }
-
     pub fn is_doc_hidden_untracked(&self, def_id: DefId) -> bool {
         self.get_crate_data(def_id.krate)
             .get_attr_flags(def_id.index)

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -3,7 +3,6 @@ use crate::rmeta::def_path_hash_map::DefPathHashMapRef;
 use crate::rmeta::table::TableBuilder;
 use crate::rmeta::*;
 
-use rustc_ast::util::comments;
 use rustc_ast::Attribute;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
@@ -772,7 +771,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
 struct AnalyzeAttrState {
     is_exported: bool,
-    may_have_doc_links: bool,
     is_doc_hidden: bool,
 }
 
@@ -790,15 +788,12 @@ fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState) -> bool {
     let mut should_encode = false;
     if rustc_feature::is_builtin_only_local(attr.name_or_empty()) {
         // Attributes marked local-only don't need to be encoded for downstream crates.
-    } else if let Some(s) = attr.doc_str() {
+    } else if attr.doc_str().is_some() {
         // We keep all doc comments reachable to rustdoc because they might be "imported" into
         // downstream crates if they use `#[doc(inline)]` to copy an item's documentation into
         // their own.
         if state.is_exported {
             should_encode = true;
-            if comments::may_have_doc_links(s.as_str()) {
-                state.may_have_doc_links = true;
-            }
         }
     } else if attr.has_name(sym::doc) {
         // If this is a `doc` attribute that doesn't have anything except maybe `inline` (as in
@@ -1139,7 +1134,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let tcx = self.tcx;
         let mut state = AnalyzeAttrState {
             is_exported: tcx.effective_visibilities(()).is_exported(def_id),
-            may_have_doc_links: false,
             is_doc_hidden: false,
         };
         let attr_iter = tcx
@@ -1151,9 +1145,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         record_array!(self.tables.attributes[def_id.to_def_id()] <- attr_iter);
 
         let mut attr_flags = AttrFlags::empty();
-        if state.may_have_doc_links {
-            attr_flags |= AttrFlags::MAY_HAVE_DOC_LINKS;
-        }
         if state.is_doc_hidden {
             attr_flags |= AttrFlags::IS_DOC_HIDDEN;
         }
@@ -1231,6 +1222,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_id.index
             }));
         }
+
+        for (def_id, res_map) in &tcx.resolutions(()).doc_link_resolutions {
+            record!(self.tables.doc_link_resolutions[def_id.to_def_id()] <- res_map);
+        }
+
+        for (def_id, traits) in &tcx.resolutions(()).doc_link_traits_in_scope {
+            record_array!(self.tables.doc_link_traits_in_scope[def_id.to_def_id()] <- traits);
+        }
     }
 
     #[instrument(level = "trace", skip(self))]
@@ -1715,6 +1714,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.lookup_stability[LOCAL_CRATE.as_def_id()] <- stability);
             }
             self.encode_deprecation(LOCAL_CRATE.as_def_id());
+            if let Some(res_map) = tcx.resolutions(()).doc_link_resolutions.get(&CRATE_DEF_ID) {
+                record!(self.tables.doc_link_resolutions[LOCAL_CRATE.as_def_id()] <- res_map);
+            }
+            if let Some(traits) = tcx.resolutions(()).doc_link_traits_in_scope.get(&CRATE_DEF_ID) {
+                record_array!(self.tables.doc_link_traits_in_scope[LOCAL_CRATE.as_def_id()] <- traits);
+            }
 
             // Normally, this information is encoded when we walk the items
             // defined in this crate. However, we skip doing that for proc-macro crates,
@@ -2225,6 +2230,18 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
 
 pub fn provide(providers: &mut Providers) {
     *providers = Providers {
+        doc_link_resolutions: |tcx, def_id| {
+            tcx.resolutions(())
+                .doc_link_resolutions
+                .get(&def_id.expect_local())
+                .expect("no resolutions for a doc link")
+        },
+        doc_link_traits_in_scope: |tcx, def_id| {
+            tcx.resolutions(())
+                .doc_link_traits_in_scope
+                .get(&def_id.expect_local())
+                .expect("no traits in scope for a doc link")
+        },
         traits_in_crate: |tcx, cnum| {
             assert_eq!(cnum, LOCAL_CRATE);
 

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -9,7 +9,7 @@ use rustc_attr as attr;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, DefKind};
+use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap};
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, DefPathHash, StableCrateId};
 use rustc_hir::definitions::DefKey;
 use rustc_hir::lang_items::LangItem;
@@ -413,6 +413,8 @@ define_tables! {
     module_reexports: Table<DefIndex, LazyArray<ModChild>>,
     deduced_param_attrs: Table<DefIndex, LazyArray<DeducedParamAttrs>>,
     trait_impl_trait_tys: Table<DefIndex, LazyValue<FxHashMap<DefId, Ty<'static>>>>,
+    doc_link_resolutions: Table<DefIndex, LazyValue<DocLinkResMap>>,
+    doc_link_traits_in_scope: Table<DefIndex, LazyArray<DefId>>,
 }
 
 #[derive(TyEncodable, TyDecodable)]
@@ -426,8 +428,7 @@ struct VariantData {
 bitflags::bitflags! {
     #[derive(Default)]
     pub struct AttrFlags: u8 {
-        const MAY_HAVE_DOC_LINKS = 1 << 0;
-        const IS_DOC_HIDDEN      = 1 << 1;
+        const IS_DOC_HIDDEN = 1 << 0;
     }
 }
 

compiler/rustc_middle/src/arena.rs

@@ -113,6 +113,7 @@ macro_rules! arena_types {
             [decode] trait_impl_trait_tys: rustc_data_structures::fx::FxHashMap<rustc_hir::def_id::DefId, rustc_middle::ty::Ty<'tcx>>,
             [] bit_set_u32: rustc_index::bit_set::BitSet<u32>,
             [] external_constraints: rustc_middle::traits::solve::ExternalConstraintsData<'tcx>,
+            [decode] doc_link_resolutions: rustc_hir::def::DocLinkResMap,
         ]);
     )
 }

compiler/rustc_middle/src/query/mod.rs

@@ -2156,4 +2156,16 @@ rustc_queries! {
         desc { |tcx| "deducing parameter attributes for {}", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
+
+    query doc_link_resolutions(def_id: DefId) -> &'tcx DocLinkResMap {
+        eval_always
+        desc { "resolutions for documentation links for a module" }
+        separate_provide_extern
+    }
+
+    query doc_link_traits_in_scope(def_id: DefId) -> &'tcx [DefId] {
+        eval_always
+        desc { "traits in scope for documentation links for a module" }
+        separate_provide_extern
+    }
 }

compiler/rustc_middle/src/ty/mod.rs

@@ -36,7 +36,7 @@ use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, CtorOf, DefKind, LifetimeRes, Res};
+use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
 use rustc_hir::Node;
 use rustc_index::vec::IndexVec;
@@ -181,6 +181,8 @@ pub struct ResolverGlobalCtxt {
     /// exist under `std`. For example, wrote `str::from_utf8` instead of `std::str::from_utf8`.
     pub confused_type_with_std_module: FxHashMap<Span, Span>,
     pub registered_tools: RegisteredTools,
+    pub doc_link_resolutions: FxHashMap<LocalDefId, DocLinkResMap>,
+    pub doc_link_traits_in_scope: FxHashMap<LocalDefId, Vec<DefId>>,
 }
 
 /// Resolutions that should only be used for lowering.

compiler/rustc_middle/src/ty/parameterized.rs

@@ -81,6 +81,8 @@ trivially_parameterized_over_tcx! {
     rustc_hir::IsAsync,
     rustc_hir::LangItem,
     rustc_hir::def::DefKind,
+    rustc_hir::def::DocLinkResMap,
+    rustc_hir::def_id::DefId,
     rustc_hir::def_id::DefIndex,
     rustc_hir::definitions::DefKey,
     rustc_index::bit_set::BitSet<u32>,

compiler/rustc_middle/src/ty/query.rs

@@ -45,7 +45,7 @@ use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
+use rustc_hir::def::{DefKind, DocLinkResMap};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
 use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};

compiler/rustc_resolve/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 bitflags = "1.2.1"
+pulldown-cmark = { version = "0.9.2", default-features = false }
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }