Auto merge of #94857 - petrochenkov:doclink2, r=oli-obk

Resolve documentation links in rustc and store the results in metadata

This PR implements MCP rust-lang/compiler-team#584.

Doc links are now resolved in rustc and stored into metadata, so rustdoc simply retrieves them through a query (local or extern),

Code that is no longer used is removed, and some code that no longer needs to be public is privatized.
The removed code includes resolver cloning, so this PR fixes #83761.

Author: bors

Cargo.lock

@@ -4615,6 +4615,7 @@ name = "rustc_resolve"
 version = "0.0.0"
 dependencies = [
  "bitflags",
+ "pulldown-cmark 0.9.2",
  "rustc_arena",
  "rustc_ast",
  "rustc_ast_pretty",
@@ -4881,7 +4882,6 @@ dependencies = [
  "itertools",
  "minifier",
  "once_cell",
- "pulldown-cmark 0.9.2",
  "rayon",
  "regex",
  "rustdoc-json-types",

compiler/rustc_data_structures/src/stable_hasher.rs

@@ -486,6 +486,14 @@ impl<HCX> ToStableHashKey<HCX> for String {
     }
 }
 
+impl<HCX, T1: ToStableHashKey<HCX>, T2: ToStableHashKey<HCX>> ToStableHashKey<HCX> for (T1, T2) {
+    type KeyType = (T1::KeyType, T2::KeyType);
+    #[inline]
+    fn to_stable_hash_key(&self, hcx: &HCX) -> Self::KeyType {
+        (self.0.to_stable_hash_key(hcx), self.1.to_stable_hash_key(hcx))
+    }
+}
+
 impl<CTX> HashStable<CTX> for bool {
     #[inline]
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {

compiler/rustc_hir/src/def.rs

@@ -2,6 +2,8 @@ use crate::hir;
 
 use rustc_ast as ast;
 use rustc_ast::NodeId;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::ToStableHashKey;
 use rustc_macros::HashStable_Generic;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::hygiene::MacroKind;
@@ -472,7 +474,8 @@ impl PartialRes {
 
 /// Different kinds of symbols can coexist even if they share the same textual name.
 /// Therefore, they each have a separate universe (known as a "namespace").
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Encodable, Decodable)]
+#[derive(HashStable_Generic)]
 pub enum Namespace {
     /// The type namespace includes `struct`s, `enum`s, `union`s, `trait`s, and `mod`s
     /// (and, by extension, crates).
@@ -499,6 +502,15 @@ impl Namespace {
     }
 }
 
+impl<CTX: crate::HashStableContext> ToStableHashKey<CTX> for Namespace {
+    type KeyType = Namespace;
+
+    #[inline]
+    fn to_stable_hash_key(&self, _: &CTX) -> Namespace {
+        *self
+    }
+}
+
 /// Just a helper ‒ separate structure for each namespace.
 #[derive(Copy, Clone, Default, Debug)]
 pub struct PerNS<T> {
@@ -760,3 +772,5 @@ pub enum LifetimeRes {
     /// HACK: This is used to recover the NodeId of an elided lifetime.
     ElidedAnchor { start: NodeId, end: NodeId },
 }
+
+pub type DocLinkResMap = FxHashMap<(Symbol, Namespace), Option<Res<NodeId>>>;

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -11,7 +11,7 @@ use rustc_data_structures::sync::{Lock, LockGuard, Lrc, OnceCell};
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::proc_macro::{AttrProcMacro, BangProcMacro, DeriveProcMacro};
-use rustc_hir::def::{CtorKind, DefKind, Res};
+use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
 use rustc_hir::diagnostic_items::DiagnosticItems;
@@ -1163,20 +1163,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         )
     }
 
-    /// Decodes all inherent impls in the crate (for rustdoc).
-    fn get_inherent_impls(self) -> impl Iterator<Item = (DefId, DefId)> + 'a {
-        (0..self.root.tables.inherent_impls.size()).flat_map(move |i| {
-            let ty_index = DefIndex::from_usize(i);
-            let ty_def_id = self.local_def_id(ty_index);
-            self.root
-                .tables
-                .inherent_impls
-                .get(self, ty_index)
-                .decode(self)
-                .map(move |impl_index| (ty_def_id, self.local_def_id(impl_index)))
-        })
-    }
-
     /// Decodes all traits in the crate (for rustdoc and rustc diagnostics).
     fn get_traits(self) -> impl Iterator<Item = DefId> + 'a {
         self.root.traits.decode(self).map(move |index| self.local_def_id(index))
@@ -1195,13 +1181,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         })
     }
 
-    fn get_all_incoherent_impls(self) -> impl Iterator<Item = DefId> + 'a {
-        self.cdata
-            .incoherent_impls
-            .values()
-            .flat_map(move |impls| impls.decode(self).map(move |idx| self.local_def_id(idx)))
-    }
-
     fn get_incoherent_impls(self, tcx: TyCtxt<'tcx>, simp: SimplifiedType) -> &'tcx [DefId] {
         if let Some(impls) = self.cdata.incoherent_impls.get(&simp) {
             tcx.arena.alloc_from_iter(impls.decode(self).map(|idx| self.local_def_id(idx)))
@@ -1598,6 +1577,24 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     fn get_is_intrinsic(self, index: DefIndex) -> bool {
         self.root.tables.is_intrinsic.get(self, index)
     }
+
+    fn get_doc_link_resolutions(self, index: DefIndex) -> DocLinkResMap {
+        self.root
+            .tables
+            .doc_link_resolutions
+            .get(self, index)
+            .expect("no resolutions for a doc link")
+            .decode(self)
+    }
+
+    fn get_doc_link_traits_in_scope(self, index: DefIndex) -> impl Iterator<Item = DefId> + 'a {
+        self.root
+            .tables
+            .doc_link_traits_in_scope
+            .get(self, index)
+            .expect("no traits in scope for a doc link")
+            .decode(self)
+    }
 }
 
 impl CrateMetadata {

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -345,6 +345,10 @@ provide! { tcx, def_id, other, cdata,
     expn_that_defined => { cdata.get_expn_that_defined(def_id.index, tcx.sess) }
     generator_diagnostic_data => { cdata.get_generator_diagnostic_data(tcx, def_id.index) }
     is_doc_hidden => { cdata.get_attr_flags(def_id.index).contains(AttrFlags::IS_DOC_HIDDEN) }
+    doc_link_resolutions => { tcx.arena.alloc(cdata.get_doc_link_resolutions(def_id.index)) }
+    doc_link_traits_in_scope => {
+        tcx.arena.alloc_from_iter(cdata.get_doc_link_traits_in_scope(def_id.index))
+    }
 }
 
 pub(in crate::rmeta) fn provide(providers: &mut Providers) {
@@ -613,36 +617,6 @@ impl CStore {
         self.get_crate_data(cnum).get_trait_impls()
     }
 
-    /// Decodes all inherent impls in the crate (for rustdoc).
-    pub fn inherent_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = (DefId, DefId)> + '_ {
-        self.get_crate_data(cnum).get_inherent_impls()
-    }
-
-    /// Decodes all incoherent inherent impls in the crate (for rustdoc).
-    pub fn incoherent_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = DefId> + '_ {
-        self.get_crate_data(cnum).get_all_incoherent_impls()
-    }
-
-    pub fn associated_item_def_ids_untracked<'a>(
-        &'a self,
-        def_id: DefId,
-        sess: &'a Session,
-    ) -> impl Iterator<Item = DefId> + 'a {
-        self.get_crate_data(def_id.krate).get_associated_item_def_ids(def_id.index, sess)
-    }
-
-    pub fn may_have_doc_links_untracked(&self, def_id: DefId) -> bool {
-        self.get_crate_data(def_id.krate)
-            .get_attr_flags(def_id.index)
-            .contains(AttrFlags::MAY_HAVE_DOC_LINKS)
-    }
-
     pub fn is_doc_hidden_untracked(&self, def_id: DefId) -> bool {
         self.get_crate_data(def_id.krate)
             .get_attr_flags(def_id.index)

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -3,7 +3,6 @@ use crate::rmeta::def_path_hash_map::DefPathHashMapRef;
 use crate::rmeta::table::TableBuilder;
 use crate::rmeta::*;
 
-use rustc_ast::util::comments;
 use rustc_ast::Attribute;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
@@ -772,7 +771,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
 struct AnalyzeAttrState {
     is_exported: bool,
-    may_have_doc_links: bool,
     is_doc_hidden: bool,
 }
 
@@ -790,15 +788,12 @@ fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState) -> bool {
     let mut should_encode = false;
     if rustc_feature::is_builtin_only_local(attr.name_or_empty()) {
         // Attributes marked local-only don't need to be encoded for downstream crates.
-    } else if let Some(s) = attr.doc_str() {
+    } else if attr.doc_str().is_some() {
         // We keep all doc comments reachable to rustdoc because they might be "imported" into
         // downstream crates if they use `#[doc(inline)]` to copy an item's documentation into
         // their own.
         if state.is_exported {
             should_encode = true;
-            if comments::may_have_doc_links(s.as_str()) {
-                state.may_have_doc_links = true;
-            }
         }
     } else if attr.has_name(sym::doc) {
         // If this is a `doc` attribute that doesn't have anything except maybe `inline` (as in
@@ -1139,7 +1134,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let tcx = self.tcx;
         let mut state = AnalyzeAttrState {
             is_exported: tcx.effective_visibilities(()).is_exported(def_id),
-            may_have_doc_links: false,
             is_doc_hidden: false,
         };
         let attr_iter = tcx
@@ -1151,9 +1145,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         record_array!(self.tables.attributes[def_id.to_def_id()] <- attr_iter);
 
         let mut attr_flags = AttrFlags::empty();
-        if state.may_have_doc_links {
-            attr_flags |= AttrFlags::MAY_HAVE_DOC_LINKS;
-        }
         if state.is_doc_hidden {
             attr_flags |= AttrFlags::IS_DOC_HIDDEN;
         }
@@ -1231,6 +1222,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_id.index
             }));
         }
+
+        for (def_id, res_map) in &tcx.resolutions(()).doc_link_resolutions {
+            record!(self.tables.doc_link_resolutions[def_id.to_def_id()] <- res_map);
+        }
+
+        for (def_id, traits) in &tcx.resolutions(()).doc_link_traits_in_scope {
+            record_array!(self.tables.doc_link_traits_in_scope[def_id.to_def_id()] <- traits);
+        }
     }
 
     #[instrument(level = "trace", skip(self))]
@@ -1715,6 +1714,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.lookup_stability[LOCAL_CRATE.as_def_id()] <- stability);
             }
             self.encode_deprecation(LOCAL_CRATE.as_def_id());
+            if let Some(res_map) = tcx.resolutions(()).doc_link_resolutions.get(&CRATE_DEF_ID) {
+                record!(self.tables.doc_link_resolutions[LOCAL_CRATE.as_def_id()] <- res_map);
+            }
+            if let Some(traits) = tcx.resolutions(()).doc_link_traits_in_scope.get(&CRATE_DEF_ID) {
+                record_array!(self.tables.doc_link_traits_in_scope[LOCAL_CRATE.as_def_id()] <- traits);
+            }
 
             // Normally, this information is encoded when we walk the items
             // defined in this crate. However, we skip doing that for proc-macro crates,
@@ -2225,6 +2230,18 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
 
 pub fn provide(providers: &mut Providers) {
     *providers = Providers {
+        doc_link_resolutions: |tcx, def_id| {
+            tcx.resolutions(())
+                .doc_link_resolutions
+                .get(&def_id.expect_local())
+                .expect("no resolutions for a doc link")
+        },
+        doc_link_traits_in_scope: |tcx, def_id| {
+            tcx.resolutions(())
+                .doc_link_traits_in_scope
+                .get(&def_id.expect_local())
+                .expect("no traits in scope for a doc link")
+        },
         traits_in_crate: |tcx, cnum| {
             assert_eq!(cnum, LOCAL_CRATE);
 

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -9,7 +9,7 @@ use rustc_attr as attr;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, DefKind};
+use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap};
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, DefPathHash, StableCrateId};
 use rustc_hir::definitions::DefKey;
 use rustc_hir::lang_items::LangItem;
@@ -413,6 +413,8 @@ define_tables! {
     module_reexports: Table<DefIndex, LazyArray<ModChild>>,
     deduced_param_attrs: Table<DefIndex, LazyArray<DeducedParamAttrs>>,
     trait_impl_trait_tys: Table<DefIndex, LazyValue<FxHashMap<DefId, Ty<'static>>>>,
+    doc_link_resolutions: Table<DefIndex, LazyValue<DocLinkResMap>>,
+    doc_link_traits_in_scope: Table<DefIndex, LazyArray<DefId>>,
 }
 
 #[derive(TyEncodable, TyDecodable)]
@@ -426,8 +428,7 @@ struct VariantData {
 bitflags::bitflags! {
     #[derive(Default)]
     pub struct AttrFlags: u8 {
-        const MAY_HAVE_DOC_LINKS = 1 << 0;
-        const IS_DOC_HIDDEN      = 1 << 1;
+        const IS_DOC_HIDDEN = 1 << 0;
     }
 }
 

compiler/rustc_middle/src/arena.rs

@@ -113,6 +113,7 @@ macro_rules! arena_types {
             [decode] trait_impl_trait_tys: rustc_data_structures::fx::FxHashMap<rustc_hir::def_id::DefId, rustc_middle::ty::Ty<'tcx>>,
             [] bit_set_u32: rustc_index::bit_set::BitSet<u32>,
             [] external_constraints: rustc_middle::traits::solve::ExternalConstraintsData<'tcx>,
+            [decode] doc_link_resolutions: rustc_hir::def::DocLinkResMap,
         ]);
     )
 }

compiler/rustc_middle/src/query/mod.rs

@@ -2156,4 +2156,16 @@ rustc_queries! {
         desc { |tcx| "deducing parameter attributes for {}", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
+
+    query doc_link_resolutions(def_id: DefId) -> &'tcx DocLinkResMap {
+        eval_always
+        desc { "resolutions for documentation links for a module" }
+        separate_provide_extern
+    }
+
+    query doc_link_traits_in_scope(def_id: DefId) -> &'tcx [DefId] {
+        eval_always
+        desc { "traits in scope for documentation links for a module" }
+        separate_provide_extern
+    }
 }

compiler/rustc_middle/src/ty/mod.rs

@@ -36,7 +36,7 @@ use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, CtorOf, DefKind, LifetimeRes, Res};
+use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
 use rustc_hir::Node;
 use rustc_index::vec::IndexVec;
@@ -181,6 +181,8 @@ pub struct ResolverGlobalCtxt {
     /// exist under `std`. For example, wrote `str::from_utf8` instead of `std::str::from_utf8`.
     pub confused_type_with_std_module: FxHashMap<Span, Span>,
     pub registered_tools: RegisteredTools,
+    pub doc_link_resolutions: FxHashMap<LocalDefId, DocLinkResMap>,
+    pub doc_link_traits_in_scope: FxHashMap<LocalDefId, Vec<DefId>>,
 }
 
 /// Resolutions that should only be used for lowering.

compiler/rustc_middle/src/ty/parameterized.rs

@@ -81,6 +81,8 @@ trivially_parameterized_over_tcx! {
     rustc_hir::IsAsync,
     rustc_hir::LangItem,
     rustc_hir::def::DefKind,
+    rustc_hir::def::DocLinkResMap,
+    rustc_hir::def_id::DefId,
     rustc_hir::def_id::DefIndex,
     rustc_hir::definitions::DefKey,
     rustc_index::bit_set::BitSet<u32>,

compiler/rustc_middle/src/ty/query.rs

@@ -45,7 +45,7 @@ use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
+use rustc_hir::def::{DefKind, DocLinkResMap};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
 use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};

compiler/rustc_resolve/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 bitflags = "1.2.1"
+pulldown-cmark = { version = "0.9.2", default-features = false }
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }