Auto merge of #18234 - Veykril:veykril/push-vzynqtlxmrnl, r=Veykril

internal: Filter out opaque tokens in some IDE feature macro descensions

crates/hir-expand/src/db.rs

@@ -16,7 +16,10 @@ use crate::{
     cfg_process,
     declarative::DeclarativeMacroExpander,
     fixup::{self, SyntaxFixupUndoInfo},
-    hygiene::{span_with_call_site_ctxt, span_with_def_site_ctxt, span_with_mixed_site_ctxt},
+    hygiene::{
+        span_with_call_site_ctxt, span_with_def_site_ctxt, span_with_mixed_site_ctxt,
+        SyntaxContextExt as _,
+    },
     proc_macro::ProcMacros,
     span_map::{RealSpanMap, SpanMap, SpanMapRef},
     tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
@@ -300,14 +303,16 @@ pub fn expand_speculative(
         token_tree_to_syntax_node(&speculative_expansion.value, expand_to, loc.def.edition);
 
     let syntax_node = node.syntax_node();
-    let token = rev_tmap
+    let (token, _) = rev_tmap
         .ranges_with_span(span_map.span_for_range(token_to_map.text_range()))
-        .filter_map(|range| syntax_node.covering_element(range).into_token())
-        .min_by_key(|t| {
-            // prefer tokens of the same kind and text
+        .filter_map(|(range, ctx)| syntax_node.covering_element(range).into_token().zip(Some(ctx)))
+        .min_by_key(|(t, ctx)| {
+            // prefer tokens of the same kind and text, as well as non opaque marked ones
             // Note the inversion of the score here, as we want to prefer the first token in case
             // of all tokens having the same score
-            (t.kind() != token_to_map.kind()) as u8 + 2 * ((t.text() != token_to_map.text()) as u8)
+            ctx.is_opaque(db) as u8
+                + 2 * (t.kind() != token_to_map.kind()) as u8
+                + 4 * ((t.text() != token_to_map.text()) as u8)
         })?;
     Some((node.syntax_node(), token))
 }

crates/hir-expand/src/hygiene.rs

@@ -151,6 +151,7 @@ pub trait SyntaxContextExt {
     fn remove_mark(&mut self, db: &dyn ExpandDatabase) -> (Option<MacroCallId>, Transparency);
     fn outer_mark(self, db: &dyn ExpandDatabase) -> (Option<MacroCallId>, Transparency);
     fn marks(self, db: &dyn ExpandDatabase) -> Vec<(MacroCallId, Transparency)>;
+    fn is_opaque(self, db: &dyn ExpandDatabase) -> bool;
 }
 
 impl SyntaxContextExt for SyntaxContextId {
@@ -177,6 +178,9 @@ impl SyntaxContextExt for SyntaxContextId {
         marks.reverse();
         marks
     }
+    fn is_opaque(self, db: &dyn ExpandDatabase) -> bool {
+        !self.is_root() && db.lookup_intern_syntax_context(self).outer_transparency.is_opaque()
+    }
 }
 
 // FIXME: Make this a SyntaxContextExt method once we have RPIT

crates/hir-expand/src/lib.rs

@@ -25,6 +25,7 @@ mod prettify_macro_expansion_;
 
 use attrs::collect_attrs;
 use rustc_hash::FxHashMap;
+use stdx::TupleExt;
 use triomphe::Arc;
 
 use std::hash::Hash;
@@ -772,14 +773,15 @@ impl ExpansionInfo {
     /// Maps the passed in file range down into a macro expansion if it is the input to a macro call.
     ///
     /// Note this does a linear search through the entire backing vector of the spanmap.
+    // FIXME: Consider adding a reverse map to ExpansionInfo to get rid of the linear search which
+    // potentially results in quadratic look ups (notably this might improve semantic highlighting perf)
     pub fn map_range_down_exact(
         &self,
         span: Span,
-    ) -> Option<InMacroFile<impl Iterator<Item = SyntaxToken> + '_>> {
-        let tokens = self
-            .exp_map
-            .ranges_with_span_exact(span)
-            .flat_map(move |range| self.expanded.value.covering_element(range).into_token());
+    ) -> Option<InMacroFile<impl Iterator<Item = (SyntaxToken, SyntaxContextId)> + '_>> {
+        let tokens = self.exp_map.ranges_with_span_exact(span).flat_map(move |(range, ctx)| {
+            self.expanded.value.covering_element(range).into_token().zip(Some(ctx))
+        });
 
         Some(InMacroFile::new(self.expanded.file_id, tokens))
     }
@@ -791,11 +793,10 @@ impl ExpansionInfo {
     pub fn map_range_down(
         &self,
         span: Span,
-    ) -> Option<InMacroFile<impl Iterator<Item = SyntaxToken> + '_>> {
-        let tokens = self
-            .exp_map
-            .ranges_with_span(span)
-            .flat_map(move |range| self.expanded.value.covering_element(range).into_token());
+    ) -> Option<InMacroFile<impl Iterator<Item = (SyntaxToken, SyntaxContextId)> + '_>> {
+        let tokens = self.exp_map.ranges_with_span(span).flat_map(move |(range, ctx)| {
+            self.expanded.value.covering_element(range).into_token().zip(Some(ctx))
+        });
 
         Some(InMacroFile::new(self.expanded.file_id, tokens))
     }
@@ -845,7 +846,8 @@ impl ExpansionInfo {
                     self.arg.file_id,
                     arg_map
                         .ranges_with_span_exact(span)
-                        .filter(|range| range.intersect(arg_range).is_some())
+                        .filter(|(range, _)| range.intersect(arg_range).is_some())
+                        .map(TupleExt::head)
                         .collect(),
                 )
             }

crates/hir/src/semantics.rs

@@ -24,6 +24,7 @@ use hir_expand::{
     builtin::{BuiltinFnLikeExpander, EagerExpander},
     db::ExpandDatabase,
     files::InRealFile,
+    hygiene::SyntaxContextExt as _,
     inert_attr_macro::find_builtin_attr_idx,
     name::AsName,
     FileRange, InMacroFile, MacroCallId, MacroFileId, MacroFileIdExt,
@@ -32,7 +33,7 @@ use intern::Symbol;
 use itertools::Itertools;
 use rustc_hash::{FxHashMap, FxHashSet};
 use smallvec::{smallvec, SmallVec};
-use span::{EditionedFileId, FileId, HirFileIdRepr};
+use span::{EditionedFileId, FileId, HirFileIdRepr, SyntaxContextId};
 use stdx::TupleExt;
 use syntax::{
     algo::skip_trivia_token,
@@ -608,7 +609,7 @@ impl<'db> SemanticsImpl<'db> {
         let quote = string.open_quote_text_range()?;
 
         let token = self.wrap_token_infile(string.syntax().clone()).into_real_file().ok()?;
-        self.descend_into_macros_breakable(token, |token| {
+        self.descend_into_macros_breakable(token, |token, _| {
             (|| {
                 let token = token.value;
                 let string = ast::String::cast(token)?;
@@ -655,7 +656,7 @@ impl<'db> SemanticsImpl<'db> {
         let original_string = ast::String::cast(original_token.clone())?;
         let original_token = self.wrap_token_infile(original_token).into_real_file().ok()?;
         let quote = original_string.open_quote_text_range()?;
-        self.descend_into_macros_breakable(original_token, |token| {
+        self.descend_into_macros_breakable(original_token, |token, _| {
             (|| {
                 let token = token.value;
                 self.resolve_offset_in_format_args(
@@ -718,7 +719,7 @@ impl<'db> SemanticsImpl<'db> {
             // node is just the token, so descend the token
             self.descend_into_macros_impl(
                 InRealFile::new(file_id, first),
-                &mut |InFile { value, .. }| {
+                &mut |InFile { value, .. }, _ctx| {
                     if let Some(node) = value
                         .parent_ancestors()
                         .take_while(|it| it.text_range() == value.text_range())
@@ -732,15 +733,15 @@ impl<'db> SemanticsImpl<'db> {
         } else {
             // Descend first and last token, then zip them to look for the node they belong to
             let mut scratch: SmallVec<[_; 1]> = smallvec![];
-            self.descend_into_macros_impl(InRealFile::new(file_id, first), &mut |token| {
+            self.descend_into_macros_impl(InRealFile::new(file_id, first), &mut |token, _ctx| {
                 scratch.push(token);
                 CONTINUE_NO_BREAKS
             });
 
             let mut scratch = scratch.into_iter();
             self.descend_into_macros_impl(
                 InRealFile::new(file_id, last),
-                &mut |InFile { value: last, file_id: last_fid }| {
+                &mut |InFile { value: last, file_id: last_fid }, _ctx| {
                     if let Some(InFile { value: first, file_id: first_fid }) = scratch.next() {
                         if first_fid == last_fid {
                             if let Some(p) = first.parent() {
@@ -763,7 +764,9 @@ impl<'db> SemanticsImpl<'db> {
         res
     }
 
-    fn is_inside_macro_call(token: &SyntaxToken) -> bool {
+    // FIXME: This isn't quite right wrt to inner attributes
+    /// Does a syntactic traversal to check whether this token might be inside a macro call
+    pub fn might_be_inside_macro_call(&self, token: &SyntaxToken) -> bool {
         token.parent_ancestors().any(|ancestor| {
             if ast::MacroCall::can_cast(ancestor.kind()) {
                 return true;
@@ -781,25 +784,14 @@ impl<'db> SemanticsImpl<'db> {
         })
     }
 
-    pub fn descend_into_macros_exact_if_in_macro(
-        &self,
-        token: SyntaxToken,
-    ) -> SmallVec<[SyntaxToken; 1]> {
-        if Self::is_inside_macro_call(&token) {
-            self.descend_into_macros_exact(token)
-        } else {
-            smallvec![token]
-        }
-    }
-
     pub fn descend_into_macros_cb(
         &self,
         token: SyntaxToken,
-        mut cb: impl FnMut(InFile<SyntaxToken>),
+        mut cb: impl FnMut(InFile<SyntaxToken>, SyntaxContextId),
     ) {
         if let Ok(token) = self.wrap_token_infile(token).into_real_file() {
-            self.descend_into_macros_impl(token, &mut |t| {
-                cb(t);
+            self.descend_into_macros_impl(token, &mut |t, ctx| {
+                cb(t, ctx);
                 CONTINUE_NO_BREAKS
             });
         }
@@ -808,7 +800,7 @@ impl<'db> SemanticsImpl<'db> {
     pub fn descend_into_macros(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> {
         let mut res = smallvec![];
         if let Ok(token) = self.wrap_token_infile(token.clone()).into_real_file() {
-            self.descend_into_macros_impl(token, &mut |t| {
+            self.descend_into_macros_impl(token, &mut |t, _ctx| {
                 res.push(t.value);
                 CONTINUE_NO_BREAKS
             });
@@ -819,10 +811,27 @@ impl<'db> SemanticsImpl<'db> {
         res
     }
 
+    pub fn descend_into_macros_no_opaque(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> {
+        let mut res = smallvec![];
+        if let Ok(token) = self.wrap_token_infile(token.clone()).into_real_file() {
+            self.descend_into_macros_impl(token, &mut |t, ctx| {
+                if !ctx.is_opaque(self.db.upcast()) {
+                    // Don't descend into opaque contexts
+                    res.push(t.value);
+                }
+                CONTINUE_NO_BREAKS
+            });
+        }
+        if res.is_empty() {
+            res.push(token);
+        }
+        res
+    }
+
     pub fn descend_into_macros_breakable<T>(
         &self,
         token: InRealFile<SyntaxToken>,
-        mut cb: impl FnMut(InFile<SyntaxToken>) -> ControlFlow<T>,
+        mut cb: impl FnMut(InFile<SyntaxToken>, SyntaxContextId) -> ControlFlow<T>,
     ) -> Option<T> {
         self.descend_into_macros_impl(token.clone(), &mut cb)
     }
@@ -834,10 +843,12 @@ impl<'db> SemanticsImpl<'db> {
         let text = token.text();
         let kind = token.kind();
 
-        self.descend_into_macros_cb(token.clone(), |InFile { value, file_id: _ }| {
+        self.descend_into_macros_cb(token.clone(), |InFile { value, file_id: _ }, ctx| {
             let mapped_kind = value.kind();
             let any_ident_match = || kind.is_any_identifier() && value.kind().is_any_identifier();
-            let matches = (kind == mapped_kind || any_ident_match()) && text == value.text();
+            let matches = (kind == mapped_kind || any_ident_match())
+                && text == value.text()
+                && !ctx.is_opaque(self.db.upcast());
             if matches {
                 r.push(value);
             }
@@ -854,17 +865,21 @@ impl<'db> SemanticsImpl<'db> {
         let text = token.text();
         let kind = token.kind();
         if let Ok(token) = self.wrap_token_infile(token.clone()).into_real_file() {
-            self.descend_into_macros_breakable(token.clone(), |InFile { value, file_id: _ }| {
-                let mapped_kind = value.kind();
-                let any_ident_match =
-                    || kind.is_any_identifier() && value.kind().is_any_identifier();
-                let matches = (kind == mapped_kind || any_ident_match()) && text == value.text();
-                if matches {
-                    ControlFlow::Break(value)
-                } else {
-                    ControlFlow::Continue(())
-                }
-            })
+            self.descend_into_macros_breakable(
+                token.clone(),
+                |InFile { value, file_id: _ }, _ctx| {
+                    let mapped_kind = value.kind();
+                    let any_ident_match =
+                        || kind.is_any_identifier() && value.kind().is_any_identifier();
+                    let matches =
+                        (kind == mapped_kind || any_ident_match()) && text == value.text();
+                    if matches {
+                        ControlFlow::Break(value)
+                    } else {
+                        ControlFlow::Continue(())
+                    }
+                },
+            )
         } else {
             None
         }
@@ -874,7 +889,7 @@ impl<'db> SemanticsImpl<'db> {
     fn descend_into_macros_impl<T>(
         &self,
         InRealFile { value: token, file_id }: InRealFile<SyntaxToken>,
-        f: &mut dyn FnMut(InFile<SyntaxToken>) -> ControlFlow<T>,
+        f: &mut dyn FnMut(InFile<SyntaxToken>, SyntaxContextId) -> ControlFlow<T>,
     ) -> Option<T> {
         let _p = tracing::info_span!("descend_into_macros_impl").entered();
         let (sa, span, file_id) = token
@@ -898,7 +913,8 @@ impl<'db> SemanticsImpl<'db> {
         // These are tracked to know which macro calls we still have to look into
         // the tokens themselves aren't that interesting as the span that is being used to map
         // things down never changes.
-        let mut stack: Vec<(_, SmallVec<[_; 2]>)> = vec![(file_id, smallvec![token])];
+        let mut stack: Vec<(_, SmallVec<[_; 2]>)> =
+            vec![(file_id, smallvec![(token, SyntaxContextId::ROOT)])];
 
         // Process the expansion of a call, pushing all tokens with our span in the expansion back onto our stack
         let process_expansion_for_token = |stack: &mut Vec<_>, macro_file| {
@@ -921,11 +937,11 @@ impl<'db> SemanticsImpl<'db> {
         // Filters out all tokens that contain the given range (usually the macro call), any such
         // token is redundant as the corresponding macro call has already been processed
         let filter_duplicates = |tokens: &mut SmallVec<_>, range: TextRange| {
-            tokens.retain(|t: &mut SyntaxToken| !range.contains_range(t.text_range()))
+            tokens.retain(|(t, _): &mut (SyntaxToken, _)| !range.contains_range(t.text_range()))
         };
 
         while let Some((expansion, ref mut tokens)) = stack.pop() {
-            while let Some(token) = tokens.pop() {
+            while let Some((token, ctx)) = tokens.pop() {
                 let was_not_remapped = (|| {
                     // First expand into attribute invocations
                     let containing_attribute_macro_call = self.with_ctx(|ctx| {
@@ -1036,7 +1052,7 @@ impl<'db> SemanticsImpl<'db> {
                                             let text_range = attr.syntax().text_range();
                                             // remove any other token in this macro input, all their mappings are the
                                             // same as this
-                                            tokens.retain(|t| {
+                                            tokens.retain(|(t, _)| {
                                                 !text_range.contains_range(t.text_range())
                                             });
                                             return process_expansion_for_token(
@@ -1093,7 +1109,7 @@ impl<'db> SemanticsImpl<'db> {
                 .is_none();
 
                 if was_not_remapped {
-                    if let ControlFlow::Break(b) = f(InFile::new(expansion, token)) {
+                    if let ControlFlow::Break(b) = f(InFile::new(expansion, token), ctx) {
                         return Some(b);
                     }
                 }

crates/ide-db/src/search.rs

@@ -529,9 +529,13 @@ impl<'a> FindUsages<'a> {
             })
             .into_iter()
             .flat_map(move |token| {
-                sema.descend_into_macros_exact_if_in_macro(token)
-                    .into_iter()
-                    .filter_map(|it| it.parent())
+                if sema.might_be_inside_macro_call(&token) {
+                    sema.descend_into_macros_exact(token)
+                } else {
+                    <_>::from([token])
+                }
+                .into_iter()
+                .filter_map(|it| it.parent())
             })
     }
 

crates/ide/src/goto_declaration.rs

@@ -29,7 +29,7 @@ pub(crate) fn goto_declaration(
         .find(|it| matches!(it.kind(), IDENT | T![self] | T![super] | T![crate] | T![Self]))?;
     let range = original_token.text_range();
     let info: Vec<NavigationTarget> = sema
-        .descend_into_macros(original_token)
+        .descend_into_macros_no_opaque(original_token)
         .iter()
         .filter_map(|token| {
             let parent = token.parent()?;

crates/ide/src/goto_definition.rs

@@ -83,7 +83,7 @@ pub(crate) fn goto_definition(
     }
 
     let navs = sema
-        .descend_into_macros(original_token.clone())
+        .descend_into_macros_no_opaque(original_token.clone())
         .into_iter()
         .filter_map(|token| {
             let parent = token.parent()?;

crates/ide/src/goto_type_definition.rs

@@ -69,7 +69,7 @@ pub(crate) fn goto_type_definition(
     }
 
     let range = token.text_range();
-    sema.descend_into_macros(token)
+    sema.descend_into_macros_no_opaque(token)
         .into_iter()
         .filter_map(|token| {
             let ty = sema

crates/ide/src/syntax_highlighting.rs

@@ -409,7 +409,8 @@ fn traverse(
                     let mut r = 0;
                     sema.descend_into_macros_breakable(
                         InRealFile::new(file_id, token.clone()),
-                        |tok| {
+                        |tok, _ctx| {
+                            // FIXME: Consider checking ctx transparency for being opaque?
                             let tok = tok.value;
                             let tok_kind = tok.kind();