From 177c70128cccdef410a38faf098745e1406ee281 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 21 Aug 2021 18:06:03 +0200 Subject: [PATCH 1/3] Map attribute input tokens correctly --- crates/hir/src/semantics.rs | 32 +++---- crates/hir_def/src/attr.rs | 20 +++-- crates/hir_def/src/lib.rs | 6 +- crates/hir_def/src/nameres/collector.rs | 2 +- crates/hir_expand/src/db.rs | 10 ++- crates/hir_expand/src/lib.rs | 114 +++++++++++++++++++----- crates/mbe/src/lib.rs | 14 +-- crates/mbe/src/token_map.rs | 21 +++++ 8 files changed, 158 insertions(+), 61 deletions(-) diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 68b52ddbc49..efe9fdec577 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -441,7 +441,7 @@ impl<'db> SemanticsImpl<'db> { .entry(file_id) .or_insert_with(|| file_id.expansion_info(self.db.upcast())) .as_ref()? - .map_token_down(token.as_ref())?; + .map_token_down(self.db.upcast(), None, token.as_ref())?; if let Some(parent) = token.value.parent() { self.cache(find_root(&parent), token.file_id); @@ -450,24 +450,21 @@ impl<'db> SemanticsImpl<'db> { return Some(token); }, ast::Item(item) => { - match self.with_ctx(|ctx| ctx.item_to_macro_call(token.with_value(item))) { - Some(call_id) => { - let file_id = call_id.as_file(); - let token = self - .expansion_info_cache - .borrow_mut() - .entry(file_id) - .or_insert_with(|| file_id.expansion_info(self.db.upcast())) - .as_ref()? - .map_token_down(token.as_ref())?; + if let Some(call_id) = self.with_ctx(|ctx| ctx.item_to_macro_call(token.with_value(item.clone()))) { + let file_id = call_id.as_file(); + let token = self + .expansion_info_cache + .borrow_mut() + .entry(file_id) + .or_insert_with(|| file_id.expansion_info(self.db.upcast())) + .as_ref()? + .map_token_down(self.db.upcast(), Some(item), token.as_ref())?; - if let Some(parent) = token.value.parent() { - self.cache(find_root(&parent), token.file_id); - } - - return Some(token); + if let Some(parent) = token.value.parent() { + self.cache(find_root(&parent), token.file_id); } - None => {} + + return Some(token); } }, _ => {} @@ -479,7 +476,6 @@ impl<'db> SemanticsImpl<'db> { }) .last() .unwrap(); - token.value } diff --git a/crates/hir_def/src/attr.rs b/crates/hir_def/src/attr.rs index b2f8e4d20d9..64b09d61638 100644 --- a/crates/hir_def/src/attr.rs +++ b/crates/hir_def/src/attr.rs @@ -2,7 +2,9 @@ use std::{ convert::{TryFrom, TryInto}, - fmt, ops, + fmt, + hash::Hash, + ops, sync::Arc, }; @@ -12,7 +14,7 @@ use either::Either; use hir_expand::{hygiene::Hygiene, name::AsName, AstId, InFile}; use itertools::Itertools; use la_arena::ArenaMap; -use mbe::{syntax_node_to_token_tree, DelimiterKind}; +use mbe::{syntax_node_to_token_tree, DelimiterKind, MappedSubTree}; use smallvec::{smallvec, SmallVec}; use syntax::{ ast::{self, AstNode, AttrsOwner}, @@ -165,11 +167,11 @@ impl RawAttrs { // Input subtree is: `(cfg, $(attr),+)` // Split it up into a `cfg` subtree and the `attr` subtrees. // FIXME: There should be a common API for this. - let mut parts = subtree.token_trees.split( + let mut parts = subtree.tree.token_trees.split( |tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ','), ); let cfg = parts.next().unwrap(); - let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() }; + let cfg = Subtree { delimiter: subtree.tree.delimiter, token_trees: cfg.to_vec() }; let cfg = CfgExpr::parse(&cfg); let index = attr.id; let attrs = parts.filter(|a| !a.is_empty()).filter_map(|attr| { @@ -652,14 +654,14 @@ pub enum AttrInput { /// `#[attr = "string"]` Literal(SmolStr), /// `#[attr(subtree)]` - TokenTree(Subtree), + TokenTree(mbe::MappedSubTree), } impl fmt::Display for AttrInput { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { AttrInput::Literal(lit) => write!(f, " = \"{}\"", lit.escape_debug()), - AttrInput::TokenTree(subtree) => subtree.fmt(f), + AttrInput::TokenTree(subtree) => subtree.tree.fmt(f), } } } @@ -679,7 +681,8 @@ impl Attr { }; Some(Interned::new(AttrInput::Literal(value))) } else if let Some(tt) = ast.token_tree() { - Some(Interned::new(AttrInput::TokenTree(syntax_node_to_token_tree(tt.syntax()).0))) + let (tree, map) = syntax_node_to_token_tree(tt.syntax()); + Some(Interned::new(AttrInput::TokenTree(MappedSubTree { tree, map }))) } else { None }; @@ -712,6 +715,7 @@ impl Attr { Some(AttrInput::TokenTree(args)) => { let mut counter = 0; let paths = args + .tree .token_trees .iter() .group_by(move |tt| { @@ -756,7 +760,7 @@ pub struct AttrQuery<'a> { impl<'a> AttrQuery<'a> { pub fn tt_values(self) -> impl Iterator { self.attrs().filter_map(|attr| match attr.input.as_deref()? { - AttrInput::TokenTree(it) => Some(it), + AttrInput::TokenTree(it) => Some(&it.tree), _ => None, }) } diff --git a/crates/hir_def/src/lib.rs b/crates/hir_def/src/lib.rs index ec8185ec360..3c9cf0e38dc 100644 --- a/crates/hir_def/src/lib.rs +++ b/crates/hir_def/src/lib.rs @@ -786,13 +786,13 @@ fn attr_macro_as_call_id( .ok_or_else(|| UnresolvedMacro { path: item_attr.path.clone() })?; let mut arg = match ¯o_attr.input { Some(input) => match &**input { - attr::AttrInput::Literal(_) => tt::Subtree::default(), + attr::AttrInput::Literal(_) => Default::default(), attr::AttrInput::TokenTree(tt) => tt.clone(), }, - None => tt::Subtree::default(), + None => Default::default(), }; // The parentheses are always disposed here. - arg.delimiter = None; + arg.tree.delimiter = None; let res = def.as_lazy_macro( db.upcast(), diff --git a/crates/hir_def/src/nameres/collector.rs b/crates/hir_def/src/nameres/collector.rs index f8b3c3949f6..246bba9a11e 100644 --- a/crates/hir_def/src/nameres/collector.rs +++ b/crates/hir_def/src/nameres/collector.rs @@ -289,7 +289,7 @@ impl DefCollector<'_> { || *attr_name == hir_expand::name![register_tool] { match attr.input.as_deref() { - Some(AttrInput::TokenTree(subtree)) => match &*subtree.token_trees { + Some(AttrInput::TokenTree(subtree)) => match &*subtree.tree.token_trees { [tt::TokenTree::Leaf(tt::Leaf::Ident(name))] => name.as_name(), _ => continue, }, diff --git a/crates/hir_expand/src/db.rs b/crates/hir_expand/src/db.rs index f0a2e352fda..b7951cc282d 100644 --- a/crates/hir_expand/src/db.rs +++ b/crates/hir_expand/src/db.rs @@ -223,7 +223,7 @@ fn parse_macro_expansion( Ok(it) => it, Err(err) => { log::debug!( - "failed to parse expanstion to {:?} = {}", + "failed to parse expansion to {:?} = {}", fragment_kind, tt.as_debug_string() ); @@ -386,11 +386,15 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult Some(attr_args), + MacroCallKind::Attr { attr_args, .. } => { + let mut attr_args = attr_args.tree.clone(); + mbe::Shift::new(¯o_arg.0).shift_all(&mut attr_args); + Some(attr_args) + } _ => None, }; - expander.expand(db, loc.krate, ¯o_arg.0, attr_arg) + expander.expand(db, loc.krate, ¯o_arg.0, attr_arg.as_ref()) } fn is_self_replicating(from: &SyntaxNode, to: &SyntaxNode) -> bool { diff --git a/crates/hir_expand/src/lib.rs b/crates/hir_expand/src/lib.rs index 4ec3e10f497..86af19626e7 100644 --- a/crates/hir_expand/src/lib.rs +++ b/crates/hir_expand/src/lib.rs @@ -27,7 +27,7 @@ use std::{hash::Hash, iter, sync::Arc}; use base_db::{impl_intern_key, salsa, CrateId, FileId, FileRange}; use syntax::{ algo::skip_trivia_token, - ast::{self, AstNode}, + ast::{self, AstNode, AttrsOwner}, Direction, SyntaxNode, SyntaxToken, TextRange, TextSize, }; @@ -36,6 +36,7 @@ use crate::{ builtin_attr::BuiltinAttrExpander, builtin_derive::BuiltinDeriveExpander, builtin_macro::{BuiltinFnLikeExpander, EagerExpander}, + db::TokenExpander, proc_macro::ProcMacroExpander, }; @@ -132,6 +133,17 @@ impl HirFileId { }; Some(InFile::new(id.file_id, def_tt)) }); + let def_or_attr_input = def.or_else(|| match loc.kind { + MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => { + let tt = ast_id + .to_node(db) + .attrs() + .nth(invoc_attr_index as usize) + .and_then(|attr| attr.token_tree())?; + Some(InFile::new(ast_id.file_id, tt)) + } + _ => None, + }); let macro_def = db.macro_def(loc.def)?; let (parse, exp_map) = db.parse_macro_expansion(macro_file).value?; @@ -140,7 +152,8 @@ impl HirFileId { Some(ExpansionInfo { expanded: InFile::new(self, parse.syntax_node()), arg: InFile::new(loc.kind.file_id(), arg_tt), - def, + attr_input_or_mac_def: def_or_attr_input, + macro_arg_shift: mbe::Shift::new(¯o_arg.0), macro_arg, macro_def, exp_map, @@ -270,7 +283,7 @@ pub enum MacroCallKind { Attr { ast_id: AstId, attr_name: String, - attr_args: tt::Subtree, + attr_args: mbe::MappedSubTree, /// Syntactical index of the invoking `#[attribute]`. /// /// Outer attributes are counted first, then inner attributes. This does not support @@ -335,11 +348,12 @@ impl MacroCallId { pub struct ExpansionInfo { expanded: InFile, arg: InFile, - /// The `macro_rules!` arguments. - def: Option>, + /// The `macro_rules!` arguments or attribute input. + attr_input_or_mac_def: Option>, - macro_def: Arc, + macro_def: Arc, macro_arg: Arc<(tt::Subtree, mbe::TokenMap)>, + macro_arg_shift: mbe::Shift, exp_map: Arc, } @@ -350,11 +364,53 @@ impl ExpansionInfo { Some(self.arg.with_value(self.arg.value.parent()?)) } - pub fn map_token_down(&self, token: InFile<&SyntaxToken>) -> Option> { + pub fn map_token_down( + &self, + db: &dyn db::AstDatabase, + item: Option, + token: InFile<&SyntaxToken>, + ) -> Option> { assert_eq!(token.file_id, self.arg.file_id); - let range = token.value.text_range().checked_sub(self.arg.value.text_range().start())?; - let token_id = self.macro_arg.1.token_by_range(range)?; - let token_id = self.macro_def.map_id_down(token_id); + let token_id = if let Some(item) = item { + let call_id = match self.expanded.file_id.0 { + HirFileIdRepr::FileId(_) => return None, + HirFileIdRepr::MacroFile(macro_file) => macro_file.macro_call_id, + }; + let loc = db.lookup_intern_macro(call_id); + + let token_range = token.value.text_range(); + match &loc.kind { + MacroCallKind::Attr { attr_args, invoc_attr_index, .. } => { + let attr = item.attrs().nth(*invoc_attr_index as usize)?; + match attr.token_tree() { + Some(token_tree) + if token_tree.syntax().text_range().contains_range(token_range) => + { + let attr_input_start = + token_tree.left_delimiter_token()?.text_range().start(); + let range = token.value.text_range().checked_sub(attr_input_start)?; + let token_id = + self.macro_arg_shift.shift(attr_args.map.token_by_range(range)?); + Some(token_id) + } + _ => None, + } + } + _ => None, + } + } else { + None + }; + + let token_id = match token_id { + Some(token_id) => token_id, + None => { + let range = + token.value.text_range().checked_sub(self.arg.value.text_range().start())?; + let token_id = self.macro_arg.1.token_by_range(range)?; + self.macro_def.map_id_down(token_id) + } + }; let range = self.exp_map.range_by_token(token_id, token.value.kind())?; @@ -365,20 +421,36 @@ impl ExpansionInfo { pub fn map_token_up( &self, + db: &dyn db::AstDatabase, token: InFile<&SyntaxToken>, ) -> Option<(InFile, Origin)> { let token_id = self.exp_map.token_by_range(token.value.text_range())?; + let (mut token_id, origin) = self.macro_def.map_id_up(token_id); - let (token_id, origin) = self.macro_def.map_id_up(token_id); - let (token_map, tt) = match origin { - mbe::Origin::Call => (&self.macro_arg.1, self.arg.clone()), - mbe::Origin::Def => match (&*self.macro_def, self.def.as_ref()) { - ( - db::TokenExpander::MacroRules { def_site_token_map, .. } - | db::TokenExpander::MacroDef { def_site_token_map, .. }, - Some(tt), - ) => (def_site_token_map, tt.syntax().cloned()), - _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"), + let call_id = match self.expanded.file_id.0 { + HirFileIdRepr::FileId(_) => return None, + HirFileIdRepr::MacroFile(macro_file) => macro_file.macro_call_id, + }; + let loc = db.lookup_intern_macro(call_id); + + let (token_map, tt) = match &loc.kind { + MacroCallKind::Attr { attr_args, .. } => match self.macro_arg_shift.unshift(token_id) { + Some(unshifted) => { + token_id = unshifted; + (&attr_args.map, self.attr_input_or_mac_def.clone()?.syntax().cloned()) + } + None => (&self.macro_arg.1, self.arg.clone()), + }, + _ => match origin { + mbe::Origin::Call => (&self.macro_arg.1, self.arg.clone()), + mbe::Origin::Def => match (&*self.macro_def, self.attr_input_or_mac_def.as_ref()) { + ( + TokenExpander::MacroRules { def_site_token_map, .. } + | TokenExpander::MacroDef { def_site_token_map, .. }, + Some(tt), + ) => (def_site_token_map, tt.syntax().cloned()), + _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"), + }, }, }; @@ -532,7 +604,7 @@ fn ascend_call_token( expansion: &ExpansionInfo, token: InFile, ) -> Option> { - let (mapped, origin) = expansion.map_token_up(token.as_ref())?; + let (mapped, origin) = expansion.map_token_up(db, token.as_ref())?; if origin != Origin::Call { return None; } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index e0bbb825b93..a2a6d02b8a2 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -69,7 +69,7 @@ pub use crate::{ parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, }, - token_map::TokenMap, + token_map::{MappedSubTree, TokenMap}, }; /// This struct contains AST for a single `macro_rules` definition. What might @@ -97,11 +97,11 @@ struct Rule { rhs: MetaTemplate, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Shift(u32); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Shift(u32); impl Shift { - fn new(tt: &tt::Subtree) -> Shift { + pub fn new(tt: &tt::Subtree) -> Shift { // Note that TokenId is started from zero, // We have to add 1 to prevent duplication. let value = max_id(tt).map_or(0, |it| it + 1); @@ -134,7 +134,7 @@ impl Shift { } /// Shift given TokenTree token id - fn shift_all(self, tt: &mut tt::Subtree) { + pub fn shift_all(self, tt: &mut tt::Subtree) { for t in &mut tt.token_trees { match t { tt::TokenTree::Leaf(leaf) => match leaf { @@ -152,14 +152,14 @@ impl Shift { } } - fn shift(self, id: tt::TokenId) -> tt::TokenId { + pub fn shift(self, id: tt::TokenId) -> tt::TokenId { if id == tt::TokenId::unspecified() { return id; } tt::TokenId(id.0 + self.0) } - fn unshift(self, id: tt::TokenId) -> Option { + pub fn unshift(self, id: tt::TokenId) -> Option { id.0.checked_sub(self.0).map(tt::TokenId) } } diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 6df3de3b3ea..3e8c4b2d6da 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -1,5 +1,7 @@ //! Mapping between `TokenId`s and the token's position in macro definitions or inputs. +use std::hash::Hash; + use parser::{SyntaxKind, T}; use syntax::{TextRange, TextSize}; @@ -24,6 +26,25 @@ impl TokenTextRange { } } +#[derive(Debug, Clone, Default)] +pub struct MappedSubTree { + pub tree: tt::Subtree, + pub map: TokenMap, +} + +impl Eq for MappedSubTree {} +impl PartialEq for MappedSubTree { + fn eq(&self, other: &Self) -> bool { + self.tree == other.tree && self.map == other.map + } +} + +impl Hash for MappedSubTree { + fn hash(&self, state: &mut H) { + self.tree.hash(state); + } +} + /// Maps `tt::TokenId` to the relative range of the original token. #[derive(Debug, PartialEq, Eq, Clone, Default)] pub struct TokenMap { From 5fb8c0ddfd6fe948f93ed36c86b05bb5076c94d0 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 21 Aug 2021 18:19:18 +0200 Subject: [PATCH 2/3] Remove MappedSubtree --- crates/hir_def/src/attr.rs | 23 +++++++++++------------ crates/hir_def/src/lib.rs | 4 ++-- crates/hir_def/src/nameres/collector.rs | 2 +- crates/hir_expand/src/db.rs | 2 +- crates/hir_expand/src/lib.rs | 6 +++--- crates/mbe/src/lib.rs | 2 +- crates/mbe/src/token_map.rs | 23 ++--------------------- 7 files changed, 21 insertions(+), 41 deletions(-) diff --git a/crates/hir_def/src/attr.rs b/crates/hir_def/src/attr.rs index 64b09d61638..163a45ca142 100644 --- a/crates/hir_def/src/attr.rs +++ b/crates/hir_def/src/attr.rs @@ -14,7 +14,7 @@ use either::Either; use hir_expand::{hygiene::Hygiene, name::AsName, AstId, InFile}; use itertools::Itertools; use la_arena::ArenaMap; -use mbe::{syntax_node_to_token_tree, DelimiterKind, MappedSubTree}; +use mbe::{syntax_node_to_token_tree, DelimiterKind}; use smallvec::{smallvec, SmallVec}; use syntax::{ ast::{self, AstNode, AttrsOwner}, @@ -160,18 +160,18 @@ impl RawAttrs { } let subtree = match attr.input.as_deref() { - Some(AttrInput::TokenTree(it)) => it, + Some(AttrInput::TokenTree(it, _)) => it, _ => return smallvec![attr.clone()], }; // Input subtree is: `(cfg, $(attr),+)` // Split it up into a `cfg` subtree and the `attr` subtrees. // FIXME: There should be a common API for this. - let mut parts = subtree.tree.token_trees.split( + let mut parts = subtree.token_trees.split( |tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ','), ); let cfg = parts.next().unwrap(); - let cfg = Subtree { delimiter: subtree.tree.delimiter, token_trees: cfg.to_vec() }; + let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() }; let cfg = CfgExpr::parse(&cfg); let index = attr.id; let attrs = parts.filter(|a| !a.is_empty()).filter_map(|attr| { @@ -260,7 +260,7 @@ impl Attrs { pub fn docs(&self) -> Option { let docs = self.by_key("doc").attrs().flat_map(|attr| match attr.input.as_deref()? { AttrInput::Literal(s) => Some(s), - AttrInput::TokenTree(_) => None, + AttrInput::TokenTree(..) => None, }); let indent = docs .clone() @@ -465,7 +465,7 @@ impl AttrsWithOwner { // FIXME: code duplication in `docs` above let docs = self.by_key("doc").attrs().flat_map(|attr| match attr.input.as_deref()? { AttrInput::Literal(s) => Some((s, attr.id)), - AttrInput::TokenTree(_) => None, + AttrInput::TokenTree(..) => None, }); let indent = docs .clone() @@ -654,14 +654,14 @@ pub enum AttrInput { /// `#[attr = "string"]` Literal(SmolStr), /// `#[attr(subtree)]` - TokenTree(mbe::MappedSubTree), + TokenTree(tt::Subtree, mbe::TokenMap), } impl fmt::Display for AttrInput { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { AttrInput::Literal(lit) => write!(f, " = \"{}\"", lit.escape_debug()), - AttrInput::TokenTree(subtree) => subtree.tree.fmt(f), + AttrInput::TokenTree(subtree, _) => subtree.fmt(f), } } } @@ -682,7 +682,7 @@ impl Attr { Some(Interned::new(AttrInput::Literal(value))) } else if let Some(tt) = ast.token_tree() { let (tree, map) = syntax_node_to_token_tree(tt.syntax()); - Some(Interned::new(AttrInput::TokenTree(MappedSubTree { tree, map }))) + Some(Interned::new(AttrInput::TokenTree(tree, map))) } else { None }; @@ -712,10 +712,9 @@ impl Attr { } match self.input.as_deref() { - Some(AttrInput::TokenTree(args)) => { + Some(AttrInput::TokenTree(args, _)) => { let mut counter = 0; let paths = args - .tree .token_trees .iter() .group_by(move |tt| { @@ -760,7 +759,7 @@ pub struct AttrQuery<'a> { impl<'a> AttrQuery<'a> { pub fn tt_values(self) -> impl Iterator { self.attrs().filter_map(|attr| match attr.input.as_deref()? { - AttrInput::TokenTree(it) => Some(&it.tree), + AttrInput::TokenTree(it, _) => Some(it), _ => None, }) } diff --git a/crates/hir_def/src/lib.rs b/crates/hir_def/src/lib.rs index 3c9cf0e38dc..ea6a2b4a6fa 100644 --- a/crates/hir_def/src/lib.rs +++ b/crates/hir_def/src/lib.rs @@ -787,12 +787,12 @@ fn attr_macro_as_call_id( let mut arg = match ¯o_attr.input { Some(input) => match &**input { attr::AttrInput::Literal(_) => Default::default(), - attr::AttrInput::TokenTree(tt) => tt.clone(), + attr::AttrInput::TokenTree(tt, map) => (tt.clone(), map.clone()), }, None => Default::default(), }; // The parentheses are always disposed here. - arg.tree.delimiter = None; + arg.0.delimiter = None; let res = def.as_lazy_macro( db.upcast(), diff --git a/crates/hir_def/src/nameres/collector.rs b/crates/hir_def/src/nameres/collector.rs index 246bba9a11e..352d6fd6542 100644 --- a/crates/hir_def/src/nameres/collector.rs +++ b/crates/hir_def/src/nameres/collector.rs @@ -289,7 +289,7 @@ impl DefCollector<'_> { || *attr_name == hir_expand::name![register_tool] { match attr.input.as_deref() { - Some(AttrInput::TokenTree(subtree)) => match &*subtree.tree.token_trees { + Some(AttrInput::TokenTree(subtree, _)) => match &*subtree.token_trees { [tt::TokenTree::Leaf(tt::Leaf::Ident(name))] => name.as_name(), _ => continue, }, diff --git a/crates/hir_expand/src/db.rs b/crates/hir_expand/src/db.rs index b7951cc282d..0c5457016ef 100644 --- a/crates/hir_expand/src/db.rs +++ b/crates/hir_expand/src/db.rs @@ -387,7 +387,7 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult { - let mut attr_args = attr_args.tree.clone(); + let mut attr_args = attr_args.0.clone(); mbe::Shift::new(¯o_arg.0).shift_all(&mut attr_args); Some(attr_args) } diff --git a/crates/hir_expand/src/lib.rs b/crates/hir_expand/src/lib.rs index 86af19626e7..ebf458fe359 100644 --- a/crates/hir_expand/src/lib.rs +++ b/crates/hir_expand/src/lib.rs @@ -283,7 +283,7 @@ pub enum MacroCallKind { Attr { ast_id: AstId, attr_name: String, - attr_args: mbe::MappedSubTree, + attr_args: (tt::Subtree, mbe::TokenMap), /// Syntactical index of the invoking `#[attribute]`. /// /// Outer attributes are counted first, then inner attributes. This does not support @@ -390,7 +390,7 @@ impl ExpansionInfo { token_tree.left_delimiter_token()?.text_range().start(); let range = token.value.text_range().checked_sub(attr_input_start)?; let token_id = - self.macro_arg_shift.shift(attr_args.map.token_by_range(range)?); + self.macro_arg_shift.shift(attr_args.1.token_by_range(range)?); Some(token_id) } _ => None, @@ -437,7 +437,7 @@ impl ExpansionInfo { MacroCallKind::Attr { attr_args, .. } => match self.macro_arg_shift.unshift(token_id) { Some(unshifted) => { token_id = unshifted; - (&attr_args.map, self.attr_input_or_mac_def.clone()?.syntax().cloned()) + (&attr_args.1, self.attr_input_or_mac_def.clone()?.syntax().cloned()) } None => (&self.macro_arg.1, self.arg.clone()), }, diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index a2a6d02b8a2..d2b955c5c81 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -69,7 +69,7 @@ pub use crate::{ parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, }, - token_map::{MappedSubTree, TokenMap}, + token_map::TokenMap, }; /// This struct contains AST for a single `macro_rules` definition. What might diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 3e8c4b2d6da..ff0c106cf25 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -5,7 +5,7 @@ use std::hash::Hash; use parser::{SyntaxKind, T}; use syntax::{TextRange, TextSize}; -#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] enum TokenTextRange { Token(TextRange), Delimiter(TextRange), @@ -26,27 +26,8 @@ impl TokenTextRange { } } -#[derive(Debug, Clone, Default)] -pub struct MappedSubTree { - pub tree: tt::Subtree, - pub map: TokenMap, -} - -impl Eq for MappedSubTree {} -impl PartialEq for MappedSubTree { - fn eq(&self, other: &Self) -> bool { - self.tree == other.tree && self.map == other.map - } -} - -impl Hash for MappedSubTree { - fn hash(&self, state: &mut H) { - self.tree.hash(state); - } -} - /// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Debug, PartialEq, Eq, Clone, Default)] +#[derive(Debug, PartialEq, Eq, Clone, Default, Hash)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. entries: Vec<(tt::TokenId, TokenTextRange)>, From 4933beca87a19233cb6de6384da29f081eb05aaf Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 22 Aug 2021 19:12:45 +0200 Subject: [PATCH 3/3] Respect attributes in Hygiene token up-mapping --- crates/hir_expand/src/hygiene.rs | 79 ++++++++++++++++++++++---------- crates/hir_expand/src/lib.rs | 10 ++-- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/crates/hir_expand/src/hygiene.rs b/crates/hir_expand/src/hygiene.rs index 84852241155..5deb59ae31f 100644 --- a/crates/hir_expand/src/hygiene.rs +++ b/crates/hir_expand/src/hygiene.rs @@ -9,12 +9,15 @@ use db::TokenExpander; use either::Either; use mbe::Origin; use parser::SyntaxKind; -use syntax::{ast, AstNode, SyntaxNode, TextRange, TextSize}; +use syntax::{ + ast::{self, AttrsOwner}, + AstNode, SyntaxNode, TextRange, TextSize, +}; use crate::{ db::{self, AstDatabase}, name::{AsName, Name}, - HirFileId, HirFileIdRepr, InFile, MacroCallLoc, MacroDefKind, MacroFile, + HirFileId, HirFileIdRepr, InFile, MacroCallKind, MacroCallLoc, MacroDefKind, MacroFile, }; #[derive(Clone, Debug)] @@ -121,11 +124,12 @@ impl HygieneFrames { #[derive(Debug, Clone, PartialEq, Eq)] struct HygieneInfo { file: MacroFile, - /// The `macro_rules!` arguments. - def_start: Option>, + /// The start offset of the `macro_rules!` arguments or attribute input. + attr_input_or_mac_def_start: Option>, macro_def: Arc, macro_arg: Arc<(tt::Subtree, mbe::TokenMap)>, + macro_arg_shift: mbe::Shift, exp_map: Arc, } @@ -136,22 +140,34 @@ impl HygieneInfo { token: TextRange, ) -> Option<(InFile, Origin)> { let token_id = self.exp_map.token_by_range(token)?; + let (mut token_id, origin) = self.macro_def.map_id_up(token_id); - let (token_id, origin) = self.macro_def.map_id_up(token_id); - let (token_map, tt) = match origin { - mbe::Origin::Call => { - let call_id = self.file.macro_call_id; - let loc: MacroCallLoc = db.lookup_intern_macro(call_id); - let arg_start = loc.kind.arg(db)?.text_range().start(); - (&self.macro_arg.1, InFile::new(loc.kind.file_id(), arg_start)) - } - mbe::Origin::Def => match (&*self.macro_def, self.def_start) { - ( - TokenExpander::MacroDef { def_site_token_map, .. } - | TokenExpander::MacroRules { def_site_token_map, .. }, - Some(tt), - ) => (def_site_token_map, tt), - _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"), + let loc = db.lookup_intern_macro(self.file.macro_call_id); + + let (token_map, tt) = match &loc.kind { + MacroCallKind::Attr { attr_args, .. } => match self.macro_arg_shift.unshift(token_id) { + Some(unshifted) => { + token_id = unshifted; + (&attr_args.1, self.attr_input_or_mac_def_start?) + } + None => ( + &self.macro_arg.1, + InFile::new(loc.kind.file_id(), loc.kind.arg(db)?.text_range().start()), + ), + }, + _ => match origin { + mbe::Origin::Call => ( + &self.macro_arg.1, + InFile::new(loc.kind.file_id(), loc.kind.arg(db)?.text_range().start()), + ), + mbe::Origin::Def => match (&*self.macro_def, &self.attr_input_or_mac_def_start) { + ( + TokenExpander::MacroDef { def_site_token_map, .. } + | TokenExpander::MacroRules { def_site_token_map, .. }, + Some(tt), + ) => (def_site_token_map, *tt), + _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"), + }, }, }; @@ -165,19 +181,34 @@ fn make_hygiene_info( macro_file: MacroFile, loc: &MacroCallLoc, ) -> Option { - let def_offset = loc.def.ast_id().left().and_then(|id| { + let def = loc.def.ast_id().left().and_then(|id| { let def_tt = match id.to_node(db) { - ast::Macro::MacroRules(mac) => mac.token_tree()?.syntax().text_range().start(), - ast::Macro::MacroDef(mac) => mac.body()?.syntax().text_range().start(), + ast::Macro::MacroRules(mac) => mac.token_tree()?, + ast::Macro::MacroDef(mac) => mac.body()?, }; Some(InFile::new(id.file_id, def_tt)) }); + let attr_input_or_mac_def = def.or_else(|| match loc.kind { + MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => { + let tt = ast_id.to_node(db).attrs().nth(invoc_attr_index as usize)?.token_tree()?; + Some(InFile::new(ast_id.file_id, tt)) + } + _ => None, + }); let macro_def = db.macro_def(loc.def)?; let (_, exp_map) = db.parse_macro_expansion(macro_file).value?; let macro_arg = db.macro_arg(macro_file.macro_call_id)?; - Some(HygieneInfo { file: macro_file, def_start: def_offset, macro_arg, macro_def, exp_map }) + Some(HygieneInfo { + file: macro_file, + attr_input_or_mac_def_start: attr_input_or_mac_def + .map(|it| it.map(|tt| tt.syntax().text_range().start())), + macro_arg_shift: mbe::Shift::new(¯o_arg.0), + macro_arg, + macro_def, + exp_map, + }) } impl HygieneFrame { @@ -214,7 +245,7 @@ impl HygieneFrame { Some(it) => it, }; - let def_site = info.def_start.map(|it| db.hygiene_frame(it.file_id)); + let def_site = info.attr_input_or_mac_def_start.map(|it| db.hygiene_frame(it.file_id)); let call_site = Some(db.hygiene_frame(calling_file)); HygieneFrame { expansion: Some(info), local_inner, krate, call_site, def_site } diff --git a/crates/hir_expand/src/lib.rs b/crates/hir_expand/src/lib.rs index ebf458fe359..40380e1df17 100644 --- a/crates/hir_expand/src/lib.rs +++ b/crates/hir_expand/src/lib.rs @@ -133,13 +133,13 @@ impl HirFileId { }; Some(InFile::new(id.file_id, def_tt)) }); - let def_or_attr_input = def.or_else(|| match loc.kind { + let attr_input_or_mac_def = def.or_else(|| match loc.kind { MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => { let tt = ast_id .to_node(db) .attrs() - .nth(invoc_attr_index as usize) - .and_then(|attr| attr.token_tree())?; + .nth(invoc_attr_index as usize)? + .token_tree()?; Some(InFile::new(ast_id.file_id, tt)) } _ => None, @@ -152,7 +152,7 @@ impl HirFileId { Some(ExpansionInfo { expanded: InFile::new(self, parse.syntax_node()), arg: InFile::new(loc.kind.file_id(), arg_tt), - attr_input_or_mac_def: def_or_attr_input, + attr_input_or_mac_def, macro_arg_shift: mbe::Shift::new(¯o_arg.0), macro_arg, macro_def, @@ -443,7 +443,7 @@ impl ExpansionInfo { }, _ => match origin { mbe::Origin::Call => (&self.macro_arg.1, self.arg.clone()), - mbe::Origin::Def => match (&*self.macro_def, self.attr_input_or_mac_def.as_ref()) { + mbe::Origin::Def => match (&*self.macro_def, &self.attr_input_or_mac_def) { ( TokenExpander::MacroRules { def_site_token_map, .. } | TokenExpander::MacroDef { def_site_token_map, .. },