mirror of
https://github.com/rust-lang/rust.git
synced 2024-11-01 06:51:58 +00:00
Move jointness censoring to proc_macro
Proc-macro API currently exposes jointness in `Punct` tokens. That is, `+` in `+one` is **non** joint. Our lexer produces jointness info for all tokens, so we need to censor it *somewhere* Previously we did this in a lexer, but it makes more sense to do this in a proc-macro server.
This commit is contained in:
parent
08deb863bd
commit
850c3219fb
@ -47,15 +47,21 @@ impl ToInternal<token::DelimToken> for Delimiter {
|
||||
}
|
||||
}
|
||||
|
||||
impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
|
||||
impl FromInternal<(TreeAndJoint, Option<tokenstream::TokenTree>, &'_ ParseSess, &'_ mut Vec<Self>)>
|
||||
for TokenTree<Group, Punct, Ident, Literal>
|
||||
{
|
||||
fn from_internal(
|
||||
((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
|
||||
((tree, is_joint), look_ahead, sess, stack): (
|
||||
TreeAndJoint,
|
||||
Option<tokenstream::TokenTree>,
|
||||
&ParseSess,
|
||||
&mut Vec<Self>,
|
||||
),
|
||||
) -> Self {
|
||||
use rustc_ast::token::*;
|
||||
|
||||
let joint = is_joint == Joint;
|
||||
let joint = is_joint == Joint
|
||||
&& matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op());
|
||||
let Token { kind, span } = match tree {
|
||||
tokenstream::TokenTree::Delimited(span, delim, tts) => {
|
||||
let delimiter = Delimiter::from_internal(delim);
|
||||
@ -445,7 +451,8 @@ impl server::TokenStreamIter for Rustc<'_> {
|
||||
loop {
|
||||
let tree = iter.stack.pop().or_else(|| {
|
||||
let next = iter.cursor.next_with_joint()?;
|
||||
Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
|
||||
let lookahead = iter.cursor.look_ahead(0);
|
||||
Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack)))
|
||||
})?;
|
||||
// A hack used to pass AST fragments to attribute and derive macros
|
||||
// as a single nonterminal token instead of a token stream.
|
||||
|
@ -262,10 +262,7 @@ impl<'a> TokenTreesReader<'a> {
|
||||
}
|
||||
_ => {
|
||||
let tt = TokenTree::Token(self.token.take());
|
||||
let mut is_joint = self.bump();
|
||||
if !self.token.is_op() {
|
||||
is_joint = NonJoint;
|
||||
}
|
||||
let is_joint = self.bump();
|
||||
Ok((tt, is_joint))
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user