diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index fb98f55a215..151acddae84 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -403,8 +403,8 @@ impl Cursor { self.index = index; } - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { - self.stream.0[self.index..].get(n).map(|(tree, _)| tree) + pub fn look_ahead(&self, n: usize) -> Option { + self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone()) } } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 765871a6396..39c82f97e0a 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -47,26 +47,15 @@ impl ToInternal for Delimiter { } } -impl - FromInternal<( - TreeAndJoint, - Option<&'_ tokenstream::TokenTree>, - &'_ ParseSess, - &'_ mut Vec, - )> for TokenTree +impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> + for TokenTree { fn from_internal( - ((tree, is_joint), look_ahead, sess, stack): ( - TreeAndJoint, - Option<&tokenstream::TokenTree>, - &ParseSess, - &mut Vec, - ), + ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec), ) -> Self { use rustc_ast::token::*; - let joint = is_joint == Joint - && matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op()); + let joint = is_joint == Joint; let Token { kind, span } = match tree { tokenstream::TokenTree::Delimited(span, delim, tts) => { let delimiter = Delimiter::from_internal(delim); @@ -456,8 +445,7 @@ impl server::TokenStreamIter for Rustc<'_> { loop { let tree = iter.stack.pop().or_else(|| { let next = iter.cursor.next_with_joint()?; - let lookahead = iter.cursor.look_ahead(0); - Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack))) + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) })?; // A hack used to pass AST fragments to attribute and derive macros // as a single nonterminal token instead of a token stream. diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index fb27ccfbd94..d5977ca3c7d 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -262,7 +262,10 @@ impl<'a> TokenTreesReader<'a> { } _ => { let tt = TokenTree::Token(self.token.take()); - let is_joint = self.bump(); + let mut is_joint = self.bump(); + if !self.token.is_op() { + is_joint = NonJoint; + } Ok((tt, is_joint)) } } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 1b2067f8f25..84edfecad19 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -822,15 +822,15 @@ impl<'a> Parser<'a> { } let frame = &self.token_cursor.frame; - match frame.tree_cursor.look_ahead(dist - 1) { + looker(&match frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { - TokenTree::Token(token) => looker(token), + TokenTree::Token(token) => token, TokenTree::Delimited(dspan, delim, _) => { - looker(&Token::new(token::OpenDelim(delim.clone()), dspan.open)) + Token::new(token::OpenDelim(delim), dspan.open) } }, - None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)), - } + None => Token::new(token::CloseDelim(frame.delim), frame.span.close), + }) } /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.