don't rely on spans when checking tokens for jointness

src/libsyntax/parse/lexer/mod.rs

@@ -123,8 +123,20 @@ impl<'a> StringReader<'a> {
     /// `Err(())` means that some errors were encountered, which can be
     /// retrieved using `buffer_fatal_errors`.
     pub fn try_next_token(&mut self) -> Result<Token, ()> {
-        let (token, _raw_span) = self.try_next_token_with_raw_span()?;
-        Ok(token)
+        assert!(self.fatal_errs.is_empty());
+        match self.scan_whitespace_or_comment() {
+            Some(comment) => Ok(comment),
+            None => {
+                let (kind, start_pos, end_pos) = if self.is_eof() {
+                    (token::Eof, self.source_file.end_pos, self.source_file.end_pos)
+                } else {
+                    let start_pos = self.pos;
+                    (self.next_token_inner()?, start_pos, self.pos)
+                };
+                let (real, _raw) = self.mk_sp_and_raw(start_pos, end_pos);
+                Ok(Token::new(kind, real))
+            }
+        }
     }
 
     /// Returns the next token, including trivia like whitespace or comments.
@@ -135,42 +147,6 @@ impl<'a> StringReader<'a> {
         self.unwrap_or_abort(res)
     }
 
-    /// Returns the next token, skipping over trivia.
-    /// Also returns an unoverriden span which can be used to check tokens
-    fn real_token(&mut self) -> (Token, Span) {
-        let res = try {
-            loop {
-                let t = self.try_next_token_with_raw_span()?;
-                match t.0.kind {
-                    token::Whitespace | token::Comment | token::Shebang(_) => continue,
-                    _ => break t,
-                }
-            }
-        };
-
-        self.unwrap_or_abort(res)
-    }
-
-    fn try_next_token_with_raw_span(&mut self) -> Result<(Token, Span), ()> {
-        assert!(self.fatal_errs.is_empty());
-        match self.scan_whitespace_or_comment() {
-            Some(comment) => {
-                let raw_span = comment.span;
-                Ok((comment, raw_span))
-            }
-            None => {
-                let (kind, start_pos, end_pos) = if self.is_eof() {
-                    (token::Eof, self.source_file.end_pos, self.source_file.end_pos)
-                } else {
-                    let start_pos = self.pos;
-                    (self.next_token_inner()?, start_pos, self.pos)
-                };
-                let (real, raw) = self.mk_sp_and_raw(start_pos, end_pos);
-                Ok((Token::new(kind, real), raw))
-            }
-        }
-    }
-
     #[inline]
     fn is_eof(&self) -> bool {
         self.ch.is_none()

src/libsyntax/parse/lexer/tokentrees.rs

@@ -1,17 +1,17 @@
-use syntax_pos::{Span, DUMMY_SP};
+use syntax_pos::Span;
 
 use crate::print::pprust::token_to_string;
 use crate::parse::lexer::{StringReader, UnmatchedBrace};
 use crate::parse::token::{self, Token};
 use crate::parse::PResult;
-use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
+use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint};
 
 impl<'a> StringReader<'a> {
     crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
         let mut tt_reader = TokenTreesReader {
             string_reader: self,
             token: Token::dummy(),
-            raw_span: DUMMY_SP,
+            joint_to_prev: Joint,
             open_braces: Vec::new(),
             unmatched_braces: Vec::new(),
             matching_delim_spans: Vec::new(),
@@ -25,7 +25,7 @@ impl<'a> StringReader<'a> {
 struct TokenTreesReader<'a> {
     string_reader: StringReader<'a>,
     token: Token,
-    raw_span: Span,
+    joint_to_prev: IsJoint,
     /// Stack of open delimiters and their spans. Used for error message.
     open_braces: Vec<(token::DelimToken, Span)>,
     unmatched_braces: Vec<UnmatchedBrace>,
@@ -205,20 +205,26 @@ impl<'a> TokenTreesReader<'a> {
             },
             _ => {
                 let tt = TokenTree::Token(self.token.take());
-                // Note that testing for joint-ness here is done via the raw
-                // source span as the joint-ness is a property of the raw source
-                // rather than wanting to take `override_span` into account.
-                let raw_span = self.raw_span;
                 self.real_token();
-                let is_joint = raw_span.hi() == self.raw_span.lo() && self.token.is_op();
+                let is_joint = self.joint_to_prev == Joint && self.token.is_op();
                 Ok((tt, if is_joint { Joint } else { NonJoint }))
             }
         }
     }
 
     fn real_token(&mut self) {
-        let (token, raw_span) = self.string_reader.real_token();
-        self.token = token;
-        self.raw_span = raw_span;
+        self.joint_to_prev = Joint;
+        loop {
+            let token = self.string_reader.next_token();
+            match token.kind {
+                token::Whitespace | token::Comment | token::Shebang(_) => {
+                    self.joint_to_prev = NonJoint;
+                }
+                _ => {
+                    self.token = token;
+                    return;
+                },
+            }
+        }
     }
 }