nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-27 09:14:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #60763 - matklad:tt-parser, r=petrochenkov

Browse Source 
Move token tree related lexer state to a separate struct

Just a types-based refactoring.

We only used a bunch of fields when tokenizing into a token tree, so let's move them out of the base lexer
This commit is contained in:
bors 2019-05-16 04:15:12 +00:00
commit 024c25dc79
3 changed files with 62 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/libsyntax/parse/lexer/mod.rs
View File

@ -62,19 +62,7 @@ pub struct StringReader<'a> {
// cache a direct reference to the source text, so that we don't have to
// retrieve it via `self.source_file.src.as_ref().unwrap()` all the time.
src: Lrc<String>,
token: token::Token,
span: Span,
/// The raw source span which *does not* take `override_span` into account
span_src_raw: Span,
/// Stack of open delimiters and their spans. Used for error message.
open_braces: Vec<(token::DelimToken, Span)>,
crate unmatched_braces: Vec<UnmatchedBrace>,
/// The type and spans for all braces
///
/// Used only for error recovery when arriving to EOF with mismatched braces.
matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
crate override_span: Option<Span>,
last_unclosed_found_span: Option<Span>,
override_span: Option<Span>,
}
impl<'a> StringReader<'a> {
@ -121,8 +109,6 @@ impl<'a> StringReader<'a> {
sp: self.peek_span,
};
self.advance_token()?;
self.span_src_raw = self.peek_span_src_raw;
Ok(ret_val)
}
@ -159,9 +145,6 @@ impl<'a> StringReader<'a> {
}
}
self.token = t.tok.clone();
self.span = t.sp;
Ok(t)
}
@ -251,14 +234,7 @@ impl<'a> StringReader<'a> {
peek_span_src_raw: syntax_pos::DUMMY_SP,
src,
fatal_errs: Vec::new(),
token: token::Eof,
span: syntax_pos::DUMMY_SP,
span_src_raw: syntax_pos::DUMMY_SP,
open_braces: Vec::new(),
unmatched_braces: Vec::new(),
matching_delim_spans: Vec::new(),
override_span,
last_unclosed_found_span: None,
}
}

67
src/libsyntax/parse/lexer/tokentrees.rs
View File

@ -1,13 +1,46 @@
use syntax_pos::Span;
use crate::print::pprust::token_to_string;
use crate::parse::lexer::{StringReader, UnmatchedBrace};
use crate::parse::{token, PResult};
use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
impl<'a> StringReader<'a> {
crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
let mut tt_reader = TokenTreesReader {
string_reader: self,
token: token::Eof,
span: syntax_pos::DUMMY_SP,
open_braces: Vec::new(),
unmatched_braces: Vec::new(),
matching_delim_spans: Vec::new(),
last_unclosed_found_span: None,
};
let res = tt_reader.parse_all_token_trees();
(res, tt_reader.unmatched_braces)
}
}
struct TokenTreesReader<'a> {
string_reader: StringReader<'a>,
token: token::Token,
span: Span,
/// Stack of open delimiters and their spans. Used for error message.
open_braces: Vec<(token::DelimToken, Span)>,
unmatched_braces: Vec<UnmatchedBrace>,
/// The type and spans for all braces
///
/// Used only for error recovery when arriving to EOF with mismatched braces.
matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
last_unclosed_found_span: Option<Span>,
}
impl<'a> TokenTreesReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
crate fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
let mut tts = Vec::new();
self.real_token();
while self.token != token::Eof {
tts.push(self.parse_token_tree()?);
}
@ -34,11 +67,12 @@ impl<'a> StringReader<'a> {
}
fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
let sm = self.sess.source_map();
let sm = self.string_reader.sess.source_map();
match self.token {
token::Eof => {
let msg = "this file contains an un-closed delimiter";
let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg);
let mut err = self.string_reader.sess.span_diagnostic
.struct_span_err(self.span, msg);
for &(_, sp) in &self.open_braces {
err.span_label(sp, "un-closed delimiter");
}
@ -46,13 +80,12 @@ impl<'a> StringReader<'a> {
if let Some((delim, _)) = self.open_braces.last() {
if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
.filter(|(d, open_sp, close_sp)| {
if let Some(close_padding) = sm.span_to_margin(*close_sp) {
if let Some(open_padding) = sm.span_to_margin(*open_sp) {
return delim == d && close_padding != open_padding;
if let Some(close_padding) = sm.span_to_margin(*close_sp) {
if let Some(open_padding) = sm.span_to_margin(*open_sp) {
return delim == d && close_padding != open_padding;
}
}
}
false
false
}).next() // these are in reverse order as they get inserted on close, but
{ // we want the last open/first close
err.span_label(
@ -164,7 +197,8 @@ impl<'a> StringReader<'a> {
// matching opening delimiter).
let token_str = token_to_string(&self.token);
let msg = format!("unexpected close delimiter: `{}`", token_str);
let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg);
let mut err = self.string_reader.sess.span_diagnostic
.struct_span_err(self.span, &msg);
err.span_label(self.span, "unexpected close delimiter");
Err(err)
},
@ -173,11 +207,20 @@ impl<'a> StringReader<'a> {
// Note that testing for joint-ness here is done via the raw
// source span as the joint-ness is a property of the raw source
// rather than wanting to take `override_span` into account.
let raw = self.span_src_raw;
// Additionally, we actually check if the *next* pair of tokens
// is joint, but this is equivalent to checking the current pair.
let raw = self.string_reader.peek_span_src_raw;
self.real_token();
let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
let is_joint = raw.hi() == self.string_reader.peek_span_src_raw.lo()
&& token::is_op(&self.token);
Ok((tt, if is_joint { Joint } else { NonJoint }))
}
}
}
fn real_token(&mut self) {
let t = self.string_reader.real_token();
self.token = t.tok;
self.span = t.sp;
}
}

12
src/libsyntax/parse/mod.rs
View File

@ -290,22 +290,22 @@ pub fn source_file_to_stream(
}
/// Given a source file, produces a sequence of token trees. Returns any buffered errors from
/// parsing the token tream.
/// parsing the token stream.
pub fn maybe_file_to_stream(
sess: &ParseSess,
source_file: Lrc<SourceFile>,
override_span: Option<Span>,
) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> {
let mut srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?;
srdr.real_token();
let srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?;
let (token_trees, unmatched_braces) = srdr.into_token_trees();
match srdr.parse_all_token_trees() {
Ok(stream) => Ok((stream, srdr.unmatched_braces)),
match token_trees {
Ok(stream) => Ok((stream, unmatched_braces)),
Err(err) => {
let mut buffer = Vec::with_capacity(1);
err.buffer(&mut buffer);
// Not using `emit_unclosed_delims` to use `db.buffer`
for unmatched in srdr.unmatched_braces {
for unmatched in unmatched_braces {
let mut db = sess.span_diagnostic.struct_span_err(unmatched.found_span, &format!(
"incorrect close delimiter: `{}`",
token_to_string(&token::Token::CloseDelim(unmatched.found_delim)),