mirror of
https://github.com/rust-lang/rust.git
synced 2024-12-09 15:13:57 +00:00
glue tokens when building token stream
This commit is contained in:
parent
8b932dfda7
commit
914e1f4564
@ -39,29 +39,29 @@ struct TokenTreesReader<'a> {
|
||||
impl<'a> TokenTreesReader<'a> {
|
||||
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
|
||||
fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
|
||||
let mut tts = Vec::new();
|
||||
let mut buf = TokenStreamBuilder::default();
|
||||
|
||||
self.real_token();
|
||||
while self.token != token::Eof {
|
||||
tts.push(self.parse_token_tree()?);
|
||||
buf.push(self.parse_token_tree()?);
|
||||
}
|
||||
|
||||
Ok(TokenStream::new(tts))
|
||||
Ok(buf.into_token_stream())
|
||||
}
|
||||
|
||||
// Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
|
||||
fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
|
||||
let mut tts = vec![];
|
||||
let mut buf = TokenStreamBuilder::default();
|
||||
loop {
|
||||
if let token::CloseDelim(..) = self.token.kind {
|
||||
return TokenStream::new(tts);
|
||||
return buf.into_token_stream();
|
||||
}
|
||||
|
||||
match self.parse_token_tree() {
|
||||
Ok(tree) => tts.push(tree),
|
||||
Ok(tree) => buf.push(tree),
|
||||
Err(mut e) => {
|
||||
e.emit();
|
||||
return TokenStream::new(tts);
|
||||
return buf.into_token_stream();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -223,8 +223,32 @@ impl<'a> TokenTreesReader<'a> {
|
||||
_ => {
|
||||
self.token = token;
|
||||
return;
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TokenStreamBuilder {
|
||||
buf: Vec<TreeAndJoint>,
|
||||
}
|
||||
|
||||
impl TokenStreamBuilder {
|
||||
fn push(&mut self, (tree, joint): TreeAndJoint) {
|
||||
if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
|
||||
if let TokenTree::Token(token) = &tree {
|
||||
if let Some(glued) = prev_token.glue(token) {
|
||||
self.buf.pop();
|
||||
self.buf.push((TokenTree::Token(glued), joint));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.buf.push((tree, joint))
|
||||
}
|
||||
|
||||
fn into_token_stream(self) -> TokenStream {
|
||||
TokenStream::new(self.buf)
|
||||
}
|
||||
}
|
||||
|
@ -551,7 +551,7 @@ impl Token {
|
||||
}
|
||||
}
|
||||
|
||||
crate fn glue(self, joint: Token) -> Option<Token> {
|
||||
crate fn glue(&self, joint: &Token) -> Option<Token> {
|
||||
let kind = match self.kind {
|
||||
Eq => match joint.kind {
|
||||
Eq => EqEq,
|
||||
|
@ -414,7 +414,7 @@ impl TokenStreamBuilder {
|
||||
let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
|
||||
if let Some(TokenTree::Token(last_token)) = last_tree_if_joint {
|
||||
if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() {
|
||||
if let Some(glued_tok) = last_token.glue(token) {
|
||||
if let Some(glued_tok) = last_token.glue(&token) {
|
||||
let last_stream = self.0.pop().unwrap();
|
||||
self.push_all_but_last_tree(&last_stream);
|
||||
let glued_tt = TokenTree::Token(glued_tok);
|
||||
|
Loading…
Reference in New Issue
Block a user