nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-26 16:54:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #14084 - Veykril:float-parse, r=Veykril

Browse Source 
fix: Fix parsing of nested tuple field accesses in a cursed way

This is absolutely terrible but seems to work. Macro fragment parsing comes next.
This commit is contained in:
bors 2023-02-07 17:25:31 +00:00
commit 7f1234492e
20 changed files with 451 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crates/hir-def/src/item_tree.rs
View File

@ -111,7 +111,8 @@ impl ItemTree {
Some(node) => node,
None => return Default::default(),
};
if never!(syntax.kind() == SyntaxKind::ERROR) {
if never!(syntax.kind() == SyntaxKind::ERROR, "{:?} from {:?} {}", file_id, syntax, syntax)
{
// FIXME: not 100% sure why these crop up, but return an empty tree to avoid a panic
return Default::default();
}
@ -133,7 +134,7 @@ impl ItemTree {
ctx.lower_macro_stmts(stmts)
},
_ => {
panic!("cannot create item tree from {syntax:?} {syntax}");
panic!("cannot create item tree for file {file_id:?} from {syntax:?} {syntax}");
},
}
};

35
crates/hir-def/src/macro_expansion_tests/mbe.rs
View File

@ -97,6 +97,41 @@ fn#19 main#20(#21)#21 {#22
"##]],
);
}
#[test]
fn float_field_acces_macro_input() {
check(
r#"
macro_rules! foo {
($expr:expr) => {
fn foo() {
$expr;
}
};
}
foo!(x .0.1);
foo!(x .2. 3);
foo!(x .4 .5);
"#,
expect![[r#"
macro_rules! foo {
($expr:expr) => {
fn foo() {
$expr;
}
};
}
fn foo() {
(x.0.1);
}
fn foo() {
(x.2.3);
}
fn foo() {
(x.4.5);
}
"#]],
);
}
#[test]
fn mbe_smoke_test() {

7
crates/hir-def/src/macro_expansion_tests/proc_macros.rs
View File

@ -104,7 +104,7 @@ macro_rules! id {
$($t)*
};
}
id /*+errors*/! {
id! {
#[proc_macros::identity]
impl Foo for WrapBj {
async fn foo(&self) {
@ -113,18 +113,17 @@ id /*+errors*/! {
}
}
"#,
expect![[r##"
expect![[r#"
macro_rules! id {
($($t:tt)*) => {
$($t)*
};
}
/* parse error: expected SEMICOLON */
#[proc_macros::identity] impl Foo for WrapBj {
async fn foo(&self ) {
self .0.id().await ;
}
}
"##]],
"#]],
);
}

40
crates/mbe/src/syntax_bridge.rs
View File

@ -95,6 +95,9 @@ pub fn token_tree_to_syntax_node(
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
tree_sink.token(kind, n_raw_tokens)
}
parser::Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
tree_sink.float_split(has_pseudo_dot)
}
parser::Step::Enter { kind } => tree_sink.start_node(kind),
parser::Step::Exit => tree_sink.finish_node(),
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
@ -796,6 +799,43 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
}
impl<'a> TtTreeSink<'a> {
/// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween.
/// This occurs when a float literal is used as a field access.
fn float_split(&mut self, has_pseudo_dot: bool) {
let (text, _span) = match self.cursor.token_tree() {
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => {
(lit.text.as_str(), lit.span)
}
_ => unreachable!(),
};
match text.split_once('.') {
Some((left, right)) => {
assert!(!left.is_empty());
self.inner.start_node(SyntaxKind::NAME_REF);
self.inner.token(SyntaxKind::INT_NUMBER, left);
self.inner.finish_node();
// here we move the exit up, the original exit has been deleted in process
self.inner.finish_node();
self.inner.token(SyntaxKind::DOT, ".");
if has_pseudo_dot {
assert!(right.is_empty(), "{left}.{right}");
} else {
self.inner.start_node(SyntaxKind::NAME_REF);
self.inner.token(SyntaxKind::INT_NUMBER, right);
self.inner.finish_node();
// the parser creates an unbalanced start node, we are required to close it here
self.inner.finish_node();
}
}
None => unreachable!(),
}
self.cursor = self.cursor.bump();
}
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
if kind == LIFETIME_IDENT {
n_tokens = 2;

7
crates/mbe/src/to_parser_input.rs
View File

@ -45,6 +45,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
res.push(kind);
if kind == FLOAT_NUMBER && !inner_text.ends_with('.') {
// Tag the token as joint if it is float with a fractional part
// we use this jointness to inform the parser about what token split
// event to emit when we encounter a float literal in a field access
res.was_joint();
}
}
tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]),

16
crates/mbe/src/tt_iter.rs
View File

@ -150,6 +150,11 @@ impl<'a> TtIter<'a> {
cursor = cursor.bump_subtree();
}
}
parser::Step::FloatSplit { .. } => {
// FIXME: We need to split the tree properly here, but mutating the token trees
// in the buffer is somewhat tricky to pull off.
cursor = cursor.bump_subtree();
}
parser::Step::Enter { .. } | parser::Step::Exit => (),
parser::Step::Error { .. } => error = true,
}
@ -166,19 +171,18 @@ impl<'a> TtIter<'a> {
if cursor.is_root() {
while curr != cursor {
if let Some(token) = curr.token_tree() {
res.push(token);
}
let Some(token) = curr.token_tree() else { break };
res.push(token.cloned());
curr = curr.bump();
}
}
self.inner = self.inner.as_slice()[res.len()..].iter();
let res = match res.len() {
1 => Some(res[0].cloned()),
0 => None,
0 | 1 => res.pop(),
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
token_trees: res,
})),
};
ExpandResult { value: res, err }

13
crates/parser/src/event.rs
View File

@ -74,7 +74,13 @@ pub(crate) enum Event {
kind: SyntaxKind,
n_raw_tokens: u8,
},
/// When we parse `foo.0.0` or `foo. 0. 0` the lexer will hand us a float literal
/// instead of an integer literal followed by a dot as the lexer has no contextual knowledge.
/// This event instructs whatever consumes the events to split the float literal into
/// the corresponding parts.
FloatSplitHack {
ends_in_dot: bool,
},
Error {
msg: String,
},
@ -125,6 +131,11 @@ pub(super) fn process(mut events: Vec<Event>) -> Output {
Event::Token { kind, n_raw_tokens } => {
res.token(kind, n_raw_tokens);
}
Event::FloatSplitHack { ends_in_dot } => {
res.float_split_hack(ends_in_dot);
let ev = mem::replace(&mut events[i + 1], Event::tombstone());
assert!(matches!(ev, Event::Finish), "{ev:?}");
}
Event::Error { msg } => res.error(msg),
}
}

106
crates/parser/src/grammar/expressions.rs
View File

@ -379,7 +379,7 @@ fn postfix_expr(
// }
T!['('] if allow_calls => call_expr(p, lhs),
T!['['] if allow_calls => index_expr(p, lhs),
T![.] => match postfix_dot_expr(p, lhs) {
T![.] => match postfix_dot_expr::<false>(p, lhs) {
Ok(it) => it,
Err(it) => {
lhs = it;
@ -393,35 +393,44 @@ fn postfix_expr(
block_like = BlockLike::NotBlock;
}
return (lhs, block_like);
}
fn postfix_dot_expr(
p: &mut Parser<'_>,
lhs: CompletedMarker,
) -> Result<CompletedMarker, CompletedMarker> {
fn postfix_dot_expr<const FLOAT_RECOVERY: bool>(
p: &mut Parser<'_>,
lhs: CompletedMarker,
) -> Result<CompletedMarker, CompletedMarker> {
if !FLOAT_RECOVERY {
assert!(p.at(T![.]));
if p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])) {
return Ok(method_call_expr(p, lhs));
}
// test await_expr
// fn foo() {
// x.await;
// x.0.await;
// x.0().await?.hello();
// }
if p.nth(1) == T![await] {
let m = lhs.precede(p);
p.bump(T![.]);
p.bump(T![await]);
return Ok(m.complete(p, AWAIT_EXPR));
}
if p.at(T![..=]) || p.at(T![..]) {
return Err(lhs);
}
Ok(field_expr(p, lhs))
}
let nth1 = if FLOAT_RECOVERY { 0 } else { 1 };
let nth2 = if FLOAT_RECOVERY { 1 } else { 2 };
if p.nth(nth1) == IDENT && (p.nth(nth2) == T!['('] || p.nth_at(nth2, T![::])) {
return Ok(method_call_expr::<FLOAT_RECOVERY>(p, lhs));
}
// test await_expr
// fn foo() {
// x.await;
// x.0.await;
// x.0().await?.hello();
// x.0.0.await;
// x.0. await;
// }
if p.nth(nth1) == T![await] {
let m = lhs.precede(p);
if !FLOAT_RECOVERY {
p.bump(T![.]);
}
p.bump(T![await]);
return Ok(m.complete(p, AWAIT_EXPR));
}
if p.at(T![..=]) || p.at(T![..]) {
return Err(lhs);
}
field_expr::<FLOAT_RECOVERY>(p, lhs)
}
// test call_expr
@ -455,11 +464,22 @@ fn index_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo();
// y.bar::<T>(1, 2,);
// x.0.0.call();
// x.0. call();
// }
fn method_call_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
fn method_call_expr<const FLOAT_RECOVERY: bool>(
p: &mut Parser<'_>,
lhs: CompletedMarker,
) -> CompletedMarker {
if FLOAT_RECOVERY {
assert!(p.nth(0) == IDENT && (p.nth(1) == T!['('] || p.nth_at(1, T![::])));
} else {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
}
let m = lhs.precede(p);
p.bump_any();
if !FLOAT_RECOVERY {
p.bump(T![.]);
}
name_ref(p);
generic_args::opt_generic_arg_list(p, true);
if p.at(T!['(']) {
@ -472,21 +492,35 @@ fn method_call_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker
// fn foo() {
// x.foo;
// x.0.bar;
// x.0.1;
// x.0. bar;
// x.0();
// }
fn field_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]));
fn field_expr<const FLOAT_RECOVERY: bool>(
p: &mut Parser<'_>,
lhs: CompletedMarker,
) -> Result<CompletedMarker, CompletedMarker> {
if !FLOAT_RECOVERY {
assert!(p.at(T![.]));
}
let m = lhs.precede(p);
p.bump(T![.]);
if !FLOAT_RECOVERY {
p.bump(T![.]);
}
if p.at(IDENT) || p.at(INT_NUMBER) {
name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
return match p.split_float(m) {
(true, m) => {
let lhs = m.complete(p, FIELD_EXPR);
postfix_dot_expr::<true>(p, lhs)
}
(false, m) => Ok(m.complete(p, FIELD_EXPR)),
};
} else {
p.error("expected field name or number");
}
m.complete(p, FIELD_EXPR)
Ok(m.complete(p, FIELD_EXPR))
}
// test try_expr

4
crates/parser/src/lib.rs
View File

@ -102,10 +102,14 @@ impl TopEntryPoint {
match step {
Step::Enter { .. } => depth += 1,
Step::Exit => depth -= 1,
Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
depth -= 1 + !has_pseudo_dot as usize
}
Step::Token { .. } | Step::Error { .. } => (),
}
}
assert!(!first, "no tree at all");
assert_eq!(depth, 0, "unbalanced tree");
}
res

61
crates/parser/src/output.rs
View File

@ -25,53 +25,88 @@ pub struct Output {
#[derive(Debug)]
pub enum Step<'a> {
Token { kind: SyntaxKind, n_input_tokens: u8 },
FloatSplit { ends_in_dot: bool },
Enter { kind: SyntaxKind },
Exit,
Error { msg: &'a str },
}
impl Output {
const EVENT_MASK: u32 = 0b1;
const TAG_MASK: u32 = 0x0000_00F0;
const N_INPUT_TOKEN_MASK: u32 = 0x0000_FF00;
const KIND_MASK: u32 = 0xFFFF_0000;
const ERROR_SHIFT: u32 = Self::EVENT_MASK.trailing_ones();
const TAG_SHIFT: u32 = Self::TAG_MASK.trailing_zeros();
const N_INPUT_TOKEN_SHIFT: u32 = Self::N_INPUT_TOKEN_MASK.trailing_zeros();
const KIND_SHIFT: u32 = Self::KIND_MASK.trailing_zeros();
const TOKEN_EVENT: u8 = 0;
const ENTER_EVENT: u8 = 1;
const EXIT_EVENT: u8 = 2;
const SPLIT_EVENT: u8 = 3;
pub fn iter(&self) -> impl Iterator<Item = Step<'_>> {
self.event.iter().map(|&event| {
if event & 0b1 == 0 {
return Step::Error { msg: self.error[(event as usize) >> 1].as_str() };
if event & Self::EVENT_MASK == 0 {
return Step::Error {
msg: self.error[(event as usize) >> Self::ERROR_SHIFT].as_str(),
};
}
let tag = ((event & 0x0000_00F0) >> 4) as u8;
let tag = ((event & Self::TAG_MASK) >> Self::TAG_SHIFT) as u8;
match tag {
0 => {
let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
let n_input_tokens = ((event & 0x0000_FF00) >> 8) as u8;
Self::TOKEN_EVENT => {
let kind: SyntaxKind =
(((event & Self::KIND_MASK) >> Self::KIND_SHIFT) as u16).into();
let n_input_tokens =
((event & Self::N_INPUT_TOKEN_MASK) >> Self::N_INPUT_TOKEN_SHIFT) as u8;
Step::Token { kind, n_input_tokens }
}
1 => {
let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
Self::ENTER_EVENT => {
let kind: SyntaxKind =
(((event & Self::KIND_MASK) >> Self::KIND_SHIFT) as u16).into();
Step::Enter { kind }
}
2 => Step::Exit,
Self::EXIT_EVENT => Step::Exit,
Self::SPLIT_EVENT => {
Step::FloatSplit { ends_in_dot: event & Self::N_INPUT_TOKEN_MASK != 0 }
}
_ => unreachable!(),
}
})
}
pub(crate) fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
let e = ((kind as u16 as u32) << 16) | ((n_tokens as u32) << 8) | 1;
let e = ((kind as u16 as u32) << Self::KIND_SHIFT)
| ((n_tokens as u32) << Self::N_INPUT_TOKEN_SHIFT)
| Self::EVENT_MASK;
self.event.push(e)
}
pub(crate) fn float_split_hack(&mut self, ends_in_dot: bool) {
let e = (Self::SPLIT_EVENT as u32) << Self::TAG_SHIFT
| ((ends_in_dot as u32) << Self::N_INPUT_TOKEN_SHIFT)
| Self::EVENT_MASK;
self.event.push(e);
}
pub(crate) fn enter_node(&mut self, kind: SyntaxKind) {
let e = ((kind as u16 as u32) << 16) | (1 << 4) | 1;
let e = ((kind as u16 as u32) << Self::KIND_SHIFT)
| ((Self::ENTER_EVENT as u32) << Self::TAG_SHIFT)
| Self::EVENT_MASK;
self.event.push(e)
}
pub(crate) fn leave_node(&mut self) {
let e = 2 << 4 | 1;
let e = (Self::EXIT_EVENT as u32) << Self::TAG_SHIFT | Self::EVENT_MASK;
self.event.push(e)
}
pub(crate) fn error(&mut self, error: String) {
let idx = self.error.len();
self.error.push(error);
let e = (idx as u32) << 1;
let e = (idx as u32) << Self::ERROR_SHIFT;
self.event.push(e);
}
}

29
crates/parser/src/parser.rs
View File

@ -181,6 +181,35 @@ impl<'t> Parser<'t> {
self.do_bump(kind, 1);
}
/// Advances the parser by one token
pub(crate) fn split_float(&mut self, mut marker: Marker) -> (bool, Marker) {
assert!(self.at(SyntaxKind::FLOAT_NUMBER));
// we have parse `<something>.`
// `<something>`.0.1
// here we need to insert an extra event
//
// `<something>`. 0. 1;
// here we need to change the follow up parse, the return value will cause us to emulate a dot
// the actual splitting happens later
let ends_in_dot = !self.inp.is_joint(self.pos);
if !ends_in_dot {
let new_marker = self.start();
let idx = marker.pos as usize;
match &mut self.events[idx] {
Event::Start { forward_parent, kind } => {
*kind = SyntaxKind::FIELD_EXPR;
*forward_parent = Some(new_marker.pos - marker.pos);
}
_ => unreachable!(),
}
marker.bomb.defuse();
marker = new_marker;
};
self.pos += 1 as usize;
self.push_event(Event::FloatSplitHack { ends_in_dot });
(ends_in_dot, marker)
}
/// Advances the parser by one token, remapping its kind.
/// This is useful to create contextual keywords from
/// identifiers. For example, the lexer creates a `union`

53
crates/parser/src/shortcuts.rs
View File

@ -43,7 +43,16 @@ impl<'a> LexedStr<'a> {
res.was_joint();
}
res.push(kind);
// Tag the token as joint if it is float with a fractional part
// we use this jointness to inform the parser about what token split
// event to emit when we encounter a float literal in a field access
if kind == SyntaxKind::FLOAT_NUMBER {
if !self.text(i).ends_with('.') {
res.was_joint();
}
}
}
was_joint = true;
}
}
@ -63,6 +72,9 @@ impl<'a> LexedStr<'a> {
Step::Token { kind, n_input_tokens: n_raw_tokens } => {
builder.token(kind, n_raw_tokens)
}
Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
builder.float_split(has_pseudo_dot)
}
Step::Enter { kind } => builder.enter(kind),
Step::Exit => builder.exit(),
Step::Error { msg } => {
@ -109,6 +121,16 @@ impl Builder<'_, '_> {
self.do_token(kind, n_tokens as usize);
}
fn float_split(&mut self, has_pseudo_dot: bool) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingEnter => unreachable!(),
State::PendingExit => (self.sink)(StrStep::Exit),
State::Normal => (),
}
self.eat_trivias();
self.do_float_split(has_pseudo_dot);
}
fn enter(&mut self, kind: SyntaxKind) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingEnter => {
@ -164,6 +186,37 @@ impl Builder<'_, '_> {
self.pos += n_tokens;
(self.sink)(StrStep::Token { kind, text });
}
fn do_float_split(&mut self, has_pseudo_dot: bool) {
let text = &self.lexed.range_text(self.pos..self.pos + 1);
self.pos += 1;
match text.split_once('.') {
Some((left, right)) => {
assert!(!left.is_empty());
(self.sink)(StrStep::Enter { kind: SyntaxKind::NAME_REF });
(self.sink)(StrStep::Token { kind: SyntaxKind::INT_NUMBER, text: left });
(self.sink)(StrStep::Exit);
// here we move the exit up, the original exit has been deleted in process
(self.sink)(StrStep::Exit);
(self.sink)(StrStep::Token { kind: SyntaxKind::DOT, text: "." });
if has_pseudo_dot {
assert!(right.is_empty(), "{left}.{right}");
self.state = State::Normal;
} else {
(self.sink)(StrStep::Enter { kind: SyntaxKind::NAME_REF });
(self.sink)(StrStep::Token { kind: SyntaxKind::INT_NUMBER, text: right });
(self.sink)(StrStep::Exit);
// the parser creates an unbalanced start node, we are required to close it here
self.state = State::PendingExit;
}
}
None => unreachable!(),
}
}
}
fn n_attached_trivias<'a>(

4
crates/parser/src/tests/prefix_entries.rs
View File

@ -51,6 +51,9 @@ fn expr() {
check(PrefixEntryPoint::Expr, "-1", "-1");
check(PrefixEntryPoint::Expr, "fn foo() {}", "fn");
check(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()");
check(PrefixEntryPoint::Expr, "foo.0", "foo.0");
check(PrefixEntryPoint::Expr, "foo.0.1", "foo.0.1");
check(PrefixEntryPoint::Expr, "foo.0. foo", "foo.0. foo");
}
#[test]
@ -88,6 +91,7 @@ fn check(entry: PrefixEntryPoint, input: &str, prefix: &str) {
for step in entry.parse(&input).iter() {
match step {
Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
Step::FloatSplit { .. } => n_tokens += 1,
Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
}
}

33
crates/parser/test_data/parser/inline/ok/0011_field_expr.rast
View File

@ -40,6 +40,39 @@ SOURCE_FILE
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
NAME_REF
INT_NUMBER "1"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
FIELD_EXPR

2
crates/parser/test_data/parser/inline/ok/0011_field_expr.rs
View File

@ -1,5 +1,7 @@
fn foo() {
x.foo;
x.0.bar;
x.0.1;
x.0. bar;
x.0();
}

43
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View File

@ -58,6 +58,49 @@ SOURCE_FILE
COMMA ","
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
NAME_REF
IDENT "call"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "call"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

2
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View File

@ -1,4 +1,6 @@
fn foo() {
x.foo();
y.bar::<T>(1, 2,);
x.0.0.call();
x.0. call();
}

35
crates/parser/test_data/parser/inline/ok/0137_await_expr.rast
View File

@ -65,6 +65,41 @@ SOURCE_FILE
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
AWAIT_EXPR
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
AWAIT_KW "await"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
AWAIT_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
INT_NUMBER "0"
DOT "."
WHITESPACE " "
AWAIT_KW "await"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

2
crates/parser/test_data/parser/inline/ok/0137_await_expr.rs
View File

@ -2,4 +2,6 @@ fn foo() {
x.await;
x.0.await;
x.0().await?.hello();
x.0.0.await;
x.0. await;
}

21
crates/tt/src/buffer.rs
View File

@ -7,7 +7,12 @@ use crate::{Leaf, Subtree, TokenTree};
struct EntryId(usize);
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
struct EntryPtr(EntryId, usize);
struct EntryPtr(
/// The index of the buffer containing the entry.
EntryId,
/// The index of the entry within the buffer.
usize,
);
/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
@ -16,8 +21,8 @@ enum Entry<'t, Span> {
// Mimicking types from proc-macro.
Subtree(Option<&'t TokenTree<Span>>, &'t Subtree<Span>, EntryId),
Leaf(&'t TokenTree<Span>),
// End entries contain a pointer to the entry from the containing
// token tree, or None if this is the outermost level.
/// End entries contain a pointer to the entry from the containing
/// token tree, or [`None`] if this is the outermost level.
End(Option<EntryPtr>),
}
@ -226,8 +231,14 @@ impl<'a, Span> Cursor<'a, Span> {
/// a cursor into that subtree
pub fn bump_subtree(self) -> Cursor<'a, Span> {
match self.entry() {
Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
_ => self.bump(),
Some(&Entry::Subtree(_, _, entry_id)) => {
Cursor::create(self.buffer, EntryPtr(entry_id, 0))
}
Some(Entry::End(exit)) => match exit {
Some(exit) => Cursor::create(self.buffer, *exit),
None => self,
},
_ => Cursor::create(self.buffer, EntryPtr(self.ptr.0, self.ptr.1 + 1)),
}
}