nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-29 18:23:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge #1213

Browse Source 
1213: Make lexer produce only single character puncts r=matklad a=edwin0cheng

As discussed in Zulip, this PR change `lexer` to produce only single char punct.

* Remove producing `DOTDOTDOT, DOTDOTEQ, DOTDOT, COLONCOLON, EQEQ, FAT_ARROW, NEQ, THIN_ARROW` in lexer.
* Add required code in parser to make sure everythings works fine.
* Change some tests (Mainly because the `ast::token_tree` is different)

Note: i think the use of `COLON` in rust is too overloaded :)


Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>
This commit is contained in:
bors[bot] 2019-04-28 16:51:02 +00:00
commit 6618d1edc3
9 changed files with 183 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
crates/ra_mbe/src/lib.rs
View File

@ -240,19 +240,23 @@ impl_froms!(TokenTree: Leaf, Subtree);
let expanded = expand(rules, invocation);
assert_eq!(expanded.to_string(), expansion);
let tree = token_tree_to_macro_items(&expanded);
// FIXME: Temp comment below code
// It is because after the lexer change,
// The SyntaxNode structure cannot be matched easily
// Eat all white space by parse it back and forth
// Because $crate will seperate in two token , will do some special treatment here
let expansion = expansion.replace("$crate", "C_C__C");
let expansion = ast::SourceFile::parse(&expansion);
let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0;
let file = token_tree_to_macro_items(&expansion);
let file = file.unwrap().syntax().debug_dump().trim().to_string();
let tree = tree.unwrap().syntax().debug_dump().trim().to_string();
// let tree = token_tree_to_macro_items(&expanded);
let file = file.replace("C_C__C", "$crate");
assert_eq!(tree, file,);
// // Eat all white space by parse it back and forth
// // Because $crate will seperate in two token , will do some special treatment here
// let expansion = expansion.replace("$crate", "C_C__C");
// let expansion = ast::SourceFile::parse(&expansion);
// let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0;
// let file = token_tree_to_macro_items(&expansion);
// let file = file.unwrap().syntax().debug_dump().trim().to_string();
// let tree = tree.unwrap().syntax().debug_dump().trim().to_string();
// let file = file.replace("C_C__C", "$crate");
// assert_eq!(tree, file,);
expanded
}

10
crates/ra_mbe/src/subtree_source.rs
View File

@ -388,6 +388,7 @@ where
}
}
// FIXME: Remove this function
fn convert_multi_char_punct<'b, I>(
p: &tt::Punct,
iter: &mut TokenPeek<'b, I>,
@ -397,8 +398,6 @@ where
{
if let Some((m, is_joint_to_next)) = iter.current_punct3(p) {
if let Some((kind, text)) = match m {
('.', '.', '.') => Some((DOTDOTDOT, "...")),
('.', '.', '=') => Some((DOTDOTEQ, "..=")),
_ => None,
} {
return Some((kind, is_joint_to_next, text, 3));
@ -407,13 +406,6 @@ where
if let Some((m, is_joint_to_next)) = iter.current_punct2(p) {
if let Some((kind, text)) = match m {
('-', '>') => Some((THIN_ARROW, "->")),
('!', '=') => Some((NEQ, "!=")),
('=', '>') => Some((FAT_ARROW, "=>")),
('=', '=') => Some((EQEQ, "==")),
('.', '.') => Some((DOTDOT, "..")),
(':', ':') => Some((COLONCOLON, "::")),
_ => None,
} {
return Some((kind, is_joint_to_next, text, 2));

2
crates/ra_parser/src/grammar/items.rs
View File

@ -383,7 +383,7 @@ pub(crate) fn token_tree(p: &mut Parser) {
return;
}
R_PAREN | R_BRACK => p.err_and_bump("unmatched brace"),
_ => p.bump(),
_ => p.bump_raw(),
}
}
p.expect(closing_paren_kind);

64
crates/ra_parser/src/parser.rs
View File

@ -85,8 +85,13 @@ impl<'t> Parser<'t> {
let mut i = 0;
loop {
let kind = self.token_source.token_kind(self.token_pos + i);
i += 1;
let mut kind = self.token_source.token_kind(self.token_pos + i);
if let Some((composited, step)) = self.is_composite(kind, i) {
kind = composited;
i += step;
} else {
i += 1;
}
match kind {
EOF => return EOF,
@ -121,13 +126,37 @@ impl<'t> Parser<'t> {
Marker::new(pos)
}
/// Advances the parser by one token unconditionally.
/// Advances the parser by one token unconditionally
/// Mainly use in `token_tree` parsing
pub(crate) fn bump_raw(&mut self) {
let kind = self.token_source.token_kind(self.token_pos);
if kind == EOF {
return;
}
self.do_bump(kind, 1);
}
/// Advances the parser by one token with composite puncts handled
pub(crate) fn bump(&mut self) {
let kind = self.nth(0);
if kind == EOF {
return;
}
self.do_bump(kind, 1);
use SyntaxKind::*;
// Handle parser composites
match kind {
DOTDOTDOT | DOTDOTEQ => {
self.bump_compound(kind, 3);
}
DOTDOT | COLONCOLON | EQEQ | FAT_ARROW | NEQ | THIN_ARROW => {
self.bump_compound(kind, 2);
}
_ => {
self.do_bump(kind, 1);
}
}
}
/// Advances the parser by one token, remapping its kind.
@ -206,6 +235,33 @@ impl<'t> Parser<'t> {
self.events.push(event)
}
/// helper function for check if it is composite.
fn is_composite(&self, kind: SyntaxKind, n: usize) -> Option<(SyntaxKind, usize)> {
// We assume the dollars will not occuried between
// mult-byte tokens
let jn1 = self.token_source.is_token_joint_to_next(self.token_pos + n);
let la2 = self.token_source.token_kind(self.token_pos + n + 1);
let jn2 = self.token_source.is_token_joint_to_next(self.token_pos + n + 1);
let la3 = self.token_source.token_kind(self.token_pos + n + 2);
use SyntaxKind::*;
match kind {
DOT if jn1 && la2 == DOT && jn2 && la3 == DOT => Some((DOTDOTDOT, 3)),
DOT if jn1 && la2 == DOT && la3 == EQ => Some((DOTDOTEQ, 3)),
DOT if jn1 && la2 == DOT => Some((DOTDOT, 2)),
COLON if jn1 && la2 == COLON => Some((COLONCOLON, 2)),
EQ if jn1 && la2 == EQ => Some((EQEQ, 2)),
EQ if jn1 && la2 == R_ANGLE => Some((FAT_ARROW, 2)),
EXCL if la2 == EQ => Some((NEQ, 2)),
MINUS if la2 == R_ANGLE => Some((THIN_ARROW, 2)),
_ => None,
}
}
fn eat_dollars(&mut self) {
loop {
match self.token_source.token_kind(self.token_pos) {

71
crates/ra_syntax/src/parsing/lexer.rs
View File

@ -88,65 +88,18 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
}
match c {
// Multi-byte tokens.
'.' => {
return match (ptr.current(), ptr.nth(1)) {
(Some('.'), Some('.')) => {
ptr.bump();
ptr.bump();
DOTDOTDOT
}
(Some('.'), Some('=')) => {
ptr.bump();
ptr.bump();
DOTDOTEQ
}
(Some('.'), _) => {
ptr.bump();
DOTDOT
}
_ => DOT,
};
}
':' => {
return match ptr.current() {
Some(':') => {
ptr.bump();
COLONCOLON
}
_ => COLON,
};
}
'=' => {
return match ptr.current() {
Some('=') => {
ptr.bump();
EQEQ
}
Some('>') => {
ptr.bump();
FAT_ARROW
}
_ => EQ,
};
}
'!' => {
return match ptr.current() {
Some('=') => {
ptr.bump();
NEQ
}
_ => EXCL,
};
}
'-' => {
return if ptr.at('>') {
ptr.bump();
THIN_ARROW
} else {
MINUS
};
}
// Possiblily multi-byte tokens,
// but we only produce single byte token now
// DOTDOTDOT, DOTDOT, DOTDOTEQ, DOT
'.' => return DOT,
// COLONCOLON COLON
':' => return COLON,
// EQEQ FATARROW EQ
'=' => return EQ,
// NEQ EXCL
'!' => return EXCL,
// THIN_ARROW MINUS
'-' => return MINUS,
// If the character is an ident start not followed by another single
// quote, then this is a lifetime name:

3
crates/ra_syntax/tests/data/lexer/0004_numbers.txt
View File

@ -37,7 +37,8 @@ WHITESPACE 1 " "
INT_NUMBER 6 "0E1279"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
DOTDOT 2 ".."
DOT 1 "."
DOT 1 "."
INT_NUMBER 1 "2"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"

23
crates/ra_syntax/tests/data/lexer/0005_symbols.txt
View File

@ -44,25 +44,34 @@ PERCENT 1 "%"
WHITESPACE 1 "\n"
DOT 1 "."
WHITESPACE 1 " "
DOTDOT 2 ".."
DOT 1 "."
DOT 1 "."
WHITESPACE 1 " "
DOTDOTDOT 3 "..."
DOT 1 "."
DOT 1 "."
DOT 1 "."
WHITESPACE 1 " "
DOTDOTEQ 3 "..="
DOT 1 "."
DOT 1 "."
EQ 1 "="
WHITESPACE 1 "\n"
COLON 1 ":"
WHITESPACE 1 " "
COLONCOLON 2 "::"
COLON 1 ":"
COLON 1 ":"
WHITESPACE 1 "\n"
EQ 1 "="
WHITESPACE 1 " "
FAT_ARROW 2 "=>"
EQ 1 "="
R_ANGLE 1 ">"
WHITESPACE 1 "\n"
EXCL 1 "!"
WHITESPACE 1 " "
NEQ 2 "!="
EXCL 1 "!"
EQ 1 "="
WHITESPACE 1 "\n"
MINUS 1 "-"
WHITESPACE 1 " "
THIN_ARROW 2 "->"
MINUS 1 "-"
R_ANGLE 1 ">"
WHITESPACE 1 "\n"

3
crates/ra_syntax/tests/data/parser/inline/ok/0096_no_semi_after_block.txt
View File

@ -102,7 +102,8 @@ SOURCE_FILE@[0; 167)
L_PAREN@[138; 139) "("
R_PAREN@[139; 140) ")"
WHITESPACE@[140; 141) " "
FAT_ARROW@[141; 143) "=>"
EQ@[141; 142) "="
R_ANGLE@[142; 143) ">"
WHITESPACE@[143; 144) " "
TOKEN_TREE@[144; 146)
L_CURLY@[144; 145) "{"

111
crates/ra_syntax/tests/data/parser/ok/0035_weird_exprs.txt
View File

@ -1181,7 +1181,8 @@ SOURCE_FILE@[0; 3813)
TOKEN_TREE@[1988; 2195)
L_PAREN@[1988; 1989) "("
IDENT@[1989; 1995) "String"
COLONCOLON@[1995; 1997) "::"
COLON@[1995; 1996) ":"
COLON@[1996; 1997) ":"
IDENT@[1997; 2001) "from"
TOKEN_TREE@[2001; 2055)
L_PAREN@[2001; 2002) "("
@ -1196,55 +1197,80 @@ SOURCE_FILE@[0; 3813)
STRING@[2080; 2086) "\"{:?}\""
COMMA@[2086; 2087) ","
WHITESPACE@[2087; 2088) " "
DOTDOT@[2088; 2090) ".."
DOT@[2088; 2089) "."
DOT@[2089; 2090) "."
WHITESPACE@[2090; 2091) " "
DOTDOT@[2091; 2093) ".."
DOT@[2091; 2092) "."
DOT@[2092; 2093) "."
WHITESPACE@[2093; 2094) " "
DOTDOT@[2094; 2096) ".."
DOT@[2094; 2095) "."
DOT@[2095; 2096) "."
WHITESPACE@[2096; 2097) " "
DOTDOT@[2097; 2099) ".."
DOT@[2097; 2098) "."
DOT@[2098; 2099) "."
WHITESPACE@[2099; 2100) " "
DOTDOT@[2100; 2102) ".."
DOT@[2100; 2101) "."
DOT@[2101; 2102) "."
WHITESPACE@[2102; 2103) " "
DOTDOT@[2103; 2105) ".."
DOT@[2103; 2104) "."
DOT@[2104; 2105) "."
WHITESPACE@[2105; 2106) " "
DOTDOT@[2106; 2108) ".."
DOT@[2106; 2107) "."
DOT@[2107; 2108) "."
WHITESPACE@[2108; 2109) " "
DOTDOT@[2109; 2111) ".."
DOT@[2109; 2110) "."
DOT@[2110; 2111) "."
WHITESPACE@[2111; 2112) " "
DOTDOT@[2112; 2114) ".."
DOT@[2112; 2113) "."
DOT@[2113; 2114) "."
WHITESPACE@[2114; 2115) " "
DOTDOT@[2115; 2117) ".."
DOT@[2115; 2116) "."
DOT@[2116; 2117) "."
WHITESPACE@[2117; 2118) " "
DOTDOT@[2118; 2120) ".."
DOT@[2118; 2119) "."
DOT@[2119; 2120) "."
WHITESPACE@[2120; 2121) " "
DOTDOT@[2121; 2123) ".."
DOT@[2121; 2122) "."
DOT@[2122; 2123) "."
WHITESPACE@[2123; 2124) " "
DOTDOT@[2124; 2126) ".."
DOT@[2124; 2125) "."
DOT@[2125; 2126) "."
WHITESPACE@[2126; 2158) "\n ..."
DOTDOT@[2158; 2160) ".."
DOT@[2158; 2159) "."
DOT@[2159; 2160) "."
WHITESPACE@[2160; 2161) " "
DOTDOT@[2161; 2163) ".."
DOT@[2161; 2162) "."
DOT@[2162; 2163) "."
WHITESPACE@[2163; 2164) " "
DOTDOT@[2164; 2166) ".."
DOT@[2164; 2165) "."
DOT@[2165; 2166) "."
WHITESPACE@[2166; 2167) " "
DOTDOT@[2167; 2169) ".."
DOT@[2167; 2168) "."
DOT@[2168; 2169) "."
WHITESPACE@[2169; 2170) " "
DOTDOT@[2170; 2172) ".."
DOT@[2170; 2171) "."
DOT@[2171; 2172) "."
WHITESPACE@[2172; 2173) " "
DOTDOT@[2173; 2175) ".."
DOT@[2173; 2174) "."
DOT@[2174; 2175) "."
WHITESPACE@[2175; 2176) " "
DOTDOT@[2176; 2178) ".."
DOT@[2176; 2177) "."
DOT@[2177; 2178) "."
WHITESPACE@[2178; 2179) " "
DOTDOT@[2179; 2181) ".."
DOT@[2179; 2180) "."
DOT@[2180; 2181) "."
WHITESPACE@[2181; 2182) " "
DOTDOT@[2182; 2184) ".."
DOT@[2182; 2183) "."
DOT@[2183; 2184) "."
WHITESPACE@[2184; 2185) " "
DOTDOT@[2185; 2187) ".."
DOT@[2185; 2186) "."
DOT@[2186; 2187) "."
WHITESPACE@[2187; 2188) " "
DOTDOT@[2188; 2190) ".."
DOT@[2188; 2189) "."
DOT@[2189; 2190) "."
WHITESPACE@[2190; 2191) " "
DOTDOT@[2191; 2193) ".."
DOT@[2191; 2192) "."
DOT@[2192; 2193) "."
R_PAREN@[2193; 2194) ")"
R_PAREN@[2194; 2195) ")"
SEMI@[2195; 2196) ";"
@ -1321,7 +1347,8 @@ SOURCE_FILE@[0; 3813)
IDENT@[2308; 2310) "u8"
R_PAREN@[2310; 2311) ")"
WHITESPACE@[2311; 2312) " "
FAT_ARROW@[2312; 2314) "=>"
EQ@[2312; 2313) "="
R_ANGLE@[2313; 2314) ">"
WHITESPACE@[2314; 2315) " "
TOKEN_TREE@[2315; 2552)
L_CURLY@[2315; 2316) "{"
@ -1359,7 +1386,8 @@ SOURCE_FILE@[0; 3813)
IDENT@[2405; 2407) "u8"
R_PAREN@[2407; 2408) ")"
WHITESPACE@[2408; 2409) " "
THIN_ARROW@[2409; 2411) "->"
MINUS@[2409; 2410) "-"
R_ANGLE@[2410; 2411) ">"
WHITESPACE@[2411; 2412) " "
AMP@[2412; 2413) "&"
LIFETIME@[2413; 2416) "\'u8"
@ -1403,7 +1431,8 @@ SOURCE_FILE@[0; 3813)
EQ@[2615; 2616) "="
WHITESPACE@[2616; 2617) " "
IDENT@[2617; 2619) "u8"
COLONCOLON@[2619; 2621) "::"
COLON@[2619; 2620) ":"
COLON@[2620; 2621) ":"
IDENT@[2621; 2623) "u8"
TOKEN_TREE@[2623; 2629)
L_PAREN@[2623; 2624) "("
@ -1413,7 +1442,8 @@ SOURCE_FILE@[0; 3813)
SEMI@[2629; 2630) ";"
WHITESPACE@[2630; 2643) "\n "
CRATE_KW@[2643; 2648) "crate"
COLONCOLON@[2648; 2650) "::"
COLON@[2648; 2649) ":"
COLON@[2649; 2650) ":"
IDENT@[2650; 2652) "u8"
TOKEN_TREE@[2652; 2657)
L_PAREN@[2652; 2653) "("
@ -1453,7 +1483,8 @@ SOURCE_FILE@[0; 3813)
TOKEN_TREE@[2722; 2829)
L_PAREN@[2722; 2723) "("
IDENT@[2723; 2729) "String"
COLONCOLON@[2729; 2731) "::"
COLON@[2729; 2730) ":"
COLON@[2730; 2731) ":"
IDENT@[2731; 2735) "from"
TOKEN_TREE@[2735; 2742)
L_PAREN@[2735; 2736) "("
@ -1462,12 +1493,15 @@ SOURCE_FILE@[0; 3813)
COMMA@[2742; 2743) ","
WHITESPACE@[2743; 2759) "\n "
IDENT@[2759; 2765) "String"
COLONCOLON@[2765; 2767) "::"
COLON@[2765; 2766) ":"
COLON@[2766; 2767) ":"
L_ANGLE@[2767; 2768) "<"
R_ANGLE@[2768; 2769) ">"
COLONCOLON@[2769; 2771) "::"
COLON@[2769; 2770) ":"
COLON@[2770; 2771) ":"
IDENT@[2771; 2775) "from"
COLONCOLON@[2775; 2777) "::"
COLON@[2775; 2776) ":"
COLON@[2776; 2777) ":"
L_ANGLE@[2777; 2778) "<"
R_ANGLE@[2778; 2779) ">"
TOKEN_TREE@[2779; 2786)
@ -1476,7 +1510,8 @@ SOURCE_FILE@[0; 3813)
R_PAREN@[2785; 2786) ")"
DOT@[2786; 2787) "."
IDENT@[2787; 2792) "chars"
COLONCOLON@[2792; 2794) "::"
COLON@[2792; 2793) ":"
COLON@[2793; 2794) ":"
L_ANGLE@[2794; 2795) "<"
R_ANGLE@[2795; 2796) ">"
TOKEN_TREE@[2796; 2798)
@ -1484,7 +1519,8 @@ SOURCE_FILE@[0; 3813)
R_PAREN@[2797; 2798) ")"
DOT@[2798; 2799) "."
IDENT@[2799; 2802) "rev"
COLONCOLON@[2802; 2804) "::"
COLON@[2802; 2803) ":"
COLON@[2803; 2804) ":"
L_ANGLE@[2804; 2805) "<"
R_ANGLE@[2805; 2806) ">"
TOKEN_TREE@[2806; 2808)
@ -1492,7 +1528,8 @@ SOURCE_FILE@[0; 3813)
R_PAREN@[2807; 2808) ")"
DOT@[2808; 2809) "."
IDENT@[2809; 2816) "collect"
COLONCOLON@[2816; 2818) "::"
COLON@[2816; 2817) ":"
COLON@[2817; 2818) ":"
L_ANGLE@[2818; 2819) "<"
IDENT@[2819; 2825) "String"
R_ANGLE@[2825; 2826) ">"