nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-21 22:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve fake_ident_or_unknown_prefix.

Browse Source 
- Rename it as `invalid_ident_or_prefix`, which matches the possible
  outputs (`InvalidIdent` or `InvalidPrefix`).
- Use the local wrapper for `is_xid_continue`, for consistency.
- Make it clear what `\u{200d}` means.
This commit is contained in:
Nicholas Nethercote 2024-11-19 14:48:48 +11:00
parent 03ee484519
commit df29f9b0c3
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
compiler/rustc_lexer/src/lib.rs
View File

@ -468,7 +468,7 @@ impl Cursor<'_> {
Literal { kind, suffix_start }
}
// Identifier starting with an emoji. Only lexed for graceful error recovery.
c if !c.is_ascii() && c.is_emoji_char() => self.fake_ident_or_unknown_prefix(),
c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident_or_prefix(),
_ => Unknown,
};
let res = Token::new(token_kind, self.pos_within_token());
@ -552,17 +552,16 @@ impl Cursor<'_> {
// we see a prefix here, it is definitely an unknown prefix.
match self.first() {
'#' | '"' | '\'' => UnknownPrefix,
c if !c.is_ascii() && c.is_emoji_char() => self.fake_ident_or_unknown_prefix(),
c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident_or_prefix(),
_ => Ident,
}
}
fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
fn invalid_ident_or_prefix(&mut self) -> TokenKind {
// Start is already eaten, eat the rest of identifier.
self.eat_while(|c| {
unicode_xid::UnicodeXID::is_xid_continue(c)
|| (!c.is_ascii() && c.is_emoji_char())
|| c == '\u{200d}'
const ZERO_WIDTH_JOINER: char = '\u{200d}';
is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
});
// Known prefixes must have been handled earlier. So if
// we see a prefix here, it is definitely an unknown prefix.