nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 16:24:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Format lexer

Browse Source
This commit is contained in:
Michael Goulet 2024-08-26 11:26:36 -04:00
parent 9aaf873396
commit 3b3e43a386
2 changed files with 23 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
compiler/rustc_lexer/src/lib.rs
View File

@ -92,7 +92,7 @@ pub enum TokenKind {
UnknownPrefix, UnknownPrefix,
/// An unknown prefix in a lifetime, like `'foo#`. /// An unknown prefix in a lifetime, like `'foo#`.
/// ///
/// Note that like above, only the `'` and prefix are included in the token /// Note that like above, only the `'` and prefix are included in the token
/// and not the separator. /// and not the separator.
UnknownPrefixLifetime, UnknownPrefixLifetime,

41
compiler/rustc_parse/src/lexer/mod.rs
View File

@ -188,9 +188,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
preceded_by_whitespace = true; preceded_by_whitespace = true;
continue; continue;
} }
rustc_lexer::TokenKind::Ident => { rustc_lexer::TokenKind::Ident => self.ident(start),
self.ident(start)
}
rustc_lexer::TokenKind::RawIdent => { rustc_lexer::TokenKind::RawIdent => {
let sym = nfc_normalize(self.str_from(start + BytePos(2))); let sym = nfc_normalize(self.str_from(start + BytePos(2)));
let span = self.mk_sp(start, self.pos); let span = self.mk_sp(start, self.pos);
@ -215,20 +213,21 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
let ident = Symbol::intern(lifetime_name); let ident = Symbol::intern(lifetime_name);
token::Lifetime(ident, IdentIsRaw::No) token::Lifetime(ident, IdentIsRaw::No)
} }
rustc_lexer::TokenKind::InvalidIdent rustc_lexer::TokenKind::InvalidIdent | rustc_lexer::TokenKind::InvalidPrefix
| rustc_lexer::TokenKind::InvalidPrefix
// Do not recover an identifier with emoji if the codepoint is a confusable // Do not recover an identifier with emoji if the codepoint is a confusable
// with a recoverable substitution token, like `➖`. // with a recoverable substitution token, like `➖`.
if !UNICODE_ARRAY if !UNICODE_ARRAY.iter().any(|&(c, _, _)| {
.iter() let sym = self.str_from(start);
.any(|&(c, _, _)| { sym.chars().count() == 1 && c == sym.chars().next().unwrap()
let sym = self.str_from(start); }) =>
sym.chars().count() == 1 && c == sym.chars().next().unwrap()
}) =>
{ {
let sym = nfc_normalize(self.str_from(start)); let sym = nfc_normalize(self.str_from(start));
let span = self.mk_sp(start, self.pos); let span = self.mk_sp(start, self.pos);
self.psess.bad_unicode_identifiers.borrow_mut().entry(sym).or_default() self.psess
.bad_unicode_identifiers
.borrow_mut()
.entry(sym)
.or_default()
.push(span); .push(span);
token::Ident(sym, IdentIsRaw::No) token::Ident(sym, IdentIsRaw::No)
} }
@ -259,9 +258,9 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
let suffix = if suffix_start < self.pos { let suffix = if suffix_start < self.pos {
let string = self.str_from(suffix_start); let string = self.str_from(suffix_start);
if string == "_" { if string == "_" {
self self.dcx().emit_err(errors::UnderscoreLiteralSuffix {
.dcx() span: self.mk_sp(suffix_start, self.pos),
.emit_err(errors::UnderscoreLiteralSuffix { span: self.mk_sp(suffix_start, self.pos) }); });
None None
} else { } else {
Some(Symbol::intern(string)) Some(Symbol::intern(string))
@ -279,7 +278,8 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1))); self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1)));
if starts_with_number { if starts_with_number {
let span = self.mk_sp(start, self.pos); let span = self.mk_sp(start, self.pos);
self.dcx().struct_err("lifetimes cannot start with a number") self.dcx()
.struct_err("lifetimes cannot start with a number")
.with_span(span) .with_span(span)
.stash(span, StashKey::LifetimeIsChar); .stash(span, StashKey::LifetimeIsChar);
} }
@ -341,16 +341,19 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
// first remove compound tokens like `<<` from `rustc_lexer`, and then add // first remove compound tokens like `<<` from `rustc_lexer`, and then add
// fancier error recovery to it, as there will be less overall work to do this // fancier error recovery to it, as there will be less overall work to do this
// way. // way.
let (token, sugg) = unicode_chars::check_for_substitution(self, start, c, repeats+1); let (token, sugg) =
unicode_chars::check_for_substitution(self, start, c, repeats + 1);
self.dcx().emit_err(errors::UnknownTokenStart { self.dcx().emit_err(errors::UnknownTokenStart {
span: self.mk_sp(start, self.pos + Pos::from_usize(repeats * c.len_utf8())), span: self.mk_sp(start, self.pos + Pos::from_usize(repeats * c.len_utf8())),
escaped: escaped_char(c), escaped: escaped_char(c),
sugg, sugg,
null: if c == '\x00' {Some(errors::UnknownTokenNull)} else {None}, null: if c == '\x00' { Some(errors::UnknownTokenNull) } else { None },
repeat: if repeats > 0 { repeat: if repeats > 0 {
swallow_next_invalid = repeats; swallow_next_invalid = repeats;
Some(errors::UnknownTokenRepeat { repeats }) Some(errors::UnknownTokenRepeat { repeats })
} else {None} } else {
None
},
}); });
if let Some(token) = token { if let Some(token) = token {