nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 00:03:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve TokenKind comments.

Browse Source 
- Improve wording.
- Use backticks consistently for examples.
This commit is contained in:
Nicholas Nethercote 2024-11-19 15:11:18 +11:00
parent df29f9b0c3
commit 2c7c3697db
1 changed files with 39 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
compiler/rustc_lexer/src/lib.rs
View File

@ -57,11 +57,10 @@ impl Token {
/// Enum representing common lexeme types.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TokenKind {
// Multi-char tokens:
/// "// comment"
/// A line comment, e.g. `// comment`.
LineComment { doc_style: Option<DocStyle> },
/// `/* block comment */`
/// A block comment, e.g. `/* block comment */`.
///
/// Block comments can be recursive, so a sequence like `/* /* */`
/// will not be considered terminated and will result in a parsing error.
@ -70,18 +69,17 @@ pub enum TokenKind {
/// Any whitespace character sequence.
Whitespace,
/// "ident" or "continue"
///
/// At this step, keywords are also considered identifiers.
/// An identifier or keyword, e.g. `ident` or `continue`.
Ident,
/// Like the above, but containing invalid unicode codepoints.
/// An identifier that is invalid because it contains emoji.
InvalidIdent,
/// "r#ident"
/// A raw identifier, e.g. "r#ident".
RawIdent,
/// An unknown prefix, like `foo#`, `foo'`, `foo"`.
/// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
/// literal prefixes that contain emoji, which are considered "invalid".
///
/// Note that only the
/// prefix (`foo`) is included in the token, not the separator (which is
@ -93,11 +91,12 @@ pub enum TokenKind {
/// An unknown prefix in a lifetime, like `'foo#`.
///
/// Note that like above, only the `'` and prefix are included in the token
/// Like `UnknownPrefix`, only the `'` and prefix are included in the token
/// and not the separator.
UnknownPrefixLifetime,
/// `'r#lt`, which in edition < 2021 is split into several tokens: `'r # lt`.
/// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
/// several tokens: `'r` and `#` and `foo`.
RawLifetime,
/// Similar to the above, but *always* an error on every edition. This is used
@ -110,70 +109,69 @@ pub enum TokenKind {
/// Split into the component tokens on older editions.
GuardedStrPrefix,
/// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
/// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
/// suffix, but may be present here on string and float literals. Users of
/// this type will need to check for and reject that case.
///
/// See [LiteralKind] for more details.
Literal { kind: LiteralKind, suffix_start: u32 },
/// "'a"
/// A lifetime, e.g. `'a`.
Lifetime { starts_with_number: bool },
// One-char tokens:
/// ";"
/// `;`
Semi,
/// ","
/// `,`
Comma,
/// "."
/// `.`
Dot,
/// "("
/// `(`
OpenParen,
/// ")"
/// `)`
CloseParen,
/// "{"
/// `{`
OpenBrace,
/// "}"
/// `}`
CloseBrace,
/// "["
/// `[`
OpenBracket,
/// "]"
/// `]`
CloseBracket,
/// "@"
/// `@`
At,
/// "#"
/// `#`
Pound,
/// "~"
/// `~`
Tilde,
/// "?"
/// `?`
Question,
/// ":"
/// `:`
Colon,
/// "$"
/// `$`
Dollar,
/// "="
/// `=`
Eq,
/// "!"
/// `!`
Bang,
/// "<"
/// `<`
Lt,
/// ">"
/// `>`
Gt,
/// "-"
/// `-`
Minus,
/// "&"
/// `&`
And,
/// "|"
/// `|`
Or,
/// "+"
/// `+`
Plus,
/// "*"
/// `*`
Star,
/// "/"
/// `/`
Slash,
/// "^"
/// `^`
Caret,
/// "%"
/// `%`
Percent,
/// Unknown token, not expected by the lexer, e.g. "№"