nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-28 09:44:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lexer: symbols

Browse Source
This commit is contained in:
Aleksey Kladov 2017-12-30 16:30:37 +03:00
parent 3e91e8b77d
commit fad3e50987
6 changed files with 144 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
grammar.ron
View File

@ -6,5 +6,24 @@ Grammar(
"WHITESPACE",
"INT_NUMBER",
"FLOAT_NUMBER",
"SEMI",
"COMMA",
"DOT",
"DOTDOT",
"DOTDOTDOT",
"DOTDOTEQ",
"L_PAREN",
"R_PAREN",
"L_CURLY",
"R_CURLY",
"L_BRACK",
"R_BRACK",
"AT",
"POUND",
"TILDE",
"QUESTION",
"COLON",
"COLONCOLON",
"DOLLAR",
]
)

41
src/lexer/mod.rs
View File

@ -37,6 +37,47 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
return scan_number(c, ptr);
}
// One-byte tokens.
match c {
';' => return SEMI,
',' => return COMMA,
'(' => return L_PAREN,
')' => return R_PAREN,
'{' => return L_CURLY,
'}' => return R_CURLY,
'[' => return L_BRACK,
']' => return R_BRACK,
'@' => return AT,
'#' => return POUND,
'~' => return TILDE,
'?' => return QUESTION,
'$' => return DOLLAR,
'.' => return match (ptr.next(), ptr.nnext()) {
(Some('.'), Some('.')) => {
ptr.bump();
ptr.bump();
DOTDOTDOT
},
(Some('.'), Some('=')) => {
ptr.bump();
ptr.bump();
DOTDOTEQ
},
(Some('.'), _) => {
ptr.bump();
DOTDOT
},
_ => DOT
},
':' => return match ptr.next() {
Some(':') => {
ptr.bump();
COLONCOLON
}
_ => COLON
},
_ => (),
}
ERROR
}

40
src/syntax_kinds.rs
View File

@ -7,14 +7,52 @@ pub const UNDERSCORE: SyntaxKind = SyntaxKind(2);
pub const WHITESPACE: SyntaxKind = SyntaxKind(3);
pub const INT_NUMBER: SyntaxKind = SyntaxKind(4);
pub const FLOAT_NUMBER: SyntaxKind = SyntaxKind(5);
pub const SEMI: SyntaxKind = SyntaxKind(6);
pub const COMMA: SyntaxKind = SyntaxKind(7);
pub const DOT: SyntaxKind = SyntaxKind(8);
pub const DOTDOT: SyntaxKind = SyntaxKind(9);
pub const DOTDOTDOT: SyntaxKind = SyntaxKind(10);
pub const DOTDOTEQ: SyntaxKind = SyntaxKind(11);
pub const L_PAREN: SyntaxKind = SyntaxKind(12);
pub const R_PAREN: SyntaxKind = SyntaxKind(13);
pub const L_CURLY: SyntaxKind = SyntaxKind(14);
pub const R_CURLY: SyntaxKind = SyntaxKind(15);
pub const L_BRACK: SyntaxKind = SyntaxKind(16);
pub const R_BRACK: SyntaxKind = SyntaxKind(17);
pub const AT: SyntaxKind = SyntaxKind(18);
pub const POUND: SyntaxKind = SyntaxKind(19);
pub const TILDE: SyntaxKind = SyntaxKind(20);
pub const QUESTION: SyntaxKind = SyntaxKind(21);
pub const COLON: SyntaxKind = SyntaxKind(22);
pub const COLONCOLON: SyntaxKind = SyntaxKind(23);
pub const DOLLAR: SyntaxKind = SyntaxKind(24);
static INFOS: [SyntaxInfo; 6] = [
static INFOS: [SyntaxInfo; 25] = [
SyntaxInfo { name: "ERROR" },
SyntaxInfo { name: "IDENT" },
SyntaxInfo { name: "UNDERSCORE" },
SyntaxInfo { name: "WHITESPACE" },
SyntaxInfo { name: "INT_NUMBER" },
SyntaxInfo { name: "FLOAT_NUMBER" },
SyntaxInfo { name: "SEMI" },
SyntaxInfo { name: "COMMA" },
SyntaxInfo { name: "DOT" },
SyntaxInfo { name: "DOTDOT" },
SyntaxInfo { name: "DOTDOTDOT" },
SyntaxInfo { name: "DOTDOTEQ" },
SyntaxInfo { name: "L_PAREN" },
SyntaxInfo { name: "R_PAREN" },
SyntaxInfo { name: "L_CURLY" },
SyntaxInfo { name: "R_CURLY" },
SyntaxInfo { name: "L_BRACK" },
SyntaxInfo { name: "R_BRACK" },
SyntaxInfo { name: "AT" },
SyntaxInfo { name: "POUND" },
SyntaxInfo { name: "TILDE" },
SyntaxInfo { name: "QUESTION" },
SyntaxInfo { name: "COLON" },
SyntaxInfo { name: "COLONCOLON" },
SyntaxInfo { name: "DOLLAR" },
];
pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {

11
tests/data/lexer/0004_number.txt
View File

@ -38,22 +38,21 @@ WHITESPACE 1 " "
INT_NUMBER 6 "0E1279"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
ERROR 1 "."
ERROR 1 "."
DOTDOT 2 ".."
INT_NUMBER 1 "2"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
ERROR 1 "."
DOT 1 "."
IDENT 3 "foo"
ERROR 1 "("
ERROR 1 ")"
L_PAREN 1 "("
R_PAREN 1 ")"
WHITESPACE 1 "\n"
INT_NUMBER 2 "0e"
ERROR 1 "+"
INT_NUMBER 1 "1"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
ERROR 1 "."
DOT 1 "."
IDENT 1 "e"
ERROR 1 "+"
INT_NUMBER 1 "1"

3
tests/data/lexer/0005_symbols.rs Normal file
View File

@ -0,0 +1,3 @@
; , ( ) { } [ ] @ # ~ ? $
. .. ... ..=
: ::

37
tests/data/lexer/0005_symbols.txt Normal file
View File

@ -0,0 +1,37 @@
SEMI 1 ";"
WHITESPACE 1 " "
COMMA 1 ","
WHITESPACE 1 " "
L_PAREN 1 "("
WHITESPACE 1 " "
R_PAREN 1 ")"
WHITESPACE 1 " "
L_CURLY 1 "{"
WHITESPACE 1 " "
R_CURLY 1 "}"
WHITESPACE 1 " "
L_BRACK 1 "["
WHITESPACE 1 " "
R_BRACK 1 "]"
WHITESPACE 1 " "
AT 1 "@"
WHITESPACE 1 " "
POUND 1 "#"
WHITESPACE 1 " "
TILDE 1 "~"
WHITESPACE 1 " "
QUESTION 1 "?"
WHITESPACE 1 " "
DOLLAR 1 "$"
WHITESPACE 1 "\n"
DOT 1 "."
WHITESPACE 1 " "
DOTDOT 2 ".."
WHITESPACE 1 " "
DOTDOTDOT 3 "..."
WHITESPACE 1 " "
DOTDOTEQ 3 "..="
WHITESPACE 1 "\n"
COLON 1 ":"
WHITESPACE 1 " "
COLONCOLON 2 "::"