From f4a6d92050fd1e3b3a92883f0cbde04876fe12dc Mon Sep 17 00:00:00 2001
From: Evgenii P <eupn@protonmail.com>
Date: Fri, 9 Aug 2019 14:08:34 +0700
Subject: [PATCH 1/3] Reduce code duplication in lexer by introducing small
macro
---
crates/ra_syntax/src/parsing/lexer.rs | 37 ++++++++++++---------------
1 file changed, 17 insertions(+), 20 deletions(-)
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 2a4343b0a65..926811d96fd 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -12,6 +12,21 @@ pub struct Token {
pub len: TextUnit,
}
+macro_rules! match_literal_kind {
+ ($kind:expr) => {
+ match $kind {
+ ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
+ ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
+ ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
+ ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
+ ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
+ ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
+ ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
+ ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
+ }
+ }
+}
+
/// Break a string up into its component tokens
pub fn tokenize(text: &str) -> Vec<Token> {
if text.is_empty() {
@@ -53,16 +68,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
}
}
ra_rustc_lexer::TokenKind::RawIdent => IDENT,
- ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind {
- ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
- ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
- ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
- ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
- ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
- ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
- ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
- ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
- },
+ ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME,
ra_rustc_lexer::TokenKind::Semi => SEMI,
ra_rustc_lexer::TokenKind::Comma => COMMA,
@@ -131,16 +137,7 @@ pub fn classify_literal(text: &str) -> Option<Token> {
return None;
}
let kind = match t.kind {
- ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind {
- ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
- ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
- ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
- ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
- ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
- ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
- ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
- ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
- },
+ ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
_ => return None,
};
Some(Token { kind, len: TextUnit::from_usize(t.len) })
From 073cf423916c8854e060a1127856dc4736fe83b1 Mon Sep 17 00:00:00 2001
From: Evgenii P <eupn@protonmail.com>
Date: Fri, 9 Aug 2019 14:23:13 +0700
Subject: [PATCH 2/3] rustfmt
---
crates/ra_syntax/src/parsing/lexer.rs | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 926811d96fd..7a9bc355d12 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -15,16 +15,16 @@ pub struct Token {
macro_rules! match_literal_kind {
($kind:expr) => {
match $kind {
- ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
- ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
- ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
- ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
- ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
- ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
- ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
- ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
- }
- }
+ ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
+ ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
+ ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
+ ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
+ ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
+ ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
+ ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
+ ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
+ }
+ };
}
/// Break a string up into its component tokens
From db4839033c1903f0926100692b69641b506c05d1 Mon Sep 17 00:00:00 2001
From: Evgenii P <eupn@protonmail.com>
Date: Fri, 9 Aug 2019 15:04:13 +0700
Subject: [PATCH 3/3] Change macro to function
---
crates/ra_syntax/src/parsing/lexer.rs | 28 +++++++++++++--------------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 7a9bc355d12..06822ea9175 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -12,19 +12,17 @@ pub struct Token {
pub len: TextUnit,
}
-macro_rules! match_literal_kind {
- ($kind:expr) => {
- match $kind {
- ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
- ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
- ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
- ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
- ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
- ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
- ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
- ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
- }
- };
+fn match_literal_kind(kind: ra_rustc_lexer::LiteralKind) -> SyntaxKind {
+ match kind {
+ ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
+ ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
+ ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
+ ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
+ ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
+ ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
+ ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
+ ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
+ }
}
/// Break a string up into its component tokens
@@ -68,7 +66,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
}
}
ra_rustc_lexer::TokenKind::RawIdent => IDENT,
- ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
+ ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME,
ra_rustc_lexer::TokenKind::Semi => SEMI,
ra_rustc_lexer::TokenKind::Comma => COMMA,
@@ -137,7 +135,7 @@ pub fn classify_literal(text: &str) -> Option<Token> {
return None;
}
let kind = match t.kind {
- ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
+ ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
_ => return None,
};
Some(Token { kind, len: TextUnit::from_usize(t.len) })