nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-12-03 12:13:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
Aleksey Kladov 2020-10-07 11:49:31 +02:00
parent faddea9353
commit fd8622e1ec
2 changed files with 22 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
crates/mbe/src/subtree_source.rs
View File

@ -2,7 +2,7 @@
use parser::{Token, TokenSource};
use std::cell::{Cell, Ref, RefCell};
use syntax::{tokenize, SmolStr, SyntaxKind, SyntaxKind::*, T};
use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
use tt::buffer::{Cursor, TokenBuffer};
#[derive(Debug, Clone, Eq, PartialEq)]
@ -155,16 +155,14 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
}
fn convert_literal(l: &tt::Literal) -> TtToken {
let mut kinds = tokenize(&l.text).0.into_iter().map(|token| token.kind);
let is_negated = l.text.starts_with('-');
let inner_text = &l.text[if is_negated { 1 } else { 0 }..];
let kind = match kinds.next() {
Some(kind) if kind.is_literal() => Some(kind),
Some(SyntaxKind::MINUS) => match kinds.next() {
Some(kind) if kind.is_literal() => Some(kind),
_ => None,
},
_ => None,
}
let kind = lex_single_syntax_kind(inner_text)
.map(|(kind, _error)| kind)
.filter(|kind| {
kind.is_literal() && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
})
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
TtToken { kind, is_joint_to_next: false, text: l.text.clone() }

23
crates/syntax/src/parsing/lexer.rs
View File

@ -1,10 +1,10 @@
//! Lexer analyzes raw input string and produces lexemes (tokens).
//! It is just a bridge to `rustc_lexer`.
use rustc_lexer::{LiteralKind as LK, RawStrError};
use std::convert::TryInto;
use rustc_lexer::{LiteralKind as LK, RawStrError};
use crate::{
SyntaxError,
SyntaxKind::{self, *},
@ -61,17 +61,18 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
(tokens, errors)
}
/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
/// encountered at the beginning of the string.
/// Returns `SyntaxKind` and `Option<SyntaxError>` if `text` parses as a single token.
///
/// Returns `None` if the string contains zero *or two or more* tokens.
/// The token is malformed if the returned error is not `None`.
///
/// Beware that unescape errors are not checked at tokenization time.
pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
lex_first_token(text)
.filter(|(token, _)| token.len == TextSize::of(text))
.map(|(token, error)| (token.kind, error))
let (first_token, err) = lex_first_token(text)?;
if first_token.len != TextSize::of(text) {
return None;
}
Some((first_token.kind, err))
}
/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and
@ -79,9 +80,11 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr
///
/// Beware that unescape errors are not checked at tokenization time.
pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
lex_first_token(text)
.filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text))
.map(|(token, _error)| token.kind)
let (single_token, err) = lex_single_syntax_kind(text)?;
if err.is_some() {
return None;
}
Some(single_token)
}
/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token