From 8d9961b75377a7bd2656b5aa1451710de8c86f60 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 28 Jul 2018 13:07:10 +0300 Subject: [PATCH] Migrate to text-unit --- Cargo.toml | 1 + src/lexer/ptr.rs | 4 +- src/lib.rs | 4 +- src/parser/event.rs | 8 ++- src/parser/input.rs | 4 +- src/text.rs | 136 --------------------------------------- src/tree/file_builder.rs | 6 +- src/tree/mod.rs | 8 +-- 8 files changed, 18 insertions(+), 153 deletions(-) delete mode 100644 src/text.rs diff --git a/Cargo.toml b/Cargo.toml index 622abcc42a1..954f3e94b0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "tools" ] [dependencies] unicode-xid = "0.1.0" +text_unit = "0.1.1" [dev-dependencies] testutils = { path = "./tests/testutils" } diff --git a/src/lexer/ptr.rs b/src/lexer/ptr.rs index 99d55b283c5..d1391fd5f2c 100644 --- a/src/lexer/ptr.rs +++ b/src/lexer/ptr.rs @@ -11,7 +11,7 @@ impl<'s> Ptr<'s> { pub fn new(text: &'s str) -> Ptr<'s> { Ptr { text, - len: TextUnit::new(0), + len: 0.into(), } } @@ -47,7 +47,7 @@ impl<'s> Ptr<'s> { pub fn bump(&mut self) -> Option { let ch = self.chars().next()?; - self.len += TextUnit::len_of_char(ch); + self.len += TextUnit::of_char(ch); Some(ch) } diff --git a/src/lib.rs b/src/lib.rs index 15345864452..b90b70c050b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,14 +16,14 @@ //#![warn(unreachable_pub)] // rust-lang/rust#47816 extern crate unicode_xid; +extern crate text_unit; -mod text; mod tree; mod lexer; mod parser; pub mod syntax_kinds; -pub use text::{TextRange, TextUnit}; +pub use text_unit::{TextRange, TextUnit}; pub use tree::{File, Node, SyntaxKind, Token}; pub(crate) use tree::{ErrorMsg, FileBuilder, Sink}; pub use lexer::{next_token, tokenize}; diff --git a/src/parser/event.rs b/src/parser/event.rs index 1c0905a38f3..ac8a55de974 100644 --- a/src/parser/event.rs +++ b/src/parser/event.rs @@ -1,5 +1,7 @@ -use {ErrorMsg, File, FileBuilder, Sink, SyntaxKind, TextUnit, Token}; -use syntax_kinds::TOMBSTONE; +use { + ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token, + syntax_kinds::TOMBSTONE, +}; use super::is_insignificant; /// `Parser` produces a flat list of `Event`s. @@ -133,7 +135,7 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec) -> Fil builder.leaf(token.kind, token.len); idx += 1 } - let mut len = TextUnit::new(0); + let mut len = 0.into(); for _ in 0..n_raw_tokens { len += tokens[idx].len; idx += 1; diff --git a/src/parser/input.rs b/src/parser/input.rs index 13589467b47..9b400b959b3 100644 --- a/src/parser/input.rs +++ b/src/parser/input.rs @@ -14,7 +14,7 @@ impl<'t> ParserInput<'t> { pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { let mut tokens = Vec::new(); let mut start_offsets = Vec::new(); - let mut len = TextUnit::new(0); + let mut len = 0.into(); for &token in raw_tokens.iter() { if !is_insignificant(token.kind) { tokens.push(token); @@ -44,7 +44,7 @@ impl<'t> ParserInput<'t> { if !(idx < self.tokens.len()) { return ""; } - let range = TextRange::from_len(self.start_offsets[idx], self.tokens[idx].len); + let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len); &self.text[range] } } diff --git a/src/text.rs b/src/text.rs deleted file mode 100644 index 4084bf44e8f..00000000000 --- a/src/text.rs +++ /dev/null @@ -1,136 +0,0 @@ -use std::fmt; -use std::ops; - -/// An text position in a source file -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TextUnit(u32); - -impl TextUnit { - /// The positional offset required for one character - pub fn len_of_char(c: char) -> TextUnit { - TextUnit(c.len_utf8() as u32) - } - - #[allow(missing_docs)] - pub fn new(val: u32) -> TextUnit { - TextUnit(val) - } -} - -impl fmt::Debug for TextUnit { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - ::fmt(self, f) - } -} - -impl fmt::Display for TextUnit { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl From for u32 { - fn from(tu: TextUnit) -> u32 { - tu.0 - } -} - -impl From for TextUnit { - fn from(tu: u32) -> TextUnit { - TextUnit::new(tu) - } -} - -impl ops::Add for TextUnit { - type Output = TextUnit; - fn add(self, rhs: TextUnit) -> TextUnit { - TextUnit(self.0 + rhs.0) - } -} - -impl ops::AddAssign for TextUnit { - fn add_assign(&mut self, rhs: TextUnit) { - self.0 += rhs.0 - } -} - -impl ops::Sub for TextUnit { - type Output = TextUnit; - fn sub(self, rhs: TextUnit) -> TextUnit { - TextUnit(self.0 - rhs.0) - } -} - -impl ops::SubAssign for TextUnit { - fn sub_assign(&mut self, rhs: TextUnit) { - self.0 -= rhs.0 - } -} - -/// A range of text in a source file -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct TextRange { - start: TextUnit, - end: TextUnit, -} - -impl fmt::Debug for TextRange { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - ::fmt(self, f) - } -} - -impl fmt::Display for TextRange { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "[{}; {})", self.start(), self.end()) - } -} - -impl TextRange { - /// An length-0 range of text - pub fn empty() -> TextRange { - TextRange::from_to(TextUnit::new(0), TextUnit::new(0)) - } - - /// The left-inclusive range (`[from..to)`) between to points in the text - pub fn from_to(from: TextUnit, to: TextUnit) -> TextRange { - assert!(from <= to, "Invalid text range [{}; {})", from, to); - TextRange { - start: from, - end: to, - } - } - - /// The range from some point over some length - pub fn from_len(from: TextUnit, len: TextUnit) -> TextRange { - TextRange::from_to(from, from + len) - } - - /// The starting position of this range - pub fn start(&self) -> TextUnit { - self.start - } - - /// The end position of this range - pub fn end(&self) -> TextUnit { - self.end - } - - /// The length of this range - pub fn len(&self) -> TextUnit { - self.end - self.start - } - - /// Is this range empty of any content? - pub fn is_empty(&self) -> bool { - self.start() == self.end() - } -} - -impl ops::Index for str { - type Output = str; - - fn index(&self, index: TextRange) -> &str { - &self[index.start().0 as usize..index.end().0 as usize] - } -} diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs index f831676c739..71260216840 100644 --- a/src/tree/file_builder.rs +++ b/src/tree/file_builder.rs @@ -31,7 +31,7 @@ impl Sink for FileBuilder { fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) { let leaf = NodeData { kind, - range: TextRange::from_len(self.pos, len), + range: TextRange::offset_len(self.pos, len), parent: None, first_child: None, next_sibling: None, @@ -44,7 +44,7 @@ impl Sink for FileBuilder { fn start_internal(&mut self, kind: SyntaxKind) { let node = NodeData { kind, - range: TextRange::from_len(self.pos, 0.into()), + range: TextRange::offset_len(self.pos, 0.into()), parent: None, first_child: None, next_sibling: None, @@ -83,7 +83,7 @@ impl FileBuilder { nodes: Vec::new(), errors: Vec::new(), in_progress: Vec::new(), - pos: TextUnit::new(0), + pos: 0.into(), } } diff --git a/src/tree/mod.rs b/src/tree/mod.rs index ebf26777be2..f7b16d7b527 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -1,9 +1,7 @@ -use text::{TextRange, TextUnit}; - -use std::fmt; -use std::cmp; - mod file_builder; + +use ::{TextRange, TextUnit}; +use std::{fmt, cmp}; pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink}; pub use syntax_kinds::SyntaxKind;