nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2025-06-05 03:38:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge #41

Browse Source 
41: G: unsafe impl & trait r=matklad a=matklad

bors r+
This commit is contained in:
bors[bot] 2018-02-04 11:04:02 +00:00
commit aa36ad008e
22 changed files with 228 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
grammar.ron
View File

@ -91,6 +91,8 @@ Grammar(
"USE_ITEM",
"STATIC_ITEM",
"CONST_ITEM",
"TRAIT_ITEM",
"IMPL_ITEM",
"EXTERN_BLOCK",
"ENUM_VARIANT",

140
src/parser/event.rs Normal file
View File

@ -0,0 +1,140 @@
use {File, FileBuilder, Sink, SyntaxKind, Token};
use syntax_kinds::TOMBSTONE;
use super::is_insignificant;
/// `Parser` produces a flat list of `Event`s.
/// They are converted to a tree-structure in
/// a separate pass, via `TreeBuilder`.
#[derive(Debug)]
pub(crate) enum Event {
/// This event signifies the start of the node.
/// It should be either abandoned (in which case the
/// `kind` is `TOMBSTONE`, and the event is ignored),
/// or completed via a `Finish` event.
///
/// All tokens between a `Start` and a `Finish` would
/// become the children of the respective node.
///
/// For left-recursive syntactic constructs, the parser produces
/// a child node before it sees a parent. `forward_parent`
/// exists to allow to tweak parent-child relationships.
///
/// Consider this path
///
/// foo::bar
///
/// The events for it would look like this:
///
///
/// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
/// | /\
/// | |
/// +------forward-parent------+
///
/// And the tree would look like this
///
/// +--PATH---------+
/// | | |
/// | | |
/// | '::' 'bar'
/// |
/// PATH
/// |
/// 'foo'
///
/// See also `CompleteMarker::precede`.
Start {
kind: SyntaxKind,
forward_parent: Option<u32>,
},
/// Complete the previous `Start` event
Finish,
/// Produce a single leaf-element.
/// `n_raw_tokens` is used to glue complex contextual tokens.
/// For example, lexer tokenizes `>>` as `>`, `>`, and
/// `n_raw_tokens = 2` is used to produced a single `>>`.
Token {
kind: SyntaxKind,
n_raw_tokens: u8,
},
Error {
message: String,
},
}
pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
let mut builder = FileBuilder::new(text);
let mut idx = 0;
let mut holes = Vec::new();
let mut forward_parents = Vec::new();
for (i, event) in events.iter().enumerate() {
if holes.last() == Some(&i) {
holes.pop();
continue;
}
match event {
&Event::Start {
kind: TOMBSTONE, ..
} => (),
&Event::Start { .. } => {
forward_parents.clear();
let mut idx = i;
loop {
let (kind, fwd) = match events[idx] {
Event::Start {
kind,
forward_parent,
} => (kind, forward_parent),
_ => unreachable!(),
};
forward_parents.push((idx, kind));
if let Some(fwd) = fwd {
idx += fwd as usize;
} else {
break;
}
}
for &(idx, kind) in forward_parents.iter().into_iter().rev() {
builder.start_internal(kind);
holes.push(idx);
}
holes.pop();
}
&Event::Finish => {
while idx < tokens.len() {
let token = tokens[idx];
if is_insignificant(token.kind) {
idx += 1;
builder.leaf(token.kind, token.len);
} else {
break;
}
}
builder.finish_internal()
}
&Event::Token {
kind: _,
mut n_raw_tokens,
} => loop {
let token = tokens[idx];
if !is_insignificant(token.kind) {
n_raw_tokens -= 1;
}
idx += 1;
builder.leaf(token.kind, token.len);
if n_raw_tokens == 0 {
break;
}
},
&Event::Error { ref message } => builder.error().message(message.clone()).emit(),
}
}
builder.finish()
}

74
src/parser/event_parser/mod.rs
View File

@ -1,74 +0,0 @@
use {SyntaxKind, Token};
#[macro_use]
mod parser;
mod grammar;
/// `Parser` produces a flat list of `Event`s.
/// They are converted to a tree-structure in
/// a separate pass, via `TreeBuilder`.
#[derive(Debug)]
pub(crate) enum Event {
/// This event signifies the start of the node.
/// It should be either abandoned (in which case the
/// `kind` is `TOMBSTONE`, and the event is ignored),
/// or completed via a `Finish` event.
///
/// All tokens between a `Start` and a `Finish` would
/// become the children of the respective node.
///
/// For left-recursive syntactic constructs, the parser produces
/// a child node before it sees a parent. `forward_parent`
/// exists to allow to tweak parent-child relationships.
///
/// Consider this path
///
/// foo::bar
///
/// The events for it would look like this:
///
///
/// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
/// | /\
/// | |
/// +------forward-parent------+
///
/// And the tree would look like this
///
/// +--PATH---------+
/// | | |
/// | | |
/// | '::' 'bar'
/// |
/// PATH
/// |
/// 'foo'
///
/// See also `CompleteMarker::precede`.
Start {
kind: SyntaxKind,
forward_parent: Option<u32>,
},
/// Complete the previous `Start` event
Finish,
/// Produce a single leaf-element.
/// `n_raw_tokens` is used to glue complex contextual tokens.
/// For example, lexer tokenizes `>>` as `>`, `>`, and
/// `n_raw_tokens = 2` is used to produced a single `>>`.
Token {
kind: SyntaxKind,
n_raw_tokens: u8,
},
Error {
message: String,
},
}
pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
let mut parser = parser::Parser::new(text, raw_tokens);
grammar::file(&mut parser);
parser.into_events()
}

0
src/parser/event_parser/grammar/attributes.rs → src/parser/grammar/attributes.rs
View File

0
src/parser/event_parser/grammar/expressions.rs → src/parser/grammar/expressions.rs
View File

0
src/parser/event_parser/grammar/items/consts.rs → src/parser/grammar/items/consts.rs
View File

18
src/parser/event_parser/grammar/items/mod.rs → src/parser/grammar/items/mod.rs
View File

@ -3,6 +3,7 @@ use super::*;
mod structs;
mod use_item;
mod consts;
mod traits;
pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
attributes::inner_attributes(p);
@ -80,6 +81,22 @@ fn item(p: &mut Parser) {
CONST_ITEM
}
},
// TODO: auto trait
// test unsafe_trait
// unsafe trait T {}
UNSAFE_KW if la == TRAIT_KW => {
p.bump();
traits::trait_item(p);
TRAIT_ITEM
}
// TODO: default impl
// test unsafe_impl
// unsafe impl Foo {}
UNSAFE_KW if la == IMPL_KW => {
p.bump();
traits::impl_item(p);
IMPL_ITEM
}
MOD_KW => {
mod_item(p);
MOD_ITEM
@ -131,6 +148,7 @@ fn extern_block(p: &mut Parser) {
p.bump();
p.expect(R_CURLY);
}
fn mod_item(p: &mut Parser) {
assert!(p.at(MOD_KW));
p.bump();

0
src/parser/event_parser/grammar/items/structs.rs → src/parser/grammar/items/structs.rs
View File

17
src/parser/grammar/items/traits.rs Normal file
View File

@ -0,0 +1,17 @@
use super::*;
pub(super) fn trait_item(p: &mut Parser) {
assert!(p.at(TRAIT_KW));
p.bump();
p.expect(IDENT);
p.expect(L_CURLY);
p.expect(R_CURLY);
}
pub(super) fn impl_item(p: &mut Parser) {
assert!(p.at(IMPL_KW));
p.bump();
p.expect(IDENT);
p.expect(L_CURLY);
p.expect(R_CURLY);
}

0
src/parser/event_parser/grammar/items/use_item.rs → src/parser/grammar/items/use_item.rs
View File

0
src/parser/event_parser/grammar/mod.rs → src/parser/grammar/mod.rs
View File

0
src/parser/event_parser/grammar/paths.rs → src/parser/grammar/paths.rs
View File

0
src/parser/event_parser/grammar/type_params.rs → src/parser/grammar/type_params.rs
View File

0
src/parser/event_parser/grammar/types.rs → src/parser/grammar/types.rs
View File

91
src/parser/mod.rs
View File

@ -1,88 +1,21 @@
use {File, FileBuilder, Sink, SyntaxKind, Token};
use {File, SyntaxKind, Token};
use syntax_kinds::*;
mod event_parser;
use self::event_parser::Event;
#[macro_use]
mod parser;
mod event;
mod grammar;
use self::event::Event;
/// Parse a sequence of tokens into the representative node tree
pub fn parse(text: String, tokens: &[Token]) -> File {
let events = event_parser::parse(&text, tokens);
from_events_to_file(text, tokens, events)
}
fn from_events_to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
let mut builder = FileBuilder::new(text);
let mut idx = 0;
let mut holes = Vec::new();
let mut forward_parents = Vec::new();
for (i, event) in events.iter().enumerate() {
if holes.last() == Some(&i) {
holes.pop();
continue;
}
match event {
&Event::Start {
kind: TOMBSTONE, ..
} => (),
&Event::Start { .. } => {
forward_parents.clear();
let mut idx = i;
loop {
let (kind, fwd) = match events[idx] {
Event::Start {
kind,
forward_parent,
} => (kind, forward_parent),
_ => unreachable!(),
};
forward_parents.push((idx, kind));
if let Some(fwd) = fwd {
idx += fwd as usize;
} else {
break;
}
}
for &(idx, kind) in forward_parents.iter().into_iter().rev() {
builder.start_internal(kind);
holes.push(idx);
}
holes.pop();
}
&Event::Finish => {
while idx < tokens.len() {
let token = tokens[idx];
if is_insignificant(token.kind) {
idx += 1;
builder.leaf(token.kind, token.len);
} else {
break;
}
}
builder.finish_internal()
}
&Event::Token {
kind: _,
mut n_raw_tokens,
} => loop {
let token = tokens[idx];
if !is_insignificant(token.kind) {
n_raw_tokens -= 1;
}
idx += 1;
builder.leaf(token.kind, token.len);
if n_raw_tokens == 0 {
break;
}
},
&Event::Error { ref message } => builder.error().message(message.clone()).emit(),
}
}
builder.finish()
let events = {
let mut parser = parser::Parser::new(&text, tokens);
grammar::file(&mut parser);
parser.into_events()
};
event::to_file(text, tokens, events)
}
fn is_insignificant(kind: SyntaxKind) -> bool {

2
src/parser/event_parser/parser.rs → src/parser/parser.rs
View File

@ -1,6 +1,6 @@
use {SyntaxKind, TextUnit, Token};
use super::Event;
use super::super::is_insignificant;
use super::is_insignificant;
use SyntaxKind::{EOF, TOMBSTONE};
pub(crate) struct Marker {

4
src/syntax_kinds.rs
View File

@ -92,6 +92,8 @@ pub enum SyntaxKind {
USE_ITEM,
STATIC_ITEM,
CONST_ITEM,
TRAIT_ITEM,
IMPL_ITEM,
EXTERN_BLOCK,
ENUM_VARIANT,
NAMED_FIELD,
@ -207,6 +209,8 @@ impl SyntaxKind {
USE_ITEM => &SyntaxInfo { name: "USE_ITEM" },
STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" },
CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" },
TRAIT_ITEM => &SyntaxInfo { name: "TRAIT_ITEM" },
IMPL_ITEM => &SyntaxInfo { name: "IMPL_ITEM" },
EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" },
ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" },
NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" },

1
tests/data/parser/inline/0007_unsafe_trait.rs Normal file
View File

@ -0,0 +1 @@
unsafe trait T {}

11
tests/data/parser/inline/0007_unsafe_trait.txt Normal file
View File

@ -0,0 +1,11 @@
FILE@[0; 18)
TRAIT_ITEM@[0; 18)
UNSAFE_KW@[0; 6)
WHITESPACE@[6; 7)
TRAIT_KW@[7; 12)
WHITESPACE@[12; 13)
IDENT@[13; 14) "T"
WHITESPACE@[14; 15)
L_CURLY@[15; 16)
R_CURLY@[16; 17)
WHITESPACE@[17; 18)

1
tests/data/parser/inline/0008_unsafe_impl.rs Normal file
View File

@ -0,0 +1 @@
unsafe impl Foo {}

11
tests/data/parser/inline/0008_unsafe_impl.txt Normal file
View File

@ -0,0 +1,11 @@
FILE@[0; 19)
IMPL_ITEM@[0; 19)
UNSAFE_KW@[0; 6)
WHITESPACE@[6; 7)
IMPL_KW@[7; 11)
WHITESPACE@[11; 12)
IDENT@[12; 15) "Foo"
WHITESPACE@[15; 16)
L_CURLY@[16; 17)
R_CURLY@[17; 18)
WHITESPACE@[18; 19)

17
tools/src/bin/collect-tests.rs
View File

@ -79,16 +79,19 @@ fn collect_tests(s: &str) -> Vec<Test> {
.map(str::trim_left)
.group_by(|line| line.starts_with(prefix));
for (is_comment, block) in comment_blocks.into_iter() {
'outer: for (is_comment, block) in comment_blocks.into_iter() {
if !is_comment {
continue;
}
let mut block = block.map(|line| &line[prefix.len()..]);
let first = block.next().unwrap();
if !first.starts_with("test ") {
continue;
}
let name = first["test ".len()..].to_string();
let name = loop {
match block.next() {
Some(line) if line.starts_with("test ") => break line["test ".len()..].to_string(),
Some(_) => (),
None => continue 'outer,
}
};
let text: String = itertools::join(block.chain(::std::iter::once("")), "\n");
assert!(!text.trim().is_empty() && text.ends_with("\n"));
res.push(Test { name, text })
@ -121,7 +124,7 @@ fn inline_tests_dir() -> PathBuf {
}
fn grammar_dir() -> PathBuf {
base_dir().join("src/parser/event_parser/grammar")
base_dir().join("src/parser/grammar")
}
fn base_dir() -> PathBuf {