diff --git a/grammar.ron b/grammar.ron index 0717b7a76945..c2fcc44f5be3 100644 --- a/grammar.ron +++ b/grammar.ron @@ -91,6 +91,8 @@ Grammar( "USE_ITEM", "STATIC_ITEM", "CONST_ITEM", + "TRAIT_ITEM", + "IMPL_ITEM", "EXTERN_BLOCK", "ENUM_VARIANT", diff --git a/src/parser/event.rs b/src/parser/event.rs new file mode 100644 index 000000000000..fd6bdc0863c7 --- /dev/null +++ b/src/parser/event.rs @@ -0,0 +1,140 @@ +use {File, FileBuilder, Sink, SyntaxKind, Token}; +use syntax_kinds::TOMBSTONE; +use super::is_insignificant; + +/// `Parser` produces a flat list of `Event`s. +/// They are converted to a tree-structure in +/// a separate pass, via `TreeBuilder`. +#[derive(Debug)] +pub(crate) enum Event { + /// This event signifies the start of the node. + /// It should be either abandoned (in which case the + /// `kind` is `TOMBSTONE`, and the event is ignored), + /// or completed via a `Finish` event. + /// + /// All tokens between a `Start` and a `Finish` would + /// become the children of the respective node. + /// + /// For left-recursive syntactic constructs, the parser produces + /// a child node before it sees a parent. `forward_parent` + /// exists to allow to tweak parent-child relationships. + /// + /// Consider this path + /// + /// foo::bar + /// + /// The events for it would look like this: + /// + /// + /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH + /// | /\ + /// | | + /// +------forward-parent------+ + /// + /// And the tree would look like this + /// + /// +--PATH---------+ + /// | | | + /// | | | + /// | '::' 'bar' + /// | + /// PATH + /// | + /// 'foo' + /// + /// See also `CompleteMarker::precede`. + Start { + kind: SyntaxKind, + forward_parent: Option, + }, + + /// Complete the previous `Start` event + Finish, + + /// Produce a single leaf-element. + /// `n_raw_tokens` is used to glue complex contextual tokens. + /// For example, lexer tokenizes `>>` as `>`, `>`, and + /// `n_raw_tokens = 2` is used to produced a single `>>`. + Token { + kind: SyntaxKind, + n_raw_tokens: u8, + }, + + Error { + message: String, + }, +} + +pub(super) fn to_file(text: String, tokens: &[Token], events: Vec) -> File { + let mut builder = FileBuilder::new(text); + let mut idx = 0; + + let mut holes = Vec::new(); + let mut forward_parents = Vec::new(); + + for (i, event) in events.iter().enumerate() { + if holes.last() == Some(&i) { + holes.pop(); + continue; + } + + match event { + &Event::Start { + kind: TOMBSTONE, .. + } => (), + + &Event::Start { .. } => { + forward_parents.clear(); + let mut idx = i; + loop { + let (kind, fwd) = match events[idx] { + Event::Start { + kind, + forward_parent, + } => (kind, forward_parent), + _ => unreachable!(), + }; + forward_parents.push((idx, kind)); + if let Some(fwd) = fwd { + idx += fwd as usize; + } else { + break; + } + } + for &(idx, kind) in forward_parents.iter().into_iter().rev() { + builder.start_internal(kind); + holes.push(idx); + } + holes.pop(); + } + &Event::Finish => { + while idx < tokens.len() { + let token = tokens[idx]; + if is_insignificant(token.kind) { + idx += 1; + builder.leaf(token.kind, token.len); + } else { + break; + } + } + builder.finish_internal() + } + &Event::Token { + kind: _, + mut n_raw_tokens, + } => loop { + let token = tokens[idx]; + if !is_insignificant(token.kind) { + n_raw_tokens -= 1; + } + idx += 1; + builder.leaf(token.kind, token.len); + if n_raw_tokens == 0 { + break; + } + }, + &Event::Error { ref message } => builder.error().message(message.clone()).emit(), + } + } + builder.finish() +} diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs deleted file mode 100644 index 7823c476c200..000000000000 --- a/src/parser/event_parser/mod.rs +++ /dev/null @@ -1,74 +0,0 @@ -use {SyntaxKind, Token}; - -#[macro_use] -mod parser; -mod grammar; - -/// `Parser` produces a flat list of `Event`s. -/// They are converted to a tree-structure in -/// a separate pass, via `TreeBuilder`. -#[derive(Debug)] -pub(crate) enum Event { - /// This event signifies the start of the node. - /// It should be either abandoned (in which case the - /// `kind` is `TOMBSTONE`, and the event is ignored), - /// or completed via a `Finish` event. - /// - /// All tokens between a `Start` and a `Finish` would - /// become the children of the respective node. - /// - /// For left-recursive syntactic constructs, the parser produces - /// a child node before it sees a parent. `forward_parent` - /// exists to allow to tweak parent-child relationships. - /// - /// Consider this path - /// - /// foo::bar - /// - /// The events for it would look like this: - /// - /// - /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH - /// | /\ - /// | | - /// +------forward-parent------+ - /// - /// And the tree would look like this - /// - /// +--PATH---------+ - /// | | | - /// | | | - /// | '::' 'bar' - /// | - /// PATH - /// | - /// 'foo' - /// - /// See also `CompleteMarker::precede`. - Start { - kind: SyntaxKind, - forward_parent: Option, - }, - - /// Complete the previous `Start` event - Finish, - - /// Produce a single leaf-element. - /// `n_raw_tokens` is used to glue complex contextual tokens. - /// For example, lexer tokenizes `>>` as `>`, `>`, and - /// `n_raw_tokens = 2` is used to produced a single `>>`. - Token { - kind: SyntaxKind, - n_raw_tokens: u8, - }, - - Error { - message: String, - }, -} - -pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec { - let mut parser = parser::Parser::new(text, raw_tokens); - grammar::file(&mut parser); - parser.into_events() -} diff --git a/src/parser/event_parser/grammar/attributes.rs b/src/parser/grammar/attributes.rs similarity index 100% rename from src/parser/event_parser/grammar/attributes.rs rename to src/parser/grammar/attributes.rs diff --git a/src/parser/event_parser/grammar/expressions.rs b/src/parser/grammar/expressions.rs similarity index 100% rename from src/parser/event_parser/grammar/expressions.rs rename to src/parser/grammar/expressions.rs diff --git a/src/parser/event_parser/grammar/items/consts.rs b/src/parser/grammar/items/consts.rs similarity index 100% rename from src/parser/event_parser/grammar/items/consts.rs rename to src/parser/grammar/items/consts.rs diff --git a/src/parser/event_parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs similarity index 90% rename from src/parser/event_parser/grammar/items/mod.rs rename to src/parser/grammar/items/mod.rs index 6d6fabbd7944..3612802e17b9 100644 --- a/src/parser/event_parser/grammar/items/mod.rs +++ b/src/parser/grammar/items/mod.rs @@ -3,6 +3,7 @@ use super::*; mod structs; mod use_item; mod consts; +mod traits; pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) { attributes::inner_attributes(p); @@ -80,6 +81,22 @@ fn item(p: &mut Parser) { CONST_ITEM } }, + // TODO: auto trait + // test unsafe_trait + // unsafe trait T {} + UNSAFE_KW if la == TRAIT_KW => { + p.bump(); + traits::trait_item(p); + TRAIT_ITEM + } + // TODO: default impl + // test unsafe_impl + // unsafe impl Foo {} + UNSAFE_KW if la == IMPL_KW => { + p.bump(); + traits::impl_item(p); + IMPL_ITEM + } MOD_KW => { mod_item(p); MOD_ITEM @@ -131,6 +148,7 @@ fn extern_block(p: &mut Parser) { p.bump(); p.expect(R_CURLY); } + fn mod_item(p: &mut Parser) { assert!(p.at(MOD_KW)); p.bump(); diff --git a/src/parser/event_parser/grammar/items/structs.rs b/src/parser/grammar/items/structs.rs similarity index 100% rename from src/parser/event_parser/grammar/items/structs.rs rename to src/parser/grammar/items/structs.rs diff --git a/src/parser/grammar/items/traits.rs b/src/parser/grammar/items/traits.rs new file mode 100644 index 000000000000..3bef9639f029 --- /dev/null +++ b/src/parser/grammar/items/traits.rs @@ -0,0 +1,17 @@ +use super::*; + +pub(super) fn trait_item(p: &mut Parser) { + assert!(p.at(TRAIT_KW)); + p.bump(); + p.expect(IDENT); + p.expect(L_CURLY); + p.expect(R_CURLY); +} + +pub(super) fn impl_item(p: &mut Parser) { + assert!(p.at(IMPL_KW)); + p.bump(); + p.expect(IDENT); + p.expect(L_CURLY); + p.expect(R_CURLY); +} diff --git a/src/parser/event_parser/grammar/items/use_item.rs b/src/parser/grammar/items/use_item.rs similarity index 100% rename from src/parser/event_parser/grammar/items/use_item.rs rename to src/parser/grammar/items/use_item.rs diff --git a/src/parser/event_parser/grammar/mod.rs b/src/parser/grammar/mod.rs similarity index 100% rename from src/parser/event_parser/grammar/mod.rs rename to src/parser/grammar/mod.rs diff --git a/src/parser/event_parser/grammar/paths.rs b/src/parser/grammar/paths.rs similarity index 100% rename from src/parser/event_parser/grammar/paths.rs rename to src/parser/grammar/paths.rs diff --git a/src/parser/event_parser/grammar/type_params.rs b/src/parser/grammar/type_params.rs similarity index 100% rename from src/parser/event_parser/grammar/type_params.rs rename to src/parser/grammar/type_params.rs diff --git a/src/parser/event_parser/grammar/types.rs b/src/parser/grammar/types.rs similarity index 100% rename from src/parser/event_parser/grammar/types.rs rename to src/parser/grammar/types.rs diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c5525ff9cdcd..f17ffbf3aa39 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,88 +1,21 @@ -use {File, FileBuilder, Sink, SyntaxKind, Token}; +use {File, SyntaxKind, Token}; use syntax_kinds::*; -mod event_parser; -use self::event_parser::Event; +#[macro_use] +mod parser; +mod event; +mod grammar; +use self::event::Event; /// Parse a sequence of tokens into the representative node tree pub fn parse(text: String, tokens: &[Token]) -> File { - let events = event_parser::parse(&text, tokens); - from_events_to_file(text, tokens, events) -} - -fn from_events_to_file(text: String, tokens: &[Token], events: Vec) -> File { - let mut builder = FileBuilder::new(text); - let mut idx = 0; - - let mut holes = Vec::new(); - let mut forward_parents = Vec::new(); - - for (i, event) in events.iter().enumerate() { - if holes.last() == Some(&i) { - holes.pop(); - continue; - } - - match event { - &Event::Start { - kind: TOMBSTONE, .. - } => (), - - &Event::Start { .. } => { - forward_parents.clear(); - let mut idx = i; - loop { - let (kind, fwd) = match events[idx] { - Event::Start { - kind, - forward_parent, - } => (kind, forward_parent), - _ => unreachable!(), - }; - forward_parents.push((idx, kind)); - if let Some(fwd) = fwd { - idx += fwd as usize; - } else { - break; - } - } - for &(idx, kind) in forward_parents.iter().into_iter().rev() { - builder.start_internal(kind); - holes.push(idx); - } - holes.pop(); - } - &Event::Finish => { - while idx < tokens.len() { - let token = tokens[idx]; - if is_insignificant(token.kind) { - idx += 1; - builder.leaf(token.kind, token.len); - } else { - break; - } - } - builder.finish_internal() - } - &Event::Token { - kind: _, - mut n_raw_tokens, - } => loop { - let token = tokens[idx]; - if !is_insignificant(token.kind) { - n_raw_tokens -= 1; - } - idx += 1; - builder.leaf(token.kind, token.len); - if n_raw_tokens == 0 { - break; - } - }, - &Event::Error { ref message } => builder.error().message(message.clone()).emit(), - } - } - builder.finish() + let events = { + let mut parser = parser::Parser::new(&text, tokens); + grammar::file(&mut parser); + parser.into_events() + }; + event::to_file(text, tokens, events) } fn is_insignificant(kind: SyntaxKind) -> bool { diff --git a/src/parser/event_parser/parser.rs b/src/parser/parser.rs similarity index 99% rename from src/parser/event_parser/parser.rs rename to src/parser/parser.rs index 6cf6ac9b9d24..3f4c8a07d0c1 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/parser.rs @@ -1,6 +1,6 @@ use {SyntaxKind, TextUnit, Token}; use super::Event; -use super::super::is_insignificant; +use super::is_insignificant; use SyntaxKind::{EOF, TOMBSTONE}; pub(crate) struct Marker { diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs index c182aea78fd2..22c615831556 100644 --- a/src/syntax_kinds.rs +++ b/src/syntax_kinds.rs @@ -92,6 +92,8 @@ pub enum SyntaxKind { USE_ITEM, STATIC_ITEM, CONST_ITEM, + TRAIT_ITEM, + IMPL_ITEM, EXTERN_BLOCK, ENUM_VARIANT, NAMED_FIELD, @@ -207,6 +209,8 @@ impl SyntaxKind { USE_ITEM => &SyntaxInfo { name: "USE_ITEM" }, STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" }, CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" }, + TRAIT_ITEM => &SyntaxInfo { name: "TRAIT_ITEM" }, + IMPL_ITEM => &SyntaxInfo { name: "IMPL_ITEM" }, EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" }, ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" }, NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" }, diff --git a/tests/data/parser/inline/0007_unsafe_trait.rs b/tests/data/parser/inline/0007_unsafe_trait.rs new file mode 100644 index 000000000000..04e021550d88 --- /dev/null +++ b/tests/data/parser/inline/0007_unsafe_trait.rs @@ -0,0 +1 @@ +unsafe trait T {} diff --git a/tests/data/parser/inline/0007_unsafe_trait.txt b/tests/data/parser/inline/0007_unsafe_trait.txt new file mode 100644 index 000000000000..d6f6a4cfa7ba --- /dev/null +++ b/tests/data/parser/inline/0007_unsafe_trait.txt @@ -0,0 +1,11 @@ +FILE@[0; 18) + TRAIT_ITEM@[0; 18) + UNSAFE_KW@[0; 6) + WHITESPACE@[6; 7) + TRAIT_KW@[7; 12) + WHITESPACE@[12; 13) + IDENT@[13; 14) "T" + WHITESPACE@[14; 15) + L_CURLY@[15; 16) + R_CURLY@[16; 17) + WHITESPACE@[17; 18) diff --git a/tests/data/parser/inline/0008_unsafe_impl.rs b/tests/data/parser/inline/0008_unsafe_impl.rs new file mode 100644 index 000000000000..41055f41d965 --- /dev/null +++ b/tests/data/parser/inline/0008_unsafe_impl.rs @@ -0,0 +1 @@ +unsafe impl Foo {} diff --git a/tests/data/parser/inline/0008_unsafe_impl.txt b/tests/data/parser/inline/0008_unsafe_impl.txt new file mode 100644 index 000000000000..a88a447cb308 --- /dev/null +++ b/tests/data/parser/inline/0008_unsafe_impl.txt @@ -0,0 +1,11 @@ +FILE@[0; 19) + IMPL_ITEM@[0; 19) + UNSAFE_KW@[0; 6) + WHITESPACE@[6; 7) + IMPL_KW@[7; 11) + WHITESPACE@[11; 12) + IDENT@[12; 15) "Foo" + WHITESPACE@[15; 16) + L_CURLY@[16; 17) + R_CURLY@[17; 18) + WHITESPACE@[18; 19) diff --git a/tools/src/bin/collect-tests.rs b/tools/src/bin/collect-tests.rs index df9d2db81583..a52e7b1193ca 100644 --- a/tools/src/bin/collect-tests.rs +++ b/tools/src/bin/collect-tests.rs @@ -79,16 +79,19 @@ fn collect_tests(s: &str) -> Vec { .map(str::trim_left) .group_by(|line| line.starts_with(prefix)); - for (is_comment, block) in comment_blocks.into_iter() { + 'outer: for (is_comment, block) in comment_blocks.into_iter() { if !is_comment { continue; } let mut block = block.map(|line| &line[prefix.len()..]); - let first = block.next().unwrap(); - if !first.starts_with("test ") { - continue; - } - let name = first["test ".len()..].to_string(); + + let name = loop { + match block.next() { + Some(line) if line.starts_with("test ") => break line["test ".len()..].to_string(), + Some(_) => (), + None => continue 'outer, + } + }; let text: String = itertools::join(block.chain(::std::iter::once("")), "\n"); assert!(!text.trim().is_empty() && text.ends_with("\n")); res.push(Test { name, text }) @@ -121,7 +124,7 @@ fn inline_tests_dir() -> PathBuf { } fn grammar_dir() -> PathBuf { - base_dir().join("src/parser/event_parser/grammar") + base_dir().join("src/parser/grammar") } fn base_dir() -> PathBuf {