diff --git a/Cargo.toml b/Cargo.toml index 109f36584..880f246f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,11 @@ pomelo = { version = "0.1.4", optional = true } thiserror = "1.0.21" serde = { version = "1.0", features = ["derive"], optional = true } petgraph = { version ="0.5", optional = true } +pp-rs = { git = "https://github.com/Kangz/glslpp-rs", rev = "4f2f72a", optional = true } [features] default = [] -glsl-in = ["pomelo"] +glsl-in = ["pomelo", "pp-rs"] glsl-validate = [] glsl-out = ["petgraph"] msl-out = [] diff --git a/src/front/glsl/lex.rs b/src/front/glsl/lex.rs index d0638c3f4..6f72c5c80 100644 --- a/src/front/glsl/lex.rs +++ b/src/front/glsl/lex.rs @@ -1,412 +1,248 @@ -use super::parser::Token; -use super::{preprocess::LinePreProcessor, token::TokenMetadata, types::parse_type}; +use super::{parser::Token, token::TokenMetadata, types::parse_type}; use crate::FastHashMap; -use std::{iter::Enumerate, str::Lines}; +use pp_rs::{ + pp::Preprocessor, + token::{Punct, Token as PPToken, TokenValue}, +}; +use std::collections::VecDeque; -fn _consume_str<'a>(input: &'a str, what: &str) -> Option<&'a str> { - if input.starts_with(what) { - Some(&input[what.len()..]) - } else { - None - } -} - -fn consume_any(input: &str, what: impl Fn(char) -> bool) -> (&str, &str, usize) { - let pos = input.find(|c| !what(c)).unwrap_or_else(|| input.len()); - let (o, i) = input.split_at(pos); - (o, i, pos) -} - -#[derive(Clone, Debug)] pub struct Lexer<'a> { - lines: Enumerate>, - input: String, - line: usize, - offset: usize, - inside_comment: bool, - pp: LinePreProcessor, + pp: Preprocessor<'a>, + tokens: VecDeque, } impl<'a> Lexer<'a> { - pub fn consume_token(&mut self) -> Option { - let start = self - .input - .find(|c: char| !c.is_whitespace()) - .unwrap_or_else(|| self.input.chars().count()); - let input = &self.input[start..]; - - let mut chars = input.chars(); - let cur = match chars.next() { - Some(c) => c, - None => { - self.input = self.input[start..].into(); - return None; - } - }; - let mut meta = TokenMetadata { - line: 0, - chars: start..start + 1, - }; - let mut consume_all = false; - let token = match cur { - ':' => Some(Token::Colon(meta)), - ';' => Some(Token::Semicolon(meta)), - ',' => Some(Token::Comma(meta)), - '.' => Some(Token::Dot(meta)), - - '(' => Some(Token::LeftParen(meta)), - ')' => Some(Token::RightParen(meta)), - '{' => Some(Token::LeftBrace(meta)), - '}' => Some(Token::RightBrace(meta)), - '[' => Some(Token::LeftBracket(meta)), - ']' => Some(Token::RightBracket(meta)), - '<' | '>' => { - let n1 = chars.next(); - let n2 = chars.next(); - match (cur, n1, n2) { - ('<', Some('<'), Some('=')) => { - meta.chars.end = start + 3; - Some(Token::LeftAssign(meta)) - } - ('>', Some('>'), Some('=')) => { - meta.chars.end = start + 3; - Some(Token::RightAssign(meta)) - } - ('<', Some('<'), _) => { - meta.chars.end = start + 2; - Some(Token::LeftOp(meta)) - } - ('>', Some('>'), _) => { - meta.chars.end = start + 2; - Some(Token::RightOp(meta)) - } - ('<', Some('='), _) => { - meta.chars.end = start + 2; - Some(Token::LeOp(meta)) - } - ('>', Some('='), _) => { - meta.chars.end = start + 2; - Some(Token::GeOp(meta)) - } - ('<', _, _) => Some(Token::LeftAngle(meta)), - ('>', _, _) => Some(Token::RightAngle(meta)), - _ => None, - } - } - '0'..='9' => { - let next = chars.next().and_then(|c| c.to_lowercase().next()); - - let hexadecimal = cur == '0' && next.map_or(false, |c| c == 'x'); - - let (number, remainder, pos) = if hexadecimal { - consume_any(&input[2..], |c| { - ('0'..='9').contains(&c) - || ('a'..='f').contains(&c) - || ('A'..='F').contains(&c) - }) - } else { - consume_any(input, |c| (('0'..='9').contains(&c) || c == '.')) - }; - - let mut remainder_chars = remainder.chars(); - let first = remainder_chars.next().and_then(|c| c.to_lowercase().next()); - - if number.find('.').is_some() { - let second = remainder_chars.next().and_then(|c| c.to_lowercase().next()); - - if (first, second) == (Some('l'), Some('f')) { - meta.chars.end = start + pos + 2; - Some(Token::DoubleConstant((meta, number.parse().unwrap()))) - } else { - meta.chars.end = start + pos; - Some(Token::FloatConstant((meta, number.parse().unwrap()))) - } - } else if first == Some('u') { - if hexadecimal { - meta.chars.end = start + pos + 3; - Some(Token::UintConstant(( - meta, - u64::from_str_radix(number, 16).unwrap(), - ))) - } else { - meta.chars.end = start + pos + 1; - Some(Token::UintConstant((meta, number.parse().unwrap()))) - } - } else if hexadecimal { - meta.chars.end = start + pos + 2; - Some(Token::IntConstant(( - meta, - i64::from_str_radix(number, 16).unwrap(), - ))) - } else { - meta.chars.end = start + pos; - Some(Token::IntConstant((meta, number.parse().unwrap()))) - } - } - 'a'..='z' | 'A'..='Z' | '_' => { - let (word, _, pos) = consume_any(input, |c| c.is_ascii_alphanumeric() || c == '_'); - meta.chars.end = start + pos; - match word { - "layout" => Some(Token::Layout(meta)), - "in" => Some(Token::In(meta)), - "out" => Some(Token::Out(meta)), - "uniform" => Some(Token::Uniform(meta)), - "flat" => Some(Token::Interpolation((meta, crate::Interpolation::Flat))), - "noperspective" => { - Some(Token::Interpolation((meta, crate::Interpolation::Linear))) - } - "smooth" => Some(Token::Interpolation(( - meta, - crate::Interpolation::Perspective, - ))), - "centroid" => { - Some(Token::Interpolation((meta, crate::Interpolation::Centroid))) - } - "sample" => Some(Token::Interpolation((meta, crate::Interpolation::Sample))), - // values - "true" => Some(Token::BoolConstant((meta, true))), - "false" => Some(Token::BoolConstant((meta, false))), - // jump statements - "continue" => Some(Token::Continue(meta)), - "break" => Some(Token::Break(meta)), - "return" => Some(Token::Return(meta)), - "discard" => Some(Token::Discard(meta)), - // selection statements - "if" => Some(Token::If(meta)), - "else" => Some(Token::Else(meta)), - "switch" => Some(Token::Switch(meta)), - "case" => Some(Token::Case(meta)), - "default" => Some(Token::Default(meta)), - // iteration statements - "while" => Some(Token::While(meta)), - "do" => Some(Token::Do(meta)), - "for" => Some(Token::For(meta)), - // types - "void" => Some(Token::Void(meta)), - "const" => Some(Token::Const(meta)), - word => { - let token = match parse_type(word) { - Some(t) => Token::TypeName((meta, t)), - None => Token::Identifier((meta, String::from(word))), - }; - Some(token) - } - } - } - '+' | '-' | '&' | '|' | '^' => { - let next = chars.next(); - if next == Some(cur) { - meta.chars.end = start + 2; - match cur { - '+' => Some(Token::IncOp(meta)), - '-' => Some(Token::DecOp(meta)), - '&' => Some(Token::AndOp(meta)), - '|' => Some(Token::OrOp(meta)), - '^' => Some(Token::XorOp(meta)), - _ => None, - } - } else { - match next { - Some('=') => { - meta.chars.end = start + 2; - match cur { - '+' => Some(Token::AddAssign(meta)), - '-' => Some(Token::SubAssign(meta)), - '&' => Some(Token::AndAssign(meta)), - '|' => Some(Token::OrAssign(meta)), - '^' => Some(Token::XorAssign(meta)), - _ => None, - } - } - _ => match cur { - '+' => Some(Token::Plus(meta)), - '-' => Some(Token::Dash(meta)), - '&' => Some(Token::Ampersand(meta)), - '|' => Some(Token::VerticalBar(meta)), - '^' => Some(Token::Caret(meta)), - _ => None, - }, - } - } - } - - '%' | '!' | '=' => match chars.next() { - Some('=') => { - meta.chars.end = start + 2; - match cur { - '%' => Some(Token::ModAssign(meta)), - '!' => Some(Token::NeOp(meta)), - '=' => Some(Token::EqOp(meta)), - _ => None, - } - } - _ => match cur { - '%' => Some(Token::Percent(meta)), - '!' => Some(Token::Bang(meta)), - '=' => Some(Token::Equal(meta)), - _ => None, - }, - }, - - '*' => match chars.next() { - Some('=') => { - meta.chars.end = start + 2; - Some(Token::MulAssign(meta)) - } - Some('/') => { - meta.chars.end = start + 2; - Some(Token::CommentEnd((meta, ()))) - } - _ => Some(Token::Star(meta)), - }, - '/' => { - match chars.next() { - Some('=') => { - meta.chars.end = start + 2; - Some(Token::DivAssign(meta)) - } - Some('/') => { - // consume rest of line - consume_all = true; - None - } - Some('*') => { - meta.chars.end = start + 2; - Some(Token::CommentStart((meta, ()))) - } - _ => Some(Token::Slash(meta)), - } - } - '#' => { - if self.offset == 0 { - let mut input = chars.as_str(); - - // skip whitespace - let word_start = input - .find(|c: char| !c.is_whitespace()) - .unwrap_or_else(|| input.chars().count()); - input = &input[word_start..]; - - let (word, _, pos) = consume_any(input, |c| c.is_alphanumeric() || c == '_'); - meta.chars.end = start + word_start + 1 + pos; - match word { - "version" => Some(Token::Version(meta)), - w => Some(Token::Unknown((meta, w.into()))), - } - - //TODO: preprocessor - // if chars.next() == Some(cur) { - // (Token::TokenPasting, chars.as_str(), start, start + 2) - // } else { - // (Token::Preprocessor, input, start, start + 1) - // } - } else { - Some(Token::Unknown((meta, '#'.to_string()))) - } - } - '~' => Some(Token::Tilde(meta)), - '?' => Some(Token::Question(meta)), - ch => Some(Token::Unknown((meta, ch.to_string()))), - }; - if let Some(token) = token { - let skip_bytes = input - .chars() - .take(token.extra().chars.end - start) - .fold(0, |acc, c| acc + c.len_utf8()); - self.input = input[skip_bytes..].into(); - Some(token) - } else { - if consume_all { - self.input = "".into(); - } else { - self.input = self.input[start..].into(); - } - None + pub fn new(input: &'a str, defines: &'a FastHashMap) -> Self { + let mut pp = Preprocessor::new(input); + for (define, value) in defines { + pp.add_define(define, value).unwrap(); //TODO: handle error + } + Lexer { + pp, + tokens: Default::default(), } } - - pub fn new(input: &'a str, defines: &FastHashMap) -> Self { - let mut lexer = Lexer { - lines: input.lines().enumerate(), - input: "".to_string(), - line: 0, - offset: 0, - inside_comment: false, - pp: LinePreProcessor::new(defines), - }; - lexer.next_line(); - lexer - } - - fn next_line(&mut self) -> bool { - if let Some((line, input)) = self.lines.next() { - let mut input = String::from(input); - - while input.ends_with('\\') { - if let Some((_, next)) = self.lines.next() { - input.pop(); - input.push_str(next); - } else { - break; - } - } - - if let Ok(processed) = self.pp.process_line(&input) { - self.input = processed.unwrap_or_default(); - self.line = line; - self.offset = 0; - true - } else { - //TODO: handle preprocessor error - false - } - } else { - false - } - } - - #[must_use] - pub fn next(&mut self) -> Option { - let token = self.consume_token(); - - if let Some(mut token) = token { - let meta = token.extra_mut(); - let end = meta.chars.end; - meta.line = self.line; - meta.chars.start += self.offset; - meta.chars.end += self.offset; - self.offset += end; - if !self.inside_comment { - match token { - Token::CommentStart(_) => { - self.inside_comment = true; - self.next() - } - _ => Some(token), - } - } else { - if let Token::CommentEnd(_) = token { - self.inside_comment = false; - } - self.next() - } - } else { - if !self.next_line() { - return None; - } - self.next() - } - } - - // #[must_use] - // pub fn peek(&mut self) -> Option { - // self.clone().next() - // } } impl<'a> Iterator for Lexer<'a> { type Item = Token; fn next(&mut self) -> Option { - self.next() + let mut meta = TokenMetadata { + line: 0, + chars: 0..0, + }; + let pp_token = match self.tokens.pop_front() { + Some(t) => t, + None => match self.pp.next()? { + Ok(t) => t, + Err((err, loc)) => { + meta.line = loc.line as usize; + meta.chars.start = loc.pos as usize; + //TODO: proper location end + meta.chars.end = loc.pos as usize + 1; + return Some(Token::Unknown((meta, err))); + } + }, + }; + + meta.line = pp_token.location.line as usize; + meta.chars.start = pp_token.location.pos as usize; + //TODO: proper location end + meta.chars.end = pp_token.location.pos as usize + 1; + Some(match pp_token.value { + TokenValue::Extension(extension) => { + for t in extension.tokens { + self.tokens.push_back(t); + } + Token::Extension((meta, ())) + } + TokenValue::Float(float) => Token::FloatConstant((meta, float.value)), + TokenValue::Ident(ident) => { + match ident.as_str() { + "layout" => Token::Layout(meta), + "in" => Token::In(meta), + "out" => Token::Out(meta), + "uniform" => Token::Uniform(meta), + "flat" => Token::Interpolation((meta, crate::Interpolation::Flat)), + "noperspective" => Token::Interpolation((meta, crate::Interpolation::Linear)), + "smooth" => Token::Interpolation((meta, crate::Interpolation::Perspective)), + "centroid" => Token::Interpolation((meta, crate::Interpolation::Centroid)), + "sample" => Token::Interpolation((meta, crate::Interpolation::Sample)), + // values + "true" => Token::BoolConstant((meta, true)), + "false" => Token::BoolConstant((meta, false)), + // jump statements + "continue" => Token::Continue(meta), + "break" => Token::Break(meta), + "return" => Token::Return(meta), + "discard" => Token::Discard(meta), + // selection statements + "if" => Token::If(meta), + "else" => Token::Else(meta), + "switch" => Token::Switch(meta), + "case" => Token::Case(meta), + "default" => Token::Default(meta), + // iteration statements + "while" => Token::While(meta), + "do" => Token::Do(meta), + "for" => Token::For(meta), + // types + "void" => Token::Void(meta), + "const" => Token::Const(meta), + + word => match parse_type(word) { + Some(t) => Token::TypeName((meta, t)), + None => Token::Identifier((meta, String::from(word))), + }, + } + } + //TODO: unsigned etc + TokenValue::Integer(integer) => Token::IntConstant((meta, integer.value as i64)), + TokenValue::Punct(punct) => match punct { + // Compound assignments + Punct::AddAssign => Token::AddAssign(meta), + Punct::SubAssign => Token::SubAssign(meta), + Punct::MulAssign => Token::MulAssign(meta), + Punct::DivAssign => Token::DivAssign(meta), + Punct::ModAssign => Token::ModAssign(meta), + Punct::LeftShiftAssign => Token::LeftAssign(meta), + Punct::RightShiftAssign => Token::RightAssign(meta), + Punct::AndAssign => Token::AndAssign(meta), + Punct::XorAssign => Token::XorAssign(meta), + Punct::OrAssign => Token::OrAssign(meta), + + // Two character punctuation + Punct::Increment => Token::IncOp(meta), + Punct::Decrement => Token::DecOp(meta), + Punct::LogicalAnd => Token::AndOp(meta), + Punct::LogicalOr => Token::OrOp(meta), + Punct::LogicalXor => Token::XorOp(meta), + Punct::LessEqual => Token::LeOp(meta), + Punct::GreaterEqual => Token::GeOp(meta), + Punct::EqualEqual => Token::EqOp(meta), + Punct::NotEqual => Token::NeOp(meta), + Punct::LeftShift => Token::LeftOp(meta), + Punct::RightShift => Token::RightOp(meta), + + // Parenthesis or similar + Punct::LeftBrace => Token::LeftBrace(meta), + Punct::RightBrace => Token::RightBrace(meta), + Punct::LeftParen => Token::LeftParen(meta), + Punct::RightParen => Token::RightParen(meta), + Punct::LeftBracket => Token::LeftBracket(meta), + Punct::RightBracket => Token::RightBracket(meta), + + // Other one character punctuation + Punct::LeftAngle => Token::LeftAngle(meta), + Punct::RightAngle => Token::RightAngle(meta), + Punct::Semicolon => Token::Semicolon(meta), + Punct::Comma => Token::Comma(meta), + Punct::Colon => Token::Colon(meta), + Punct::Dot => Token::Dot(meta), + Punct::Equal => Token::Equal(meta), + Punct::Bang => Token::Bang(meta), + Punct::Minus => Token::Dash(meta), + Punct::Tilde => Token::Tilde(meta), + Punct::Plus => Token::Plus(meta), + Punct::Star => Token::Star(meta), + Punct::Slash => Token::Slash(meta), + Punct::Percent => Token::Percent(meta), + Punct::Pipe => Token::VerticalBar(meta), + Punct::Caret => Token::Caret(meta), + Punct::Ampersand => Token::Ampersand(meta), + Punct::Question => Token::Question(meta), + }, + TokenValue::Pragma(pragma) => { + for t in pragma.tokens { + self.tokens.push_back(t); + } + Token::Pragma((meta, ())) + } + TokenValue::Version(version) => { + for t in version.tokens { + self.tokens.push_back(t); + } + Token::Version(meta) + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::{ + super::{parser::Token::*, token::TokenMetadata}, + Lexer, + }; + + #[test] + fn lex_tokens() { + let defines = crate::FastHashMap::default(); + + // line comments + let mut lex = Lexer::new("#version 450\nvoid main () {}", &defines); + assert_eq!( + lex.next().unwrap(), + Version(TokenMetadata { + line: 1, + chars: 1..2 //TODO + }) + ); + assert_eq!( + lex.next().unwrap(), + IntConstant(( + TokenMetadata { + line: 1, + chars: 9..10 //TODO + }, + 450 + )) + ); + assert_eq!( + lex.next().unwrap(), + Void(TokenMetadata { + line: 2, + chars: 0..1 //TODO + }) + ); + assert_eq!( + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 2, + chars: 5..6 //TODO + }, + "main".into() + )) + ); + assert_eq!( + lex.next().unwrap(), + LeftParen(TokenMetadata { + line: 2, + chars: 10..11 //TODO + }) + ); + assert_eq!( + lex.next().unwrap(), + RightParen(TokenMetadata { + line: 2, + chars: 11..12 //TODO + }) + ); + assert_eq!( + lex.next().unwrap(), + LeftBrace(TokenMetadata { + line: 2, + chars: 13..14 //TODO + }) + ); + assert_eq!( + lex.next().unwrap(), + RightBrace(TokenMetadata { + line: 2, + chars: 14..15 //TODO + }) + ); + assert_eq!(lex.next(), None); } } diff --git a/src/front/glsl/lex_tests.rs b/src/front/glsl/lex_tests.rs deleted file mode 100644 index b2bde5fed..000000000 --- a/src/front/glsl/lex_tests.rs +++ /dev/null @@ -1,469 +0,0 @@ -use super::{lex::Lexer, parser::Token::*, token::TokenMetadata}; - -#[test] -fn tokens() { - let defines = crate::FastHashMap::default(); - - // line comments - let mut lex = Lexer::new("void main // myfunction\n//()\n{}", &defines); - assert_eq!( - lex.next().unwrap(), - Void(TokenMetadata { - line: 0, - chars: 0..4 - }) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 5..9 - }, - "main".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - LeftBrace(TokenMetadata { - line: 2, - chars: 0..1 - }) - ); - assert_eq!( - lex.next().unwrap(), - RightBrace(TokenMetadata { - line: 2, - chars: 1..2 - }) - ); - assert_eq!(lex.next(), None); - - // multi line comment - let mut lex = Lexer::new("void main /* comment [] {}\n/**\n{}*/{}", &defines); - assert_eq!( - lex.next().unwrap(), - Void(TokenMetadata { - line: 0, - chars: 0..4 - }) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 5..9 - }, - "main".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - LeftBrace(TokenMetadata { - line: 2, - chars: 4..5 - }) - ); - assert_eq!( - lex.next().unwrap(), - RightBrace(TokenMetadata { - line: 2, - chars: 5..6 - }) - ); - assert_eq!(lex.next(), None); - - // identifiers - let mut lex = Lexer::new("id123_OK 92No æNoø No¾ No好", &defines); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 0..8 - }, - "id123_OK".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - IntConstant(( - TokenMetadata { - line: 0, - chars: 9..11 - }, - 92 - )) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 11..13 - }, - "No".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - Unknown(( - TokenMetadata { - line: 0, - chars: 14..15 - }, - 'æ'.to_string() - )) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 15..17 - }, - "No".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - Unknown(( - TokenMetadata { - line: 0, - chars: 17..18 - }, - 'ø'.to_string() - )) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 19..21 - }, - "No".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - Unknown(( - TokenMetadata { - line: 0, - chars: 21..22 - }, - '¾'.to_string() - )) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 23..25 - }, - "No".into() - )) - ); - assert_eq!( - lex.next().unwrap(), - Unknown(( - TokenMetadata { - line: 0, - chars: 25..26 - }, - '好'.to_string() - )) - ); - assert_eq!(lex.next(), None); - - // version - let mut lex = Lexer::new("#version 890 core", &defines); - assert_eq!( - lex.next().unwrap(), - Version(TokenMetadata { - line: 0, - chars: 0..8 - }) - ); - assert_eq!( - lex.next().unwrap(), - IntConstant(( - TokenMetadata { - line: 0, - chars: 9..12 - }, - 890 - )) - ); - assert_eq!( - lex.next().unwrap(), - Identifier(( - TokenMetadata { - line: 0, - chars: 13..17 - }, - "core".into() - )) - ); - assert_eq!(lex.next(), None); - - // operators - let mut lex = Lexer::new( - "+ - * | & % / += -= *= |= &= %= /= ++ -- || && ^^", - &defines, - ); - assert_eq!( - lex.next().unwrap(), - Plus(TokenMetadata { - line: 0, - chars: 0..1 - }) - ); - assert_eq!( - lex.next().unwrap(), - Dash(TokenMetadata { - line: 0, - chars: 2..3 - }) - ); - assert_eq!( - lex.next().unwrap(), - Star(TokenMetadata { - line: 0, - chars: 4..5 - }) - ); - assert_eq!( - lex.next().unwrap(), - VerticalBar(TokenMetadata { - line: 0, - chars: 6..7 - }) - ); - assert_eq!( - lex.next().unwrap(), - Ampersand(TokenMetadata { - line: 0, - chars: 8..9 - }) - ); - assert_eq!( - lex.next().unwrap(), - Percent(TokenMetadata { - line: 0, - chars: 10..11 - }) - ); - assert_eq!( - lex.next().unwrap(), - Slash(TokenMetadata { - line: 0, - chars: 12..13 - }) - ); - assert_eq!( - lex.next().unwrap(), - AddAssign(TokenMetadata { - line: 0, - chars: 14..16 - }) - ); - assert_eq!( - lex.next().unwrap(), - SubAssign(TokenMetadata { - line: 0, - chars: 17..19 - }) - ); - assert_eq!( - lex.next().unwrap(), - MulAssign(TokenMetadata { - line: 0, - chars: 20..22 - }) - ); - assert_eq!( - lex.next().unwrap(), - OrAssign(TokenMetadata { - line: 0, - chars: 23..25 - }) - ); - assert_eq!( - lex.next().unwrap(), - AndAssign(TokenMetadata { - line: 0, - chars: 26..28 - }) - ); - assert_eq!( - lex.next().unwrap(), - ModAssign(TokenMetadata { - line: 0, - chars: 29..31 - }) - ); - assert_eq!( - lex.next().unwrap(), - DivAssign(TokenMetadata { - line: 0, - chars: 32..34 - }) - ); - assert_eq!( - lex.next().unwrap(), - IncOp(TokenMetadata { - line: 0, - chars: 35..37 - }) - ); - assert_eq!( - lex.next().unwrap(), - DecOp(TokenMetadata { - line: 0, - chars: 38..40 - }) - ); - assert_eq!( - lex.next().unwrap(), - OrOp(TokenMetadata { - line: 0, - chars: 41..43 - }) - ); - assert_eq!( - lex.next().unwrap(), - AndOp(TokenMetadata { - line: 0, - chars: 44..46 - }) - ); - assert_eq!( - lex.next().unwrap(), - XorOp(TokenMetadata { - line: 0, - chars: 47..49 - }) - ); - assert_eq!(lex.next(), None); - - // float constants - let mut lex = Lexer::new("120.0 130.0lf 140.0Lf 150.0LF", &defines); - assert_eq!( - lex.next().unwrap(), - FloatConstant(( - TokenMetadata { - line: 0, - chars: 0..5 - }, - 120.0, - )) - ); - assert_eq!( - lex.next().unwrap(), - DoubleConstant(( - TokenMetadata { - line: 0, - chars: 6..13 - }, - 130.0, - )) - ); - assert_eq!( - lex.next().unwrap(), - DoubleConstant(( - TokenMetadata { - line: 0, - chars: 14..21 - }, - 140.0, - )) - ); - assert_eq!( - lex.next().unwrap(), - DoubleConstant(( - TokenMetadata { - line: 0, - chars: 22..29 - }, - 150.0, - )) - ); - assert_eq!(lex.next(), None); - - // Integer constants - let mut lex = Lexer::new("120 130u 140U 150 0x1f 0xf2U 0xF1u", &defines); - assert_eq!( - lex.next().unwrap(), - IntConstant(( - TokenMetadata { - line: 0, - chars: 0..3 - }, - 120, - )) - ); - assert_eq!( - lex.next().unwrap(), - UintConstant(( - TokenMetadata { - line: 0, - chars: 4..8 - }, - 130, - )) - ); - assert_eq!( - lex.next().unwrap(), - UintConstant(( - TokenMetadata { - line: 0, - chars: 9..13 - }, - 140, - )) - ); - assert_eq!( - lex.next().unwrap(), - IntConstant(( - TokenMetadata { - line: 0, - chars: 14..17 - }, - 150, - )) - ); - assert_eq!( - lex.next().unwrap(), - IntConstant(( - TokenMetadata { - line: 0, - chars: 18..22 - }, - 31, - )) - ); - assert_eq!( - lex.next().unwrap(), - UintConstant(( - TokenMetadata { - line: 0, - chars: 23..28 - }, - 242, - )) - ); - assert_eq!( - lex.next().unwrap(), - UintConstant(( - TokenMetadata { - line: 0, - chars: 29..34 - }, - 241, - )) - ); - assert_eq!(lex.next(), None); -} diff --git a/src/front/glsl/mod.rs b/src/front/glsl/mod.rs index 92365f59d..327b8424c 100644 --- a/src/front/glsl/mod.rs +++ b/src/front/glsl/mod.rs @@ -1,17 +1,10 @@ use crate::{FastHashMap, Module, ShaderStage}; mod lex; -#[cfg(test)] -mod lex_tests; - -mod preprocess; -#[cfg(test)] -mod preprocess_tests; mod ast; use ast::Program; -use lex::Lexer; mod error; pub use error::ParseError; mod constants; @@ -31,7 +24,7 @@ pub struct Options { pub fn parse_str(source: &str, options: &Options) -> Result { let mut program = Program::new(&options.entry_points); - let lex = Lexer::new(source, &options.defines); + let lex = lex::Lexer::new(source, &options.defines); let mut parser = parser::Parser::new(&mut program); for token in lex { parser.parse(token)?; diff --git a/src/front/glsl/parser.rs b/src/front/glsl/parser.rs index 06b2d3e6d..97fa3a35c 100644 --- a/src/front/glsl/parser.rs +++ b/src/front/glsl/parser.rs @@ -12,6 +12,7 @@ pomelo! { Statement, StorageAccess, StorageClass, StructMember, SwitchCase, Type, TypeInner, UnaryOperator, FunctionArgument }; + use pp_rs::token::PreprocessorError; } %token #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum Token {}; %parser pub struct Parser<'a, 'b> {}; @@ -31,9 +32,9 @@ pomelo! { ErrorKind::ParserStackOverflow } - %type Unknown String; - %type CommentStart (); - %type CommentEnd (); + %type Unknown PreprocessorError; + %type Pragma (); + %type Extension (); %type Identifier String; // constants diff --git a/src/front/glsl/parser_tests.rs b/src/front/glsl/parser_tests.rs index 2fb3357de..de0b4db41 100644 --- a/src/front/glsl/parser_tests.rs +++ b/src/front/glsl/parser_tests.rs @@ -32,7 +32,7 @@ fn version() { .err() .unwrap() ), - "InvalidVersion(TokenMetadata { line: 0, chars: 9..14 }, 99000)" + "InvalidVersion(TokenMetadata { line: 1, chars: 9..10 }, 99000)" //TODO: location ); assert_eq!( @@ -40,7 +40,7 @@ fn version() { "{:?}", parse_program("#version 449", &entry_points).err().unwrap() ), - "InvalidVersion(TokenMetadata { line: 0, chars: 9..12 }, 449)" + "InvalidVersion(TokenMetadata { line: 1, chars: 9..10 }, 449)" //TODO: location ); assert_eq!( @@ -50,7 +50,7 @@ fn version() { .err() .unwrap() ), - "InvalidProfile(TokenMetadata { line: 0, chars: 13..18 }, \"smart\")" + "InvalidProfile(TokenMetadata { line: 1, chars: 13..14 }, \"smart\")" //TODO: location ); assert_eq!( @@ -60,7 +60,7 @@ fn version() { .err() .unwrap() ), - "InvalidToken(Unknown((TokenMetadata { line: 1, chars: 11..12 }, \"#\")))" + "InvalidToken(Unknown((TokenMetadata { line: 2, chars: 11..12 }, UnexpectedHash)))" ); // valid versions diff --git a/src/front/glsl/preprocess.rs b/src/front/glsl/preprocess.rs deleted file mode 100644 index f9b5a3fe7..000000000 --- a/src/front/glsl/preprocess.rs +++ /dev/null @@ -1,152 +0,0 @@ -use crate::FastHashMap; -use thiserror::Error; - -#[derive(Clone, Debug, Error)] -#[cfg_attr(test, derive(PartialEq))] -pub enum Error { - #[error("unmatched else")] - UnmatchedElse, - #[error("unmatched endif")] - UnmatchedEndif, - #[error("missing macro name")] - MissingMacro, -} - -#[derive(Clone, Debug)] -pub struct IfState { - true_branch: bool, - else_seen: bool, -} - -#[derive(Clone, Debug)] -pub struct LinePreProcessor { - defines: FastHashMap, - if_stack: Vec, - inside_comment: bool, - in_preprocess: bool, -} - -impl LinePreProcessor { - pub fn new(defines: &FastHashMap) -> Self { - LinePreProcessor { - defines: defines.clone(), - if_stack: vec![], - inside_comment: false, - in_preprocess: false, - } - } - - fn subst_defines(&self, input: &str) -> String { - //TODO: don't subst in commments, strings literals? - self.defines - .iter() - .fold(input.to_string(), |acc, (k, v)| acc.replace(k, v)) - } - - pub fn process_line(&mut self, line: &str) -> Result, Error> { - let mut skip = !self.if_stack.last().map(|i| i.true_branch).unwrap_or(true); - let mut inside_comment = self.inside_comment; - let mut in_preprocess = inside_comment && self.in_preprocess; - // single-line comment - let mut processed = line; - if let Some(pos) = line.find("//") { - processed = line.split_at(pos).0; - } - // multi-line comment - let mut processed_string: String; - loop { - if inside_comment { - if let Some(pos) = processed.find("*/") { - processed = processed.split_at(pos + 2).1; - inside_comment = false; - self.inside_comment = false; - continue; - } - } else if let Some(pos) = processed.find("/*") { - if let Some(end_pos) = processed[pos + 2..].find("*/") { - // comment ends during this line - processed_string = processed.to_string(); - processed_string.replace_range(pos..pos + end_pos + 4, ""); - processed = &processed_string; - } else { - processed = processed.split_at(pos).0; - inside_comment = true; - } - continue; - } - break; - } - // strip leading whitespace - processed = processed.trim_start(); - if processed.starts_with('#') && !self.inside_comment { - let mut iter = processed[1..] - .trim_start() - .splitn(2, |c: char| c.is_whitespace()); - if let Some(directive) = iter.next() { - skip = true; - in_preprocess = true; - match directive { - "version" => { - skip = false; - } - "define" => { - let rest = iter.next().ok_or(Error::MissingMacro)?; - let pos = rest - .find(|c: char| !c.is_ascii_alphanumeric() && c != '_' && c != '(') - .unwrap_or_else(|| rest.len()); - let (key, mut value) = rest.split_at(pos); - value = value.trim(); - self.defines.insert(key.into(), self.subst_defines(value)); - } - "undef" => { - let rest = iter.next().ok_or(Error::MissingMacro)?; - let key = rest.trim(); - self.defines.remove(key); - } - "ifdef" => { - let rest = iter.next().ok_or(Error::MissingMacro)?; - let key = rest.trim(); - self.if_stack.push(IfState { - true_branch: self.defines.contains_key(key), - else_seen: false, - }); - } - "ifndef" => { - let rest = iter.next().ok_or(Error::MissingMacro)?; - let key = rest.trim(); - self.if_stack.push(IfState { - true_branch: !self.defines.contains_key(key), - else_seen: false, - }); - } - "else" => { - let if_state = self.if_stack.last_mut().ok_or(Error::UnmatchedElse)?; - if !if_state.else_seen { - // this is first else - if_state.true_branch = !if_state.true_branch; - if_state.else_seen = true; - } else { - return Err(Error::UnmatchedElse); - } - } - "endif" => { - self.if_stack.pop().ok_or(Error::UnmatchedEndif)?; - } - _ => {} - } - } - } - let res = if !skip && !self.inside_comment { - Ok(Some(self.subst_defines(&line))) - } else { - Ok(if in_preprocess && !self.in_preprocess { - Some("".to_string()) - } else { - None - }) - }; - self.in_preprocess = in_preprocess || skip; - self.inside_comment = inside_comment; - res - } -} diff --git a/src/front/glsl/preprocess_tests.rs b/src/front/glsl/preprocess_tests.rs deleted file mode 100644 index 44e9bf27b..000000000 --- a/src/front/glsl/preprocess_tests.rs +++ /dev/null @@ -1,228 +0,0 @@ -use super::preprocess::{Error, LinePreProcessor}; -use std::{iter::Enumerate, str::Lines}; - -#[derive(Clone, Debug)] -pub struct PreProcessor<'a> { - lines: Enumerate>, - input: String, - line: usize, - offset: usize, - line_pp: LinePreProcessor, -} - -impl<'a> PreProcessor<'a> { - pub fn new(input: &'a str, defines: crate::FastHashMap) -> Self { - let mut lexer = PreProcessor { - lines: input.lines().enumerate(), - input: "".to_string(), - line: 0, - offset: 0, - line_pp: LinePreProcessor::new(&defines), - }; - lexer.next_line(); - lexer - } - - fn next_line(&mut self) -> bool { - if let Some((line, input)) = self.lines.next() { - let mut input = String::from(input); - - while input.ends_with('\\') { - if let Some((_, next)) = self.lines.next() { - input.pop(); - input.push_str(next); - } else { - break; - } - } - - self.input = input; - self.line = line; - self.offset = 0; - true - } else { - false - } - } - - pub fn process(&mut self) -> Result { - let mut res = String::new(); - loop { - let line = &self.line_pp.process_line(&self.input)?; - if let Some(line) = line { - res.push_str(line); - } - if !self.next_line() { - break; - } - if line.is_some() { - res.push_str("\n"); - } - } - Ok(res) - } -} - -#[test] -fn preprocess() { - // line continuation - let mut pp = PreProcessor::new( - "void main my_\ - func", - Default::default(), - ); - assert_eq!(pp.process().unwrap(), "void main my_func"); - - // preserve #version - let mut pp = PreProcessor::new( - "#version 450 core\n\ - void main()", - Default::default(), - ); - assert_eq!(pp.process().unwrap(), "#version 450 core\nvoid main()"); - - // simple define - let mut pp = PreProcessor::new( - "#define FOO 42 \n\ - fun=FOO", - Default::default(), - ); - assert_eq!(pp.process().unwrap(), "\nfun=42"); - - // ifdef with else - let mut pp = PreProcessor::new( - "#define FOO\n\ - #ifdef FOO\n\ - foo=42\n\ - #endif\n\ - some=17\n\ - #ifdef BAR\n\ - bar=88\n\ - #else\n\ - mm=49\n\ - #endif\n\ - done=1", - Default::default(), - ); - assert_eq!( - pp.process().unwrap(), - "\n\ - foo=42\n\ - \n\ - some=17\n\ - \n\ - mm=49\n\ - \n\ - done=1" - ); - - // nested ifdef/ifndef - let mut pp = PreProcessor::new( - "#define FOO\n\ - #define BOO\n\ - #ifdef FOO\n\ - foo=42\n\ - #ifdef BOO\n\ - boo=44\n\ - #endif\n\ - ifd=0\n\ - #ifndef XYZ\n\ - nxyz=8\n\ - #endif\n\ - #endif\n\ - some=17\n\ - #ifdef BAR\n\ - bar=88\n\ - #else\n\ - mm=49\n\ - #endif\n\ - done=1", - Default::default(), - ); - assert_eq!( - pp.process().unwrap(), - "\n\ - foo=42\n\ - \n\ - boo=44\n\ - \n\ - ifd=0\n\ - \n\ - nxyz=8\n\ - \n\ - some=17\n\ - \n\ - mm=49\n\ - \n\ - done=1" - ); - - // undef - let mut pp = PreProcessor::new( - "#define FOO\n\ - #ifdef FOO\n\ - foo=42\n\ - #endif\n\ - some=17\n\ - #undef FOO\n\ - #ifdef FOO\n\ - foo=88\n\ - #else\n\ - nofoo=66\n\ - #endif\n\ - done=1", - Default::default(), - ); - assert_eq!( - pp.process().unwrap(), - "\n\ - foo=42\n\ - \n\ - some=17\n\ - \n\ - nofoo=66\n\ - \n\ - done=1" - ); - - // single-line comment - let mut pp = PreProcessor::new( - "#define FOO 42//1234\n\ - fun=FOO", - Default::default(), - ); - assert_eq!(pp.process().unwrap(), "\nfun=42"); - - // multi-line comments - let mut pp = PreProcessor::new( - "#define FOO 52/*/1234\n\ - #define FOO 88\n\ - end of comment*/ /* one more comment */ #define FOO 56\n\ - fun=FOO", - Default::default(), - ); - assert_eq!(pp.process().unwrap(), "\nfun=56"); - - // unmatched endif - let mut pp = PreProcessor::new( - "#ifdef FOO\n\ - foo=42\n\ - #endif\n\ - #endif", - Default::default(), - ); - assert_eq!(pp.process(), Err(Error::UnmatchedEndif)); - - // unmatched else - let mut pp = PreProcessor::new( - "#ifdef FOO\n\ - foo=42\n\ - #else\n\ - bar=88\n\ - #else\n\ - bad=true\n\ - #endif", - Default::default(), - ); - assert_eq!(pp.process(), Err(Error::UnmatchedElse)); -}