Fix illegal chars in identifier for glsl-new

src/front/glsl_new/lex.rs

@@ -107,7 +107,7 @@ pub fn consume_token(mut input: &str) -> (Option<Token>, &str) {
             }
         }
         'a'..='z' | 'A'..='Z' | '_' => {
-            let (word, rest, pos) = consume_any(input, |c| c.is_alphanumeric() || c == '_');
+            let (word, rest, pos) = consume_any(input, |c| c.is_ascii_alphanumeric() || c == '_');
             meta.chars.end = start + pos;
             match word {
                 "void" => (Some(Token::Void(meta)), rest),
@@ -227,7 +227,7 @@ pub fn consume_token(mut input: &str) -> (Option<Token>, &str) {
         }
         '~' => (Some(Token::Tilde(meta)), chars.as_str()),
         '?' => (Some(Token::Question(meta)), chars.as_str()),
-        _ => (None, chars.as_str()),
+        ch => (Some(Token::Unknown((meta, ch))), chars.as_str()),
     }
 }
 

src/front/glsl_new/lex_tests.rs

@@ -60,3 +60,53 @@ fn glsl_lex_line_comment() {
         "RightBrace(TokenMetadata { line: 2, chars: 1..2 })"
     );
 }
+
+#[test]
+fn glsl_lex_identifier() {
+    let source = "id123_OK 92No æNoø No¾ No好";
+    let lex = Lexer::new(source);
+    let tokens: Vec<Token> = lex.collect();
+    assert_eq!(tokens.len(), 10);
+
+    let mut iter = tokens.iter();
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 0..8 }, \"id123_OK\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "IntConstant((TokenMetadata { line: 0, chars: 9..11 }, 92))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 11..13 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 14..15 }, \'æ\'))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 15..17 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 17..18 }, \'ø\'))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 19..21 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 21..22 }, \'¾\'))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 23..25 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 25..26 }, \'好\'))"
+    );
+}

src/front/glsl_new/parser.rs

@@ -24,6 +24,7 @@ pomelo! {
         ErrorKind::ParserStackOverflow
     }
 
+    %type Unknown char;
     %type Identifier String;
     %type IntConstant i64;
     %type UintConstant u64;