Detect confusing unicode characters and show the alternative

src/libsyntax/diagnostic.rs

@@ -174,6 +174,10 @@ impl SpanHandler {
         self.handler.emit(Some((&self.cm, sp)), msg, Bug);
         panic!(ExplicitBug);
     }
+    pub fn span_bug_no_panic(&self, sp: Span, msg: &str) {
+        self.handler.emit(Some((&self.cm, sp)), msg, Bug);
+        self.handler.bump_err_count();
+    }
     pub fn span_unimpl(&self, sp: Span, msg: &str) -> ! {
         self.span_bug(sp, &format!("unimplemented {}", msg));
     }

src/libsyntax/parse/lexer/mod.rs

@@ -26,6 +26,7 @@ use std::rc::Rc;
 pub use ext::tt::transcribe::{TtReader, new_tt_reader, new_tt_reader_with_doc_flag};
 
 pub mod comments;
+mod unicode_chars;
 
 pub trait Reader {
     fn is_eof(&self) -> bool;
@@ -1224,7 +1225,8 @@ impl<'a> StringReader<'a> {
           c => {
               let last_bpos = self.last_pos;
               let bpos = self.pos;
-              panic!(self.fatal_span_char(last_bpos, bpos, "unknown start of token", c));
+              unicode_chars::check_for_substitution(&self, c);
+              panic!(self.fatal_span_char(last_bpos, bpos, "unknown start of token", c))
           }
         }
     }

src/libsyntax/parse/lexer/unicode_chars.rs

@@ -0,0 +1,186 @@
+// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Characters and their corresponding confusables were collected from
+// http://www.unicode.org/Public/security/revision-06/confusables.txt
+
+use codemap::mk_sp as make_span;
+use super::StringReader;
+
+const UNICODE_ARRAY: &'static [(char, &'static str, char)] = &[
+    ('ߺ', "Nko Lajanyalan", '_'),
+    ('﹍', "Dashed Low Line", '_'),
+    ('﹎', "Centreline Low Line", '_'),
+    ('﹏', "Wavy Low Line", '_'),
+    ('‐', "Hyphen", '-'),
+    ('‑', "Non-Breaking Hyphen", '-'),
+    ('‒', "Figure Dash", '-'),
+    ('–', "En Dash", '-'),
+    ('﹘', "Small Em Dash", '-'),
+    ('⁃', "Hyphen Bullet", '-'),
+    ('˗', "Modifier Letter Minus Sign", '-'),
+    ('−', "Minus Sign", '-'),
+    ('٫', "Arabic Decimal Separator", ','),
+    ('‚', "Single Low-9 Quotation Mark", ','),
+    ('ꓹ', "Lisu Letter Tone Na Po", ','),
+    (';', "Greek Question Mark", ';'),
+    ('ः', "Devanagari Sign Visarga", ':'),
+    ('ઃ', "Gujarati Sign Visarga", ':'),
+    ('：', "Fullwidth Colon", ':'),
+    ('։', "Armenian Full Stop", ':'),
+    ('܃', "Syriac Supralinear Colon", ':'),
+    ('܄', "Syriac Sublinear Colon", ':'),
+    ('︰', "Presentation Form For Vertical Two Dot Leader", ':'),
+    ('᠃', "Mongolian Full Stop", ':'),
+    ('᠉', "Mongolian Manchu Full Stop", ':'),
+    ('⁚', "Two Dot Punctuation", ':'),
+    ('׃', "Hebrew Punctuation Sof Pasuq", ':'),
+    ('˸', "Modifier Letter Raised Colon", ':'),
+    ('꞉', "Modifier Letter Colon", ':'),
+    ('∶', "Ratio", ':'),
+    ('ː', "Modifier Letter Triangular Colon", ':'),
+    ('ꓽ', "Lisu Letter Tone Mya Jeu", ':'),
+    ('！', "Fullwidth Exclamation Mark", '!'),
+    ('ǃ', "Latin Letter Retroflex Click", '!'),
+    ('ʔ', "Latin Letter Glottal Stop", '?'),
+    ('ॽ', "Devanagari Letter Glottal Stop", '?'),
+    ('Ꭾ', "Cherokee Letter He", '?'),
+    ('𝅭', "Musical Symbol Combining Augmentation Dot", '.'),
+    ('․', "One Dot Leader", '.'),
+    ('۔', "Arabic Full Stop", '.'),
+    ('܁', "Syriac Supralinear Full Stop", '.'),
+    ('܂', "Syriac Sublinear Full Stop", '.'),
+    ('꘎', "Vai Full Stop", '.'),
+    ('𐩐', "Kharoshthi Punctuation Dot", '.'),
+    ('٠', "Arabic-Indic Digit Zero", '.'),
+    ('۰', "Extended Arabic-Indic Digit Zero", '.'),
+    ('ꓸ', "Lisu Letter Tone Mya Ti", '.'),
+    ('՝', "Armenian Comma", '\''),
+    ('＇', "Fullwidth Apostrophe", '\''),
+    ('‘', "Left Single Quotation Mark", '\''),
+    ('’', "Right Single Quotation Mark", '\''),
+    ('‛', "Single High-Reversed-9 Quotation Mark", '\''),
+    ('′', "Prime", '\''),
+    ('‵', "Reversed Prime", '\''),
+    ('՚', "Armenian Apostrophe", '\''),
+    ('׳', "Hebrew Punctuation Geresh", '\''),
+    ('`', "Greek Varia", '\''),
+    ('｀', "Fullwidth Grave Accent", '\''),
+    ('΄', "Greek Tonos", '\''),
+    ('´', "Greek Oxia", '\''),
+    ('᾽', "Greek Koronis", '\''),
+    ('᾿', "Greek Psili", '\''),
+    ('῾', "Greek Dasia", '\''),
+    ('ʹ', "Modifier Letter Prime", '\''),
+    ('ʹ', "Greek Numeral Sign", '\''),
+    ('ˊ', "Modifier Letter Acute Accent", '\''),
+    ('ˋ', "Modifier Letter Grave Accent", '\''),
+    ('˴', "Modifier Letter Middle Grave Accent", '\''),
+    ('ʻ', "Modifier Letter Turned Comma", '\''),
+    ('ʽ', "Modifier Letter Reversed Comma", '\''),
+    ('ʼ', "Modifier Letter Apostrophe", '\''),
+    ('ʾ', "Modifier Letter Right Half Ring", '\''),
+    ('ꞌ', "Latin Small Letter Saltillo", '\''),
+    ('י', "Hebrew Letter Yod", '\''),
+    ('ߴ', "Nko High Tone Apostrophe", '\''),
+    ('ߵ', "Nko Low Tone Apostrophe", '\''),
+    ('［', "Fullwidth Left Square Bracket", '('),
+    ('❨', "Medium Left Parenthesis Ornament", '('),
+    ('❲', "Light Left Tortoise Shell Bracket Ornament", '('),
+    ('〔', "Left Tortoise Shell Bracket", '('),
+    ('﴾', "Ornate Left Parenthesis", '('),
+    ('］', "Fullwidth Right Square Bracket", ')'),
+    ('❩', "Medium Right Parenthesis Ornament", ')'),
+    ('❳', "Light Right Tortoise Shell Bracket Ornament", ')'),
+    ('〕', "Right Tortoise Shell Bracket", ')'),
+    ('﴿', "Ornate Right Parenthesis", ')'),
+    ('❴', "Medium Left Curly Bracket Ornament", '{'),
+    ('❵', "Medium Right Curly Bracket Ornament", '}'),
+    ('⁎', "Low Asterisk", '*'),
+    ('٭', "Arabic Five Pointed Star", '*'),
+    ('∗', "Asterisk Operator", '*'),
+    ('᜵', "Philippine Single Punctuation", '/'),
+    ('⁁', "Caret Insertion Point", '/'),
+    ('∕', "Division Slash", '/'),
+    ('⁄', "Fraction Slash", '/'),
+    ('╱', "Box Drawings Light Diagonal Upper Right To Lower Left", '/'),
+    ('⟋', "Mathematical Rising Diagonal", '/'),
+    ('⧸', "Big Solidus", '/'),
+    ('㇓', "Cjk Stroke Sp", '/'),
+    ('〳', "Vertical Kana Repeat Mark Upper Half", '/'),
+    ('丿', "Cjk Unified Ideograph-4E3F", '/'),
+    ('⼃', "Kangxi Radical Slash", '/'),
+    ('＼', "Fullwidth Reverse Solidus", '\\'),
+    ('﹨', "Small Reverse Solidus", '\\'),
+    ('∖', "Set Minus", '\\'),
+    ('⟍', "Mathematical Falling Diagonal", '\\'),
+    ('⧵', "Reverse Solidus Operator", '\\'),
+    ('⧹', "Big Reverse Solidus", '\\'),
+    ('㇔', "Cjk Stroke D", '\\'),
+    ('丶', "Cjk Unified Ideograph-4E36", '\\'),
+    ('⼂', "Kangxi Radical Dot", '\\'),
+    ('ꝸ', "Latin Small Letter Um", '&'),
+    ('﬩', "Hebrew Letter Alternative Plus Sign", '+'),
+    ('‹', "Single Left-Pointing Angle Quotation Mark", '<'),
+    ('❮', "Heavy Left-Pointing Angle Quotation Mark Ornament", '<'),
+    ('˂', "Modifier Letter Left Arrowhead", '<'),
+    ('꓿', "Lisu Punctuation Full Stop", '='),
+    ('›', "Single Right-Pointing Angle Quotation Mark", '>'),
+    ('❯', "Heavy Right-Pointing Angle Quotation Mark Ornament", '>'),
+    ('˃', "Modifier Letter Right Arrowhead", '>'),
+    ('Ⲻ', "Coptic Capital Letter Dialect-P Ni", '-'),
+    ('Ɂ', "Latin Capital Letter Glottal Stop", '?'),
+    ('Ⳇ', "Coptic Capital Letter Old Coptic Esh", '/'), ];
+
+const ASCII_ARRAY: &'static [(char, &'static str)] = &[
+    ('_', "Underscore"),
+    ('-', "Minus/Hyphen"),
+    (',', "Comma"),
+    (';', "Semicolon"),
+    (':', "Colon"),
+    ('!', "Exclamation Mark"),
+    ('?', "Question Mark"),
+    ('.', "Period"),
+    ('\'', "Single Quote"),
+    ('(', "Left Parenthesis"),
+    (')', "Right Parenthesis"),
+    ('{', "Left Curly Brace"),
+    ('}', "Right Curly Brace"),
+    ('*', "Asterisk"),
+    ('/', "Slash"),
+    ('\\', "Backslash"),
+    ('&', "Ampersand"),
+    ('+', "Plus Sign"),
+    ('<', "Less-Than Sign"),
+    ('=', "Equals Sign"),
+    ('>', "Greater-Than Sign"), ];
+
+pub fn check_for_substitution(reader: &StringReader, ch: char) {
+    UNICODE_ARRAY
+    .iter()
+    .find(|&&(c, _, _)| c == ch)
+    .map(|&(_, u_name, ascii_char)| {
+        let span = make_span(reader.last_pos, reader.pos);
+        match ASCII_ARRAY.iter().find(|&&(c, _)| c == ascii_char) {
+            Some(&(ascii_char, ascii_name)) => {
+                let msg =
+                    format!("unicode character '{}' ({}) looks much like '{}' ({}), but it's not",
+                            ch, u_name, ascii_char, ascii_name);
+                reader.help_span(span, &msg);
+            },
+            None => {
+                reader
+                .span_diagnostic
+                .span_bug_no_panic(span,
+                                   &format!("substitution character not found for '{}'", ch));
+            }
+        }
+    });
+}

src/test/parse-fail/unicode-chars.rs

@@ -0,0 +1,18 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: -Z parse-only
+// ignore-tidy-linelength
+
+fn main() {
+    let y = 0;
+    //~^ ERROR unknown start of token: \u{37e}
+    //~^^ HELP unicode character ';' (Greek Question Mark) looks much like ';' (Semicolon), but it's not
+}