Auto merge of #72918 - Dylan-DPC:rollup-pnvj62b, r=Dylan-DPC

Rollup of 4 pull requests

Successful merges:

 - #72884 (RawString error reporting cleanup )
 - #72888 (Add a warning about infinite reading in read_(until|line))
 - #72914 (Minor: off-by-one error in RELEASES.md)
 - #72916 (Update README.md)

Failed merges:

r? @ghost

README.md

@@ -23,7 +23,7 @@ or reading the [rustc dev guide][rustcguidebuild].
 
 [rustcguidebuild]: https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html
 
-### Building on Unix-like system
+### Building on a Unix-like system
 1. Make sure you have installed the dependencies:
 
    * `g++` 5.1 or later or `clang++` 3.5 or later

RELEASES.md

@@ -43,7 +43,7 @@ Libraries
 - [Unicode 13 is now supported.][69929]
 - [`String` now implements `From<&mut str>`.][69661]
 - [`IoSlice` now implements `Copy`.][69403]
-- [`Vec<T>` now implements `From<[T; N]>`.][68692] Where `N` is less than 32.
+- [`Vec<T>` now implements `From<[T; N]>`.][68692] Where `N` is at most 32.
 - [`proc_macro::LexError` now implements `fmt::Display` and `Error`.][68899]
 - [`from_le_bytes`, `to_le_bytes`, `from_be_bytes`, `to_be_bytes`,
   `from_ne_bytes`, and `to_ne_bytes` methods are now `const` for all

src/librustc_lexer/src/lib.rs

@@ -29,7 +29,7 @@ mod tests;
 use self::LiteralKind::*;
 use self::TokenKind::*;
 use crate::cursor::{Cursor, EOF_CHAR};
-use std::convert::TryInto;
+use std::convert::TryFrom;
 
 /// Parsed token.
 /// It doesn't contain information about data that has been parsed,
@@ -142,84 +142,24 @@ pub enum LiteralKind {
     /// "b"abc"", "b"abc"
     ByteStr { terminated: bool },
     /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a"
-    RawStr(UnvalidatedRawStr),
+    RawStr { n_hashes: u16, err: Option<RawStrError> },
     /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a"
-    RawByteStr(UnvalidatedRawStr),
+    RawByteStr { n_hashes: u16, err: Option<RawStrError> },
-}
-
-/// Represents something that looks like a raw string, but may have some
-/// problems. Use `.validate()` to convert it into something
-/// usable.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub struct UnvalidatedRawStr {
-    /// The prefix (`r###"`) is valid
-    valid_start: bool,
-
-    /// The postfix (`"###`) is valid
-    valid_end: bool,
-
-    /// The number of leading `#`
-    n_start_hashes: usize,
-    /// The number of trailing `#`. `n_end_hashes` <= `n_start_hashes`
-    n_end_hashes: usize,
-    /// The offset starting at `r` or `br` where the user may have intended to end the string.
-    /// Currently, it is the longest sequence of pattern `"#+"`.
-    possible_terminator_offset: Option<usize>,
 }
 
 /// Error produced validating a raw string. Represents cases like:
-/// - `r##~"abcde"##`: `LexRawStrError::InvalidStarter`
+/// - `r##~"abcde"##`: `InvalidStarter`
-/// - `r###"abcde"##`: `LexRawStrError::NoTerminator { expected: 3, found: 2, possible_terminator_offset: Some(11)`
+/// - `r###"abcde"##`: `NoTerminator { expected: 3, found: 2, possible_terminator_offset: Some(11)`
-/// - Too many `#`s (>65536): `TooManyDelimiters`
+/// - Too many `#`s (>65535): `TooManyDelimiters`
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub enum LexRawStrError {
+pub enum RawStrError {
     /// Non `#` characters exist between `r` and `"` eg. `r#~"..`
-    InvalidStarter,
+    InvalidStarter { bad_char: char },
     /// The string was never terminated. `possible_terminator_offset` is the number of characters after `r` or `br` where they
     /// may have intended to terminate it.
     NoTerminator { expected: usize, found: usize, possible_terminator_offset: Option<usize> },
-    /// More than 65536 `#`s exist.
+    /// More than 65535 `#`s exist.
-    TooManyDelimiters,
+    TooManyDelimiters { found: usize },
-}
-
-/// Raw String that contains a valid prefix (`#+"`) and postfix (`"#+`) where
-/// there are a matching number of `#` characters in both. Note that this will
-/// not consume extra trailing `#` characters: `r###"abcde"####` is lexed as a
-/// `ValidatedRawString { n_hashes: 3 }` followed by a `#` token.
-#[derive(Debug, Eq, PartialEq, Copy, Clone)]
-pub struct ValidatedRawStr {
-    n_hashes: u16,
-}
-
-impl ValidatedRawStr {
-    pub fn num_hashes(&self) -> u16 {
-        self.n_hashes
-    }
-}
-
-impl UnvalidatedRawStr {
-    pub fn validate(self) -> Result<ValidatedRawStr, LexRawStrError> {
-        if !self.valid_start {
-            return Err(LexRawStrError::InvalidStarter);
-        }
-
-        // Only up to 65535 `#`s are allowed in raw strings
-        let n_start_safe: u16 =
-            self.n_start_hashes.try_into().map_err(|_| LexRawStrError::TooManyDelimiters)?;
-
-        if self.n_start_hashes > self.n_end_hashes || !self.valid_end {
-            Err(LexRawStrError::NoTerminator {
-                expected: self.n_start_hashes,
-                found: self.n_end_hashes,
-                possible_terminator_offset: self.possible_terminator_offset,
-            })
-        } else {
-            // Since the lexer should never produce a literal with n_end > n_start, if n_start <= n_end,
-            // they must be equal.
-            debug_assert_eq!(self.n_start_hashes, self.n_end_hashes);
-            Ok(ValidatedRawStr { n_hashes: n_start_safe })
-        }
-    }
 }
 
 /// Base of numeric literal encoding according to its prefix.
@@ -354,12 +294,12 @@ impl Cursor<'_> {
             'r' => match (self.first(), self.second()) {
                 ('#', c1) if is_id_start(c1) => self.raw_ident(),
                 ('#', _) | ('"', _) => {
-                    let raw_str_i = self.raw_double_quoted_string(1);
+                    let (n_hashes, err) = self.raw_double_quoted_string(1);
                     let suffix_start = self.len_consumed();
-                    if raw_str_i.n_end_hashes == raw_str_i.n_start_hashes {
+                    if err.is_none() {
                         self.eat_literal_suffix();
                     }
-                    let kind = RawStr(raw_str_i);
+                    let kind = RawStr { n_hashes, err };
                     Literal { kind, suffix_start }
                 }
                 _ => self.ident(),
@@ -389,14 +329,12 @@ impl Cursor<'_> {
                 }
                 ('r', '"') | ('r', '#') => {
                     self.bump();
-                    let raw_str_i = self.raw_double_quoted_string(2);
+                    let (n_hashes, err) = self.raw_double_quoted_string(2);
                     let suffix_start = self.len_consumed();
-                    let terminated = raw_str_i.n_start_hashes == raw_str_i.n_end_hashes;
+                    if err.is_none() {
-                    if terminated {
                         self.eat_literal_suffix();
                     }
-
+                    let kind = RawByteStr { n_hashes, err };
-                    let kind = RawByteStr(raw_str_i);
                     Literal { kind, suffix_start }
                 }
                 _ => self.ident(),
@@ -692,27 +630,34 @@ impl Cursor<'_> {
         false
     }
 
-    /// Eats the double-quoted string and returns an `UnvalidatedRawStr`.
+    /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
-    fn raw_double_quoted_string(&mut self, prefix_len: usize) -> UnvalidatedRawStr {
+    fn raw_double_quoted_string(&mut self, prefix_len: usize) -> (u16, Option<RawStrError>) {
+        // Wrap the actual function to handle the error with too many hashes.
+        // This way, it eats the whole raw string.
+        let (n_hashes, err) = self.raw_string_unvalidated(prefix_len);
+        // Only up to 65535 `#`s are allowed in raw strings
+        match u16::try_from(n_hashes) {
+            Ok(num) => (num, err),
+            // We lie about the number of hashes here :P
+            Err(_) => (0, Some(RawStrError::TooManyDelimiters { found: n_hashes })),
+        }
+    }
+
+    fn raw_string_unvalidated(&mut self, prefix_len: usize) -> (usize, Option<RawStrError>) {
         debug_assert!(self.prev() == 'r');
-        let mut valid_start: bool = false;
         let start_pos = self.len_consumed();
-        let (mut possible_terminator_offset, mut max_hashes) = (None, 0);
+        let mut possible_terminator_offset = None;
+        let mut max_hashes = 0;
 
         // Count opening '#' symbols.
         let n_start_hashes = self.eat_while(|c| c == '#');
 
         // Check that string is started.
         match self.bump() {
-            Some('"') => valid_start = true,
+            Some('"') => (),
-            _ => {
+            c => {
-                return UnvalidatedRawStr {
+                let c = c.unwrap_or(EOF_CHAR);
-                    valid_start,
+                return (n_start_hashes, Some(RawStrError::InvalidStarter { bad_char: c }));
-                    valid_end: false,
-                    n_start_hashes,
-                    n_end_hashes: 0,
-                    possible_terminator_offset,
-                };
             }
         }
 
@@ -722,13 +667,14 @@ impl Cursor<'_> {
             self.eat_while(|c| c != '"');
 
             if self.is_eof() {
-                return UnvalidatedRawStr {
+                return (
-                    valid_start,
-                    valid_end: false,
                     n_start_hashes,
-                    n_end_hashes: max_hashes,
+                    Some(RawStrError::NoTerminator {
-                    possible_terminator_offset,
+                        expected: n_start_hashes,
-                };
+                        found: max_hashes,
+                        possible_terminator_offset,
+                    }),
+                );
             }
 
             // Eat closing double quote.
@@ -737,7 +683,7 @@ impl Cursor<'_> {
             // Check that amount of closing '#' symbols
             // is equal to the amount of opening ones.
             // Note that this will not consume extra trailing `#` characters:
-            // `r###"abcde"####` is lexed as a `LexedRawString { n_hashes: 3 }`
+            // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
             // followed by a `#` token.
             let mut hashes_left = n_start_hashes;
             let is_closing_hash = |c| {
@@ -751,13 +697,7 @@ impl Cursor<'_> {
             let n_end_hashes = self.eat_while(is_closing_hash);
 
             if n_end_hashes == n_start_hashes {
-                return UnvalidatedRawStr {
+                return (n_start_hashes, None);
-                    valid_start,
-                    valid_end: true,
-                    n_start_hashes,
-                    n_end_hashes,
-                    possible_terminator_offset: None,
-                };
             } else if n_end_hashes > max_hashes {
                 // Keep track of possible terminators to give a hint about
                 // where there might be a missing terminator

src/librustc_lexer/src/tests.rs

@@ -2,77 +2,37 @@
 mod tests {
     use crate::*;
 
-    fn check_raw_str(
+    fn check_raw_str(s: &str, expected_hashes: u16, expected_err: Option<RawStrError>) {
-        s: &str,
-        expected: UnvalidatedRawStr,
-        validated: Result<ValidatedRawStr, LexRawStrError>,
-    ) {
         let s = &format!("r{}", s);
         let mut cursor = Cursor::new(s);
         cursor.bump();
-        let tok = cursor.raw_double_quoted_string(0);
+        let (n_hashes, err) = cursor.raw_double_quoted_string(0);
-        assert_eq!(tok, expected);
+        assert_eq!(n_hashes, expected_hashes);
-        assert_eq!(tok.validate(), validated);
+        assert_eq!(err, expected_err);
     }
 
     #[test]
     fn test_naked_raw_str() {
-        check_raw_str(
+        check_raw_str(r#""abc""#, 0, None);
-            r#""abc""#,
-            UnvalidatedRawStr {
-                n_start_hashes: 0,
-                n_end_hashes: 0,
-                valid_start: true,
-                valid_end: true,
-                possible_terminator_offset: None,
-            },
-            Ok(ValidatedRawStr { n_hashes: 0 }),
-        );
     }
 
     #[test]
     fn test_raw_no_start() {
-        check_raw_str(
+        check_raw_str(r##""abc"#"##, 0, None);
-            r##""abc"#"##,
-            UnvalidatedRawStr {
-                n_start_hashes: 0,
-                n_end_hashes: 0,
-                valid_start: true,
-                valid_end: true,
-                possible_terminator_offset: None,
-            },
-            Ok(ValidatedRawStr { n_hashes: 0 }),
-        );
     }
 
     #[test]
     fn test_too_many_terminators() {
         // this error is handled in the parser later
-        check_raw_str(
+        check_raw_str(r###"#"abc"##"###, 1, None);
-            r###"#"abc"##"###,
-            UnvalidatedRawStr {
-                n_start_hashes: 1,
-                n_end_hashes: 1,
-                valid_end: true,
-                valid_start: true,
-                possible_terminator_offset: None,
-            },
-            Ok(ValidatedRawStr { n_hashes: 1 }),
-        );
     }
 
     #[test]
     fn test_unterminated() {
         check_raw_str(
             r#"#"abc"#,
-            UnvalidatedRawStr {
+            1,
-                n_start_hashes: 1,
+            Some(RawStrError::NoTerminator {
-                n_end_hashes: 0,
-                valid_end: false,
-                valid_start: true,
-                possible_terminator_offset: None,
-            },
-            Err(LexRawStrError::NoTerminator {
                 expected: 1,
                 found: 0,
                 possible_terminator_offset: None,
@@ -80,14 +40,8 @@ mod tests {
         );
         check_raw_str(
             r###"##"abc"#"###,
-            UnvalidatedRawStr {
+            2,
-                n_start_hashes: 2,
+            Some(RawStrError::NoTerminator {
-                n_end_hashes: 1,
-                valid_start: true,
-                valid_end: false,
-                possible_terminator_offset: Some(7),
-            },
-            Err(LexRawStrError::NoTerminator {
                 expected: 2,
                 found: 1,
                 possible_terminator_offset: Some(7),
@@ -96,14 +50,8 @@ mod tests {
         // We're looking for "# not just any #
         check_raw_str(
             r###"##"abc#"###,
-            UnvalidatedRawStr {
+            2,
-                n_start_hashes: 2,
+            Some(RawStrError::NoTerminator {
-                n_end_hashes: 0,
-                valid_start: true,
-                valid_end: false,
-                possible_terminator_offset: None,
-            },
-            Err(LexRawStrError::NoTerminator {
                 expected: 2,
                 found: 0,
                 possible_terminator_offset: None,
@@ -113,17 +61,7 @@ mod tests {
 
     #[test]
     fn test_invalid_start() {
-        check_raw_str(
+        check_raw_str(r##"#~"abc"#"##, 1, Some(RawStrError::InvalidStarter { bad_char: '~' }));
-            r##"#~"abc"#"##,
-            UnvalidatedRawStr {
-                n_start_hashes: 1,
-                n_end_hashes: 0,
-                valid_start: false,
-                valid_end: false,
-                possible_terminator_offset: None,
-            },
-            Err(LexRawStrError::InvalidStarter),
-        );
     }
 
     #[test]
@@ -131,14 +69,8 @@ mod tests {
         // https://github.com/rust-lang/rust/issues/70677
         check_raw_str(
             r#"""#,
-            UnvalidatedRawStr {
+            0,
-                n_start_hashes: 0,
+            Some(RawStrError::NoTerminator {
-                n_end_hashes: 0,
-                valid_start: true,
-                valid_end: false,
-                possible_terminator_offset: None,
-            },
-            Err(LexRawStrError::NoTerminator {
                 expected: 0,
                 found: 0,
                 possible_terminator_offset: None,

src/librustc_parse/lexer/mod.rs

@@ -3,7 +3,7 @@ use rustc_ast::util::comments;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{error_code, Applicability, DiagnosticBuilder, FatalError};
 use rustc_lexer::Base;
-use rustc_lexer::{unescape, LexRawStrError, UnvalidatedRawStr, ValidatedRawStr};
+use rustc_lexer::{unescape, RawStrError};
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{BytePos, Pos, Span};
@@ -49,13 +49,12 @@ impl<'a> StringReader<'a> {
         // Make sure external source is loaded first, before accessing it.
         // While this can't show up during normal parsing, `retokenize` may
         // be called with a source file from an external crate.
-        sess.source_map().ensure_source_file_source_present(source_file.clone());
+        sess.source_map().ensure_source_file_source_present(Lrc::clone(&source_file));
 
-        // FIXME(eddyb) use `Lrc<str>` or similar to avoid cloning the `String`.
         let src = if let Some(src) = &source_file.src {
-            src.clone()
+            Lrc::clone(&src)
         } else if let Some(src) = source_file.external_src.borrow().get_source() {
-            src.clone()
+            Lrc::clone(&src)
         } else {
             sess.span_diagnostic
                 .bug(&format!("cannot lex `source_file` without source: {}", source_file.name));
@@ -125,10 +124,7 @@ impl<'a> StringReader<'a> {
 
         debug!("try_next_token: {:?}({:?})", token.kind, self.str_from(start));
 
-        // This could use `?`, but that makes code significantly (10-20%) slower.
-        // https://github.com/rust-lang/rust/issues/37939
         let kind = self.cook_lexer_token(token.kind, start);
-
         let span = self.mk_sp(start, self.pos);
         Token::new(kind, span)
     }
@@ -153,15 +149,6 @@ impl<'a> StringReader<'a> {
         self.err_span(self.mk_sp(from_pos, to_pos), m)
     }
 
-    fn struct_span_fatal(
-        &self,
-        from_pos: BytePos,
-        to_pos: BytePos,
-        m: &str,
-    ) -> DiagnosticBuilder<'a> {
-        self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), m)
-    }
-
     fn struct_fatal_span_char(
         &self,
         from_pos: BytePos,
@@ -359,15 +346,13 @@ impl<'a> StringReader<'a> {
                 }
                 (token::ByteStr, Mode::ByteStr, 2, 1) // b" "
             }
-            rustc_lexer::LiteralKind::RawStr(unvalidated_raw_str) => {
+            rustc_lexer::LiteralKind::RawStr { n_hashes, err } => {
-                let valid_raw_str = self.validate_and_report_errors(start, unvalidated_raw_str);
+                self.report_raw_str_error(start, err);
-                let n_hashes = valid_raw_str.num_hashes();
                 let n = u32::from(n_hashes);
                 (token::StrRaw(n_hashes), Mode::RawStr, 2 + n, 1 + n) // r##" "##
             }
-            rustc_lexer::LiteralKind::RawByteStr(unvalidated_raw_str) => {
+            rustc_lexer::LiteralKind::RawByteStr { n_hashes, err } => {
-                let validated_raw_str = self.validate_and_report_errors(start, unvalidated_raw_str);
+                self.report_raw_str_error(start, err);
-                let n_hashes = validated_raw_str.num_hashes();
                 let n = u32::from(n_hashes);
                 (token::ByteStrRaw(n_hashes), Mode::RawByteStr, 3 + n, 1 + n) // br##" "##
             }
@@ -382,12 +367,7 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::Float { base, empty_exponent } => {
                 if empty_exponent {
-                    let mut err = self.struct_span_fatal(
+                    self.err_span_(start, self.pos, "expected at least one digit in exponent");
-                        start,
-                        self.pos,
-                        "expected at least one digit in exponent",
-                    );
-                    err.emit();
                 }
 
                 match base {
@@ -459,33 +439,25 @@ impl<'a> StringReader<'a> {
         }
     }
 
-    fn validate_and_report_errors(
+    fn report_raw_str_error(&self, start: BytePos, opt_err: Option<RawStrError>) {
-        &self,
+        match opt_err {
-        start: BytePos,
+            Some(RawStrError::InvalidStarter { bad_char }) => {
-        unvalidated_raw_str: UnvalidatedRawStr,
+                self.report_non_started_raw_string(start, bad_char)
-    ) -> ValidatedRawStr {
-        match unvalidated_raw_str.validate() {
-            Err(LexRawStrError::InvalidStarter) => self.report_non_started_raw_string(start),
-            Err(LexRawStrError::NoTerminator { expected, found, possible_terminator_offset }) => {
-                self.report_unterminated_raw_string(
-                    start,
-                    expected,
-                    possible_terminator_offset,
-                    found,
-                )
             }
-            Err(LexRawStrError::TooManyDelimiters) => self.report_too_many_hashes(start),
+            Some(RawStrError::NoTerminator { expected, found, possible_terminator_offset }) => self
-            Ok(valid) => valid,
+                .report_unterminated_raw_string(start, expected, possible_terminator_offset, found),
+            Some(RawStrError::TooManyDelimiters { found }) => {
+                self.report_too_many_hashes(start, found)
+            }
+            None => (),
         }
     }
 
-    fn report_non_started_raw_string(&self, start: BytePos) -> ! {
+    fn report_non_started_raw_string(&self, start: BytePos, bad_char: char) -> ! {
-        let bad_char = self.str_from(start).chars().last().unwrap();
         self.struct_fatal_span_char(
             start,
             self.pos,
-            "found invalid character; only `#` is allowed \
+            "found invalid character; only `#` is allowed in raw string delimitation",
-                 in raw string delimitation",
             bad_char,
         )
         .emit();
@@ -530,11 +502,17 @@ impl<'a> StringReader<'a> {
         FatalError.raise()
     }
 
-    fn report_too_many_hashes(&self, start: BytePos) -> ! {
+    /// Note: It was decided to not add a test case, because it would be to big.
+    /// https://github.com/rust-lang/rust/pull/50296#issuecomment-392135180
+    fn report_too_many_hashes(&self, start: BytePos, found: usize) -> ! {
         self.fatal_span_(
             start,
             self.pos,
-            "too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
+            &format!(
+                "too many `#` symbols: raw strings may be delimited \
+                by up to 65535 `#` symbols, but found {}",
+                found
+            ),
         )
         .raise();
     }

src/libstd/io/mod.rs

@@ -1883,6 +1883,10 @@ pub trait BufRead: Read {
     ///
     /// If successful, this function will return the total number of bytes read.
     ///
+    /// This function is blocking and should be used carefully: it is possible for
+    /// an attacker to continuously send bytes without ever sending the delimiter
+    /// or EOF.
+    ///
     /// # Errors
     ///
     /// This function will ignore all instances of [`ErrorKind::Interrupted`] and
@@ -1945,6 +1949,10 @@ pub trait BufRead: Read {
     ///
     /// If this function returns `Ok(0)`, the stream has reached EOF.
     ///
+    /// This function is blocking and should be used carefully: it is possible for
+    /// an attacker to continuously send bytes without ever sending a newline
+    /// or EOF.
+    ///
     /// # Errors
     ///
     /// This function has the same error semantics as [`read_until`] and will