Auto merge of #85359 - lrh2000:reserved-prefixes, r=nikomatsakis

Reserve prefixed identifiers and literals (RFC 3101)

This PR denies any identifiers immediately followed by one of three tokens `"`, `'` or `#`, which is stricter than the requirements of RFC 3101 but may be necessary according to the discussion at [Zulip].

[Zulip]: https://rust-lang.zulipchat.com/#narrow/stream/268952-edition-2021/topic/reserved.20prefixes/near/238470099

The tracking issue #84599 says we'll add a feature gate named `reserved_prefixes`, but I don't think I can do this because it is impossible for the lexer to know whether a feature is enabled or not. I guess determining the behavior by the edition information should be enough.

Fixes #84599
This commit is contained in:
bors 2021-06-27 20:33:25 +00:00
commit e8cb1a4a56
16 changed files with 518 additions and 9 deletions

View File

@ -66,6 +66,13 @@ pub enum TokenKind {
Ident,
/// "r#ident"
RawIdent,
/// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the
/// prefix (`foo`) is included in the token, not the separator (which is
/// lexed as its own distinct token). In Rust 2021 and later, reserved
/// prefixes are reported as errors; in earlier editions, they result in a
/// (allowed by default) lint, and are treated as regular identifier
/// tokens.
UnknownPrefix,
/// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
Literal { kind: LiteralKind, suffix_start: usize },
/// "'a"
@ -323,7 +330,7 @@ impl Cursor<'_> {
let kind = RawStr { n_hashes, err };
Literal { kind, suffix_start }
}
_ => self.ident(),
_ => self.ident_or_unknown_prefix(),
},
// Byte literal, byte string literal, raw byte string literal or identifier.
@ -358,12 +365,12 @@ impl Cursor<'_> {
let kind = RawByteStr { n_hashes, err };
Literal { kind, suffix_start }
}
_ => self.ident(),
_ => self.ident_or_unknown_prefix(),
},
// Identifier (this should be checked after other variant that can
// start as identifier).
c if is_id_start(c) => self.ident(),
c if is_id_start(c) => self.ident_or_unknown_prefix(),
// Numeric literal.
c @ '0'..='9' => {
@ -487,11 +494,16 @@ impl Cursor<'_> {
RawIdent
}
fn ident(&mut self) -> TokenKind {
fn ident_or_unknown_prefix(&mut self) -> TokenKind {
debug_assert!(is_id_start(self.prev()));
// Start is already eaten, eat the rest of identifier.
self.eat_while(is_id_continue);
Ident
// Known prefixes must have been handled earlier. So if
// we see a prefix here, it is definitely a unknown prefix.
match self.first() {
'#' | '"' | '\'' => UnknownPrefix,
_ => Ident,
}
}
fn number(&mut self, first_digit: char) -> LiteralKind {

View File

@ -723,6 +723,15 @@ pub trait LintContext: Sized {
BuiltinLintDiagnostics::OrPatternsBackCompat(span,suggestion) => {
db.span_suggestion(span, "use pat_param to preserve semantics", suggestion, Applicability::MachineApplicable);
}
BuiltinLintDiagnostics::ReservedPrefix(span) => {
db.span_label(span, "unknown prefix");
db.span_suggestion_verbose(
span.shrink_to_hi(),
"insert whitespace here to avoid this being parsed as a prefix in Rust 2021",
" ".into(),
Applicability::MachineApplicable,
);
}
}
// Rewrap `db`, and pass control to the user.
decorate(LintDiagnosticBuilder::new(db));

View File

@ -2973,6 +2973,7 @@ declare_lint_pass! {
OR_PATTERNS_BACK_COMPAT,
LARGE_ASSIGNMENTS,
FUTURE_PRELUDE_COLLISION,
RESERVED_PREFIX,
]
}
@ -3263,3 +3264,39 @@ declare_lint! {
reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
};
}
declare_lint! {
/// The `reserved_prefix` lint detects identifiers that will be parsed as a
/// prefix instead in Rust 2021.
///
/// ### Example
///
/// ```rust,compile_fail
/// #![deny(reserved_prefix)]
///
/// macro_rules! m {
/// (z $x:expr) => ();
/// }
///
/// m!(z"hey");
/// ```
///
/// {{produces}}
///
/// ### Explanation
///
/// In Rust 2015 and 2018, `z"hey"` is two tokens: the identifier `z`
/// followed by the string literal `"hey"`. In Rust 2021, the `z` is
/// considered a prefix for `"hey"`.
///
/// This lint suggests to add whitespace between the `z` and `"hey"` tokens
/// to keep them separated in Rust 2021.
pub RESERVED_PREFIX,
Allow,
"identifiers that will be parsed as a prefix in Rust 2021",
@future_incompatible = FutureIncompatibleInfo {
reference: "issue #84978 <https://github.com/rust-lang/rust/issues/84978>",
reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
};
crate_level_only
}

View File

@ -300,6 +300,7 @@ pub enum BuiltinLintDiagnostics {
ExternDepSpec(String, ExternDepSpec),
ProcMacroBackCompat(String),
OrPatternsBackCompat(Span, String),
ReservedPrefix(Span),
}
/// Lints that are buffered up early on in the `Session` before the

View File

@ -1,12 +1,14 @@
use rustc_ast::ast::AttrStyle;
use rustc_ast::ast::{self, AttrStyle};
use rustc_ast::token::{self, CommentKind, Token, TokenKind};
use rustc_ast::tokenstream::{Spacing, TokenStream};
use rustc_errors::{error_code, Applicability, DiagnosticBuilder, FatalError, PResult};
use rustc_lexer::unescape::{self, Mode};
use rustc_lexer::{Base, DocStyle, RawStrError};
use rustc_session::lint::builtin::RESERVED_PREFIX;
use rustc_session::lint::BuiltinLintDiagnostics;
use rustc_session::parse::ParseSess;
use rustc_span::symbol::{sym, Symbol};
use rustc_span::{BytePos, Pos, Span};
use rustc_span::{edition::Edition, BytePos, Pos, Span};
use tracing::debug;
@ -166,12 +168,18 @@ impl<'a> StringReader<'a> {
self.cook_doc_comment(content_start, content, CommentKind::Block, doc_style)
}
rustc_lexer::TokenKind::Whitespace => return None,
rustc_lexer::TokenKind::Ident | rustc_lexer::TokenKind::RawIdent => {
rustc_lexer::TokenKind::Ident
| rustc_lexer::TokenKind::RawIdent
| rustc_lexer::TokenKind::UnknownPrefix => {
let is_raw_ident = token == rustc_lexer::TokenKind::RawIdent;
let is_unknown_prefix = token == rustc_lexer::TokenKind::UnknownPrefix;
let mut ident_start = start;
if is_raw_ident {
ident_start = ident_start + BytePos(2);
}
if is_unknown_prefix {
self.report_unknown_prefix(start);
}
let sym = nfc_normalize(self.str_from(ident_start));
let span = self.mk_sp(start, self.pos);
self.sess.symbol_gallery.insert(sym, span);
@ -491,6 +499,42 @@ impl<'a> StringReader<'a> {
FatalError.raise()
}
// RFC 3101 introduced the idea of (reserved) prefixes. As of Rust 2021,
// using a (unknown) prefix is an error. In earlier editions, however, they
// only result in a (allowed by default) lint, and are treated as regular
// identifier tokens.
fn report_unknown_prefix(&self, start: BytePos) {
let prefix_span = self.mk_sp(start, self.pos);
let msg = format!("prefix `{}` is unknown", self.str_from_to(start, self.pos));
let expn_data = prefix_span.ctxt().outer_expn_data();
if expn_data.edition >= Edition::Edition2021 {
// In Rust 2021, this is a hard error.
let mut err = self.sess.span_diagnostic.struct_span_err(prefix_span, &msg);
err.span_label(prefix_span, "unknown prefix");
if expn_data.is_root() {
err.span_suggestion_verbose(
prefix_span.shrink_to_hi(),
"consider inserting whitespace here",
" ".into(),
Applicability::MachineApplicable,
);
}
err.note("prefixed identifiers and literals are reserved since Rust 2021");
err.emit();
} else {
// Before Rust 2021, only emit a lint for migration.
self.sess.buffer_lint_with_diagnostic(
&RESERVED_PREFIX,
prefix_span,
ast::CRATE_NODE_ID,
&msg,
BuiltinLintDiagnostics::ReservedPrefix(prefix_span),
);
}
}
/// Note: It was decided to not add a test case, because it would be too big.
/// <https://github.com/rust-lang/rust/pull/50296#issuecomment-392135180>
fn report_too_many_hashes(&self, start: BytePos, found: usize) -> ! {

View File

@ -413,7 +413,7 @@ impl<'a> Classifier<'a> {
},
c => c,
},
TokenKind::RawIdent => Class::Ident,
TokenKind::RawIdent | TokenKind::UnknownPrefix => Class::Ident,
TokenKind::Lifetime { .. } => Class::Lifetime,
};
// Anything that didn't return above is the simple case where we the

View File

@ -0,0 +1,25 @@
// force-host
// edition:2018
// no-prefer-dynamic
#![crate_type = "proc-macro"]
extern crate proc_macro;
use proc_macro::TokenStream;
use std::str::FromStr;
#[proc_macro]
pub fn number_of_tokens_in_a_prefixed_integer_literal(_: TokenStream) -> TokenStream {
TokenStream::from_str("hey#123").unwrap().into_iter().count().to_string().parse().unwrap()
}
#[proc_macro]
pub fn number_of_tokens_in_a_prefixed_char_literal(_: TokenStream) -> TokenStream {
TokenStream::from_str("hey#'a'").unwrap().into_iter().count().to_string().parse().unwrap()
}
#[proc_macro]
pub fn number_of_tokens_in_a_prefixed_string_literal(_: TokenStream) -> TokenStream {
TokenStream::from_str("hey#\"abc\"").unwrap().into_iter().count().to_string().parse().unwrap()
}

View File

@ -0,0 +1,25 @@
// force-host
// edition:2021
// no-prefer-dynamic
#![crate_type = "proc-macro"]
extern crate proc_macro;
use proc_macro::TokenStream;
use std::str::FromStr;
#[proc_macro]
pub fn number_of_tokens_in_a_prefixed_integer_literal(_: TokenStream) -> TokenStream {
TokenStream::from_str("hey#123").unwrap().into_iter().count().to_string().parse().unwrap()
}
#[proc_macro]
pub fn number_of_tokens_in_a_prefixed_char_literal(_: TokenStream) -> TokenStream {
TokenStream::from_str("hey#'a'").unwrap().into_iter().count().to_string().parse().unwrap()
}
#[proc_macro]
pub fn number_of_tokens_in_a_prefixed_string_literal(_: TokenStream) -> TokenStream {
TokenStream::from_str("hey#\"abc\"").unwrap().into_iter().count().to_string().parse().unwrap()
}

View File

@ -0,0 +1,38 @@
// check-pass
// run-rustfix
// compile-flags: -Z unstable-options --edition 2018
#![warn(reserved_prefix)]
macro_rules! m2 {
($a:tt $b:tt) => {};
}
macro_rules! m3 {
($a:tt $b:tt $c:tt) => {};
}
fn main() {
m2!(z "hey");
//~^ WARNING prefix `z` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
m2!(prefix "hey");
//~^ WARNING prefix `prefix` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
m3!(hey #123);
//~^ WARNING prefix `hey` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
m3!(hey #hey);
//~^ WARNING prefix `hey` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
}
macro_rules! quote {
(# name = # kind # value) => {};
}
quote! {
#name = #kind #value
//~^ WARNING prefix `kind` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
}

View File

@ -0,0 +1,38 @@
// check-pass
// run-rustfix
// compile-flags: -Z unstable-options --edition 2018
#![warn(reserved_prefix)]
macro_rules! m2 {
($a:tt $b:tt) => {};
}
macro_rules! m3 {
($a:tt $b:tt $c:tt) => {};
}
fn main() {
m2!(z"hey");
//~^ WARNING prefix `z` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
m2!(prefix"hey");
//~^ WARNING prefix `prefix` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
m3!(hey#123);
//~^ WARNING prefix `hey` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
m3!(hey#hey);
//~^ WARNING prefix `hey` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
}
macro_rules! quote {
(# name = # kind # value) => {};
}
quote! {
#name = #kind#value
//~^ WARNING prefix `kind` is unknown [reserved_prefix]
//~| WARNING hard error in Rust 2021
}

View File

@ -0,0 +1,72 @@
warning: prefix `z` is unknown
--> $DIR/reserved-prefixes-migration.rs:16:9
|
LL | m2!(z"hey");
| ^ unknown prefix
|
note: the lint level is defined here
--> $DIR/reserved-prefixes-migration.rs:5:9
|
LL | #![warn(reserved_prefix)]
| ^^^^^^^^^^^^^^^
= warning: this is accepted in the current edition (Rust 2018) but is a hard error in Rust 2021!
= note: for more information, see issue #84978 <https://github.com/rust-lang/rust/issues/84978>
help: insert whitespace here to avoid this being parsed as a prefix in Rust 2021
|
LL | m2!(z "hey");
| --
warning: prefix `prefix` is unknown
--> $DIR/reserved-prefixes-migration.rs:19:9
|
LL | m2!(prefix"hey");
| ^^^^^^ unknown prefix
|
= warning: this is accepted in the current edition (Rust 2018) but is a hard error in Rust 2021!
= note: for more information, see issue #84978 <https://github.com/rust-lang/rust/issues/84978>
help: insert whitespace here to avoid this being parsed as a prefix in Rust 2021
|
LL | m2!(prefix "hey");
| --
warning: prefix `hey` is unknown
--> $DIR/reserved-prefixes-migration.rs:22:9
|
LL | m3!(hey#123);
| ^^^ unknown prefix
|
= warning: this is accepted in the current edition (Rust 2018) but is a hard error in Rust 2021!
= note: for more information, see issue #84978 <https://github.com/rust-lang/rust/issues/84978>
help: insert whitespace here to avoid this being parsed as a prefix in Rust 2021
|
LL | m3!(hey #123);
| --
warning: prefix `hey` is unknown
--> $DIR/reserved-prefixes-migration.rs:25:9
|
LL | m3!(hey#hey);
| ^^^ unknown prefix
|
= warning: this is accepted in the current edition (Rust 2018) but is a hard error in Rust 2021!
= note: for more information, see issue #84978 <https://github.com/rust-lang/rust/issues/84978>
help: insert whitespace here to avoid this being parsed as a prefix in Rust 2021
|
LL | m3!(hey #hey);
| --
warning: prefix `kind` is unknown
--> $DIR/reserved-prefixes-migration.rs:35:14
|
LL | #name = #kind#value
| ^^^^ unknown prefix
|
= warning: this is accepted in the current edition (Rust 2018) but is a hard error in Rust 2021!
= note: for more information, see issue #84978 <https://github.com/rust-lang/rust/issues/84978>
help: insert whitespace here to avoid this being parsed as a prefix in Rust 2021
|
LL | #name = #kind #value
| --
warning: 5 warnings emitted

View File

@ -0,0 +1,21 @@
// edition:2018
// aux-build:reserved-prefixes-macro-2018.rs
// aux-build:reserved-prefixes-macro-2021.rs
extern crate reserved_prefixes_macro_2018 as m2018;
extern crate reserved_prefixes_macro_2021 as m2021;
fn main() {
// Ok:
m2018::number_of_tokens_in_a_prefixed_integer_literal!();
m2018::number_of_tokens_in_a_prefixed_char_literal!();
m2018::number_of_tokens_in_a_prefixed_string_literal!();
// Error, even though *this* crate is 2018:
m2021::number_of_tokens_in_a_prefixed_integer_literal!();
//~^ ERROR prefix `hey` is unknown
m2021::number_of_tokens_in_a_prefixed_char_literal!();
//~^ ERROR prefix `hey` is unknown
m2021::number_of_tokens_in_a_prefixed_string_literal!();
//~^ ERROR prefix `hey` is unknown
}

View File

@ -0,0 +1,29 @@
error: prefix `hey` is unknown
--> $DIR/reserved-prefixes-via-macro-2.rs:15:5
|
LL | m2021::number_of_tokens_in_a_prefixed_integer_literal!();
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
= note: this error originates in the macro `m2021::number_of_tokens_in_a_prefixed_integer_literal` (in Nightly builds, run with -Z macro-backtrace for more info)
error: prefix `hey` is unknown
--> $DIR/reserved-prefixes-via-macro-2.rs:17:5
|
LL | m2021::number_of_tokens_in_a_prefixed_char_literal!();
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
= note: this error originates in the macro `m2021::number_of_tokens_in_a_prefixed_char_literal` (in Nightly builds, run with -Z macro-backtrace for more info)
error: prefix `hey` is unknown
--> $DIR/reserved-prefixes-via-macro-2.rs:19:5
|
LL | m2021::number_of_tokens_in_a_prefixed_string_literal!();
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
= note: this error originates in the macro `m2021::number_of_tokens_in_a_prefixed_string_literal` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 3 previous errors

View File

@ -0,0 +1,12 @@
// run-pass
// edition:2021
// aux-build:reserved-prefixes-macro-2018.rs
extern crate reserved_prefixes_macro_2018 as m2018;
fn main() {
// Ok, even though *this* crate is 2021:
assert_eq!(m2018::number_of_tokens_in_a_prefixed_integer_literal!(), 3);
assert_eq!(m2018::number_of_tokens_in_a_prefixed_char_literal!(), 3);
assert_eq!(m2018::number_of_tokens_in_a_prefixed_string_literal!(), 3);
}

View File

@ -0,0 +1,36 @@
// compile-flags: -Z unstable-options --edition 2021
macro_rules! demo2 {
( $a:tt $b:tt ) => { println!("two tokens") };
}
macro_rules! demo3 {
( $a:tt $b:tt $c:tt ) => { println!("three tokens") };
}
macro_rules! demo4 {
( $a:tt $b:tt $c:tt $d:tt ) => { println!("four tokens") };
}
fn main() {
demo3!(foo#bar); //~ ERROR prefix `foo` is unknown
demo2!(foo"bar"); //~ ERROR prefix `foo` is unknown
demo2!(foo'b'); //~ ERROR prefix `foo` is unknown
demo2!(foo'b); //~ ERROR prefix `foo` is unknown
demo3!(foo# bar); //~ ERROR prefix `foo` is unknown
demo4!(foo#! bar); //~ ERROR prefix `foo` is unknown
demo4!(foo## bar); //~ ERROR prefix `foo` is unknown
demo4!(foo#bar#);
//~^ ERROR prefix `foo` is unknown
//~| ERROR prefix `bar` is unknown
demo3!(foo # bar);
demo3!(foo #bar);
demo4!(foo!#bar);
demo4!(foo ##bar);
demo3!(r"foo"#bar);
demo3!(r#foo#bar);
}

View File

@ -0,0 +1,110 @@
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:16:12
|
LL | demo3!(foo#bar);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo3!(foo #bar);
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:17:12
|
LL | demo2!(foo"bar");
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo2!(foo "bar");
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:18:12
|
LL | demo2!(foo'b');
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo2!(foo 'b');
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:20:12
|
LL | demo2!(foo'b);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo2!(foo 'b);
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:21:12
|
LL | demo3!(foo# bar);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo3!(foo # bar);
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:22:12
|
LL | demo4!(foo#! bar);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo4!(foo #! bar);
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:23:12
|
LL | demo4!(foo## bar);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo4!(foo ## bar);
| --
error: prefix `foo` is unknown
--> $DIR/reserved-prefixes.rs:25:12
|
LL | demo4!(foo#bar#);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo4!(foo #bar#);
| --
error: prefix `bar` is unknown
--> $DIR/reserved-prefixes.rs:25:16
|
LL | demo4!(foo#bar#);
| ^^^ unknown prefix
|
= note: prefixed identifiers and literals are reserved since Rust 2021
help: consider inserting whitespace here
|
LL | demo4!(foo#bar #);
| --
error: aborting due to 9 previous errors