Revert #96682.

The change was "Show invisible delimiters (within comments) when pretty
printing". It's useful to show these delimiters, but is a breaking
change for some proc macros.

Fixes #97608.

compiler/rustc_ast/src/token.rs

@@ -50,12 +50,11 @@ pub enum Delimiter {
     Brace,
     /// `[ ... ]`
     Bracket,
-    /// `/*«*/ ... /*»*/`
+    /// `Ø ... Ø`
     /// An invisible delimiter, that may, for example, appear around tokens coming from a
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
-    /// Invisible delimiters are not directly writable in normal Rust code except as comments.
+    /// Invisible delimiters might not survive roundtrip of a token stream through a string.
-    /// Therefore, they might not survive a roundtrip of a token stream through a string.
     Invisible,
 }
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -590,29 +590,15 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
                     self.nbsp();
                 }
                 self.word("{");
-                let empty = tts.is_empty();
+                if !tts.is_empty() {
-                if !empty {
                     self.space();
                 }
                 self.ibox(0);
                 self.print_tts(tts, convert_dollar_crate);
                 self.end();
+                let empty = tts.is_empty();
                 self.bclose(span, empty);
             }
-            Some(Delimiter::Invisible) => {
-                self.word("/*«*/");
-                let empty = tts.is_empty();
-                if !empty {
-                    self.space();
-                }
-                self.ibox(0);
-                self.print_tts(tts, convert_dollar_crate);
-                self.end();
-                if !empty {
-                    self.space();
-                }
-                self.word("/*»*/");
-            }
             Some(delim) => {
                 let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
                 self.word(token_str);
@@ -786,8 +772,9 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::CloseDelim(Delimiter::Bracket) => "]".into(),
             token::OpenDelim(Delimiter::Brace) => "{".into(),
             token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible) => "/*«*/".into(),
+            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
-            token::CloseDelim(Delimiter::Invisible) => "/*»*/".into(),
+                "".into()
+            }
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

library/proc_macro/src/lib.rs

@@ -703,12 +703,11 @@ pub enum Delimiter {
     /// `[ ... ]`
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     Bracket,
-    /// `/*«*/ ... /*»*/`
+    /// `Ø ... Ø`
     /// An invisible delimiter, that may, for example, appear around tokens coming from a
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
-    /// Invisible delimiters are not directly writable in normal Rust code except as comments.
+    /// Invisible delimiters might not survive roundtrip of a token stream through a string.
-    /// Therefore, they might not survive a roundtrip of a token stream through a string.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     None,
 }

src/test/ui/proc-macro/auxiliary/expand-expr.rs

@@ -12,15 +12,6 @@ use std::str::FromStr;
 
 #[proc_macro]
 pub fn expand_expr_is(input: TokenStream) -> TokenStream {
-    expand_expr_is_inner(input, false)
-}
-
-#[proc_macro]
-pub fn expand_expr_is_trim(input: TokenStream) -> TokenStream {
-    expand_expr_is_inner(input, true)
-}
-
-fn expand_expr_is_inner(input: TokenStream, trim_invisible: bool) -> TokenStream {
     let mut iter = input.into_iter();
     let mut expected_tts = Vec::new();
     loop {
@@ -31,18 +22,14 @@ fn expand_expr_is_inner(input: TokenStream, trim_invisible: bool) -> TokenStream
         }
     }
 
-    // If requested, trim the "invisible" delimiters at the start and end.
+    let expected = expected_tts.into_iter().collect::<TokenStream>();
-    let expected = expected_tts.into_iter().collect::<TokenStream>().to_string();
+    let expanded = iter.collect::<TokenStream>().expand_expr().expect("expand_expr failed");
-    let expected = if trim_invisible {
+    assert!(
-        let len1 = "/*«*/ ".len();
+        expected.to_string() == expanded.to_string(),
-        let len2 = " /*»*/".len();
+        "assert failed\nexpected: `{}`\nexpanded: `{}`",
-        &expected[len1..expected.len() - len2]
+        expected.to_string(),
-    } else {
+        expanded.to_string()
-        &expected[..]
+    );
-    };
-    let expanded = iter.collect::<TokenStream>().expand_expr().unwrap().to_string();
-
-    assert_eq!(expected, expanded);
 
     TokenStream::new()
 }

src/test/ui/proc-macro/capture-macro-rules-invoke.stdout

@@ -1,5 +1,4 @@
 PRINT-BANG INPUT (DISPLAY): self
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ self /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,
@@ -14,10 +13,8 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
 ]
 PRINT-BANG INPUT (DISPLAY): 1 + 1, { "a" }, let a = 1;, String, my_name, 'a, my_val = 30,
 std::option::Option, pub(in some::path) , [a b c], -30
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 1 + 1 /*»*/, /*«*/ { "a" } /*»*/, /*«*/ let a = 1 /*»*/, /*«*/
+PRINT-BANG RE-COLLECTED (DISPLAY): 1 + 1, { "a" }, let a = 1, String, my_name, 'a, my_val = 30,
-String /*»*/, my_name, /*«*/ 'a /*»*/, /*«*/ my_val = 30 /*»*/, /*«*/
+std :: option :: Option, pub(in some :: path), [a b c], - 30
-std :: option :: Option /*»*/, /*«*/ pub(in some :: path) /*»*/, [a b c],
-/*«*/ - 30 /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,
@@ -298,7 +295,6 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): (a, b)
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ (a, b) /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/capture-unglued-token.stdout

@@ -1,5 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): Vec<u8>
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ Vec < u8 > /*»*/
+PRINT-BANG RE-COLLECTED (DISPLAY): Vec < u8 >
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/expand-expr.rs

@@ -2,9 +2,9 @@
 
 extern crate expand_expr;
 
-use expand_expr::{check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is};
+use expand_expr::{
-use expand_expr::{expand_expr_is_trim, recursive_expand};
+    check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is, recursive_expand,
-
+};
 
 // Check builtin macros can be expanded.
 
@@ -47,21 +47,21 @@ macro_rules! echo_expr {
 
 macro_rules! simple_lit {
     ($l:literal) => {
-        expand_expr_is_trim!($l, $l);
+        expand_expr_is!($l, $l);
-        expand_expr_is_trim!($l, echo_lit!($l));
+        expand_expr_is!($l, echo_lit!($l));
-        expand_expr_is_trim!($l, echo_expr!($l));
+        expand_expr_is!($l, echo_expr!($l));
-        expand_expr_is_trim!($l, echo_tts!($l));
+        expand_expr_is!($l, echo_tts!($l));
-        expand_expr_is_trim!($l, echo_pm!($l));
+        expand_expr_is!($l, echo_pm!($l));
         const _: () = {
             macro_rules! mac {
                 () => {
                     $l
                 };
             }
-            expand_expr_is_trim!($l, mac!());
+            expand_expr_is!($l, mac!());
-            expand_expr_is_trim!($l, echo_expr!(mac!()));
+            expand_expr_is!($l, echo_expr!(mac!()));
-            expand_expr_is_trim!($l, echo_tts!(mac!()));
+            expand_expr_is!($l, echo_tts!(mac!()));
-            expand_expr_is_trim!($l, echo_pm!(mac!()));
+            expand_expr_is!($l, echo_pm!(mac!()));
         };
     };
 }

src/test/ui/proc-macro/expr-stmt-nonterminal-tokens.stdout

@@ -1,6 +1,5 @@
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = #[allow(warnings)] 0 ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = #[allow(warnings)] #[allow(warnings)] 0 ; 0 }, }
-{ V = { let _ = /*«*/ #[allow(warnings)] #[allow(warnings)] 0 /*»*/ ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -124,7 +123,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { 0; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ 0 /*»*/ } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { 0 } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -204,7 +203,6 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { {} } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ {} /*»*/ } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -283,7 +281,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { PATH; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ PATH /*»*/ } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { PATH } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -361,7 +359,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { 0 + 1; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ 0 + 1 /*»*/ } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { 0 + 1 } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -452,7 +450,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { PATH + 1; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ PATH + 1 /*»*/ } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { PATH + 1 } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",

src/test/ui/proc-macro/issue-75734-pp-paren.stdout

@@ -96,7 +96,6 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): 1 + 1 * 2
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 1 + 1 /*»*/ * 2
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/issue-78675-captured-inner-attrs.stdout

@@ -1,7 +1,7 @@
 PRINT-BANG INPUT (DISPLAY): foo! { #[fake_attr] mod bar {
     #![doc = r" Foo"]
 } }
-PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): foo! { #[fake_attr] /*«*/ mod bar { #! [doc = r" Foo"] } /*»*/ }
+PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): foo! { #[fake_attr] mod bar { #! [doc = r" Foo"] } }
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Ident {
         ident: "foo",

src/test/ui/proc-macro/issue-80760-empty-stmt.stdout

@@ -1,5 +1,4 @@
 PRINT-BANG INPUT (DISPLAY): ;
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ ; /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/nested-nonterminal-tokens.stdout

@@ -1,6 +1,4 @@
 PRINT-BANG INPUT (DISPLAY): 0 + 1 + 2 + 3
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 0 + 1 + 2 /*»*/ + 3
-PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): /*«*/ /*«*/ /*«*/ 0 /*»*/ + 1 /*»*/ + 2 /*»*/ + 3
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/nodelim-groups.stdout

@@ -1,5 +1,4 @@
 PRINT-BANG INPUT (DISPLAY): "hi" 1 + (25) + 1 (1 + 1)
-PRINT-BANG RE-COLLECTED (DISPLAY): "hi" /*«*/ 1 + (25) + 1 /*»*/ (1 + 1)
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Literal {
         kind: Str,
@@ -72,9 +71,6 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): "hi" "hello".len() + "world".len() (1 + 1)
-PRINT-BANG RE-COLLECTED (DISPLAY): "hi" /*«*/ "hello".len() + "world".len() /*»*/ (1 + 1)
-PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): "hi" /*«*/ /*«*/ "hello".len() /*»*/ + /*«*/ "world".len() /*»*/ /*»*/
-(1 + 1)
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Literal {
         kind: Str,

src/test/ui/proc-macro/nonterminal-expansion.stdout

@@ -1,5 +1,5 @@
 PRINT-ATTR_ARGS INPUT (DISPLAY): a, line!(), b
-PRINT-ATTR_ARGS RE-COLLECTED (DISPLAY): a, /*«*/ line! () /*»*/, b
+PRINT-ATTR_ARGS RE-COLLECTED (DISPLAY): a, line! (), b
 PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [
     Ident {
         ident: "a",

src/test/ui/proc-macro/nonterminal-token-hygiene.stdout

@@ -1,5 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): struct S;
-PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ struct S ; /*»*/
+PRINT-BANG RE-COLLECTED (DISPLAY): struct S ;
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/parent-source-spans.rs

@@ -8,16 +8,16 @@ use parent_source_spans::parent_source_spans;
 
 macro one($a:expr, $b:expr) {
     two!($a, $b);
-    //~^ ERROR first parent: /*«*/ "hello" /*»*/
+    //~^ ERROR first parent: "hello"
-    //~| ERROR second parent: /*«*/ "world" /*»*/
+    //~| ERROR second parent: "world"
 }
 
 macro two($a:expr, $b:expr) {
     three!($a, $b);
-    //~^ ERROR first final: /*«*/ "hello" /*»*/
+    //~^ ERROR first final: "hello"
-    //~| ERROR second final: /*«*/ "world" /*»*/
+    //~| ERROR second final: "world"
-    //~| ERROR first final: /*«*/ "yay" /*»*/
+    //~| ERROR first final: "yay"
-    //~| ERROR second final: /*«*/ "rust" /*»*/
+    //~| ERROR second final: "rust"
 }
 
 // forwarding tokens directly doesn't create a new source chain
@@ -34,16 +34,16 @@ macro four($($tokens:tt)*) {
 
 fn main() {
     one!("hello", "world");
-    //~^ ERROR first grandparent: /*«*/ "hello" /*»*/
+    //~^ ERROR first grandparent: "hello"
-    //~| ERROR second grandparent: /*«*/ "world" /*»*/
+    //~| ERROR second grandparent: "world"
-    //~| ERROR first source: /*«*/ "hello" /*»*/
+    //~| ERROR first source: "hello"
-    //~| ERROR second source: /*«*/ "world" /*»*/
+    //~| ERROR second source: "world"
 
     two!("yay", "rust");
-    //~^ ERROR first parent: /*«*/ "yay" /*»*/
+    //~^ ERROR first parent: "yay"
-    //~| ERROR second parent: /*«*/ "rust" /*»*/
+    //~| ERROR second parent: "rust"
-    //~| ERROR first source: /*«*/ "yay" /*»*/
+    //~| ERROR first source: "yay"
-    //~| ERROR second source: /*«*/ "rust" /*»*/
+    //~| ERROR second source: "rust"
 
     three!("hip", "hop");
     //~^ ERROR first final: "hip"

src/test/ui/proc-macro/parent-source-spans.stderr

@@ -1,4 +1,4 @@
-error: first final: /*«*/ "hello" /*»*/
+error: first final: "hello"
   --> $DIR/parent-source-spans.rs:16:12
    |
 LL |     three!($a, $b);
@@ -9,7 +9,7 @@ LL |     one!("hello", "world");
    |
    = note: this error originates in the macro `two` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: second final: /*«*/ "world" /*»*/
+error: second final: "world"
   --> $DIR/parent-source-spans.rs:16:16
    |
 LL |     three!($a, $b);
@@ -20,7 +20,7 @@ LL |     one!("hello", "world");
    |
    = note: this error originates in the macro `two` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: first parent: /*«*/ "hello" /*»*/
+error: first parent: "hello"
   --> $DIR/parent-source-spans.rs:10:5
    |
 LL |     two!($a, $b);
@@ -31,7 +31,7 @@ LL |     one!("hello", "world");
    |
    = note: this error originates in the macro `one` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: second parent: /*«*/ "world" /*»*/
+error: second parent: "world"
   --> $DIR/parent-source-spans.rs:10:5
    |
 LL |     two!($a, $b);
@@ -42,31 +42,31 @@ LL |     one!("hello", "world");
    |
    = note: this error originates in the macro `one` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: first grandparent: /*«*/ "hello" /*»*/
+error: first grandparent: "hello"
   --> $DIR/parent-source-spans.rs:36:5
    |
 LL |     one!("hello", "world");
    |     ^^^^^^^^^^^^^^^^^^^^^^
 
-error: second grandparent: /*«*/ "world" /*»*/
+error: second grandparent: "world"
   --> $DIR/parent-source-spans.rs:36:5
    |
 LL |     one!("hello", "world");
    |     ^^^^^^^^^^^^^^^^^^^^^^
 
-error: first source: /*«*/ "hello" /*»*/
+error: first source: "hello"
   --> $DIR/parent-source-spans.rs:36:5
    |
 LL |     one!("hello", "world");
    |     ^^^^^^^^^^^^^^^^^^^^^^
 
-error: second source: /*«*/ "world" /*»*/
+error: second source: "world"
   --> $DIR/parent-source-spans.rs:36:5
    |
 LL |     one!("hello", "world");
    |     ^^^^^^^^^^^^^^^^^^^^^^
 
-error: first final: /*«*/ "yay" /*»*/
+error: first final: "yay"
   --> $DIR/parent-source-spans.rs:16:12
    |
 LL |     three!($a, $b);
@@ -77,7 +77,7 @@ LL |     two!("yay", "rust");
    |
    = note: this error originates in the macro `two` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: second final: /*«*/ "rust" /*»*/
+error: second final: "rust"
   --> $DIR/parent-source-spans.rs:16:16
    |
 LL |     three!($a, $b);
@@ -88,25 +88,25 @@ LL |     two!("yay", "rust");
    |
    = note: this error originates in the macro `two` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: first parent: /*«*/ "yay" /*»*/
+error: first parent: "yay"
   --> $DIR/parent-source-spans.rs:42:5
    |
 LL |     two!("yay", "rust");
    |     ^^^^^^^^^^^^^^^^^^^
 
-error: second parent: /*«*/ "rust" /*»*/
+error: second parent: "rust"
   --> $DIR/parent-source-spans.rs:42:5
    |
 LL |     two!("yay", "rust");
    |     ^^^^^^^^^^^^^^^^^^^
 
-error: first source: /*«*/ "yay" /*»*/
+error: first source: "yay"
   --> $DIR/parent-source-spans.rs:42:5
    |
 LL |     two!("yay", "rust");
    |     ^^^^^^^^^^^^^^^^^^^
 
-error: second source: /*«*/ "rust" /*»*/
+error: second source: "rust"
   --> $DIR/parent-source-spans.rs:42:5
    |
 LL |     two!("yay", "rust");