Auto merge of #49124 - abonander:attr-macro-stmt-expr, r=abonander

Expand Attributes on Statements and Expressions

This enables attribute-macro expansion on statements and expressions while retaining the `stmt_expr_attributes` feature requirement for attributes on expressions.

closes #41475
cc #38356  @petrochenkov @jseyfried
r? @nrc

src/libsyntax/ast.rs

@@ -837,6 +837,13 @@ impl Stmt {
             _ => false,
         }
     }
+
+    pub fn is_expr(&self) -> bool {
+        match self.node {
+            StmtKind::Expr(_) => true,
+            _ => false,
+        }
+    }
 }
 
 impl fmt::Debug for Stmt {

src/libsyntax/config.rs

@@ -149,17 +149,24 @@ impl<'a> StripUnconfigured<'a> {
     fn visit_expr_attrs(&mut self, attrs: &[ast::Attribute]) {
         // flag the offending attributes
         for attr in attrs.iter() {
-            if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) {
-                let mut err = feature_err(self.sess,
-                                          "stmt_expr_attributes",
-                                          attr.span,
-                                          GateIssue::Language,
-                                          EXPLAIN_STMT_ATTR_SYNTAX);
-                if attr.is_sugared_doc {
-                    err.help("`///` is for documentation comments. For a plain comment, use `//`.");
-                }
-                err.emit();
+            self.maybe_emit_expr_attr_err(attr);
+        }
+    }
+
+    /// If attributes are not allowed on expressions, emit an error for `attr`
+    pub fn maybe_emit_expr_attr_err(&self, attr: &ast::Attribute) {
+        if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) {
+            let mut err = feature_err(self.sess,
+                                      "stmt_expr_attributes",
+                                      attr.span,
+                                      GateIssue::Language,
+                                      EXPLAIN_STMT_ATTR_SYNTAX);
+
+            if attr.is_sugared_doc {
+                err.help("`///` is for documentation comments. For a plain comment, use `//`.");
             }
+
+            err.emit();
         }
     }
 

src/libsyntax/ext/base.rs

@@ -38,6 +38,8 @@ pub enum Annotatable {
     Item(P<ast::Item>),
     TraitItem(P<ast::TraitItem>),
     ImplItem(P<ast::ImplItem>),
+    Stmt(P<ast::Stmt>),
+    Expr(P<ast::Expr>),
 }
 
 impl HasAttrs for Annotatable {
@@ -46,6 +48,8 @@ impl HasAttrs for Annotatable {
             Annotatable::Item(ref item) => &item.attrs,
             Annotatable::TraitItem(ref trait_item) => &trait_item.attrs,
             Annotatable::ImplItem(ref impl_item) => &impl_item.attrs,
+            Annotatable::Stmt(ref stmt) => stmt.attrs(),
+            Annotatable::Expr(ref expr) => &expr.attrs,
         }
     }
 
@@ -54,6 +58,8 @@ impl HasAttrs for Annotatable {
             Annotatable::Item(item) => Annotatable::Item(item.map_attrs(f)),
             Annotatable::TraitItem(trait_item) => Annotatable::TraitItem(trait_item.map_attrs(f)),
             Annotatable::ImplItem(impl_item) => Annotatable::ImplItem(impl_item.map_attrs(f)),
+            Annotatable::Stmt(stmt) => Annotatable::Stmt(stmt.map_attrs(f)),
+            Annotatable::Expr(expr) => Annotatable::Expr(expr.map_attrs(f)),
         }
     }
 }
@@ -64,6 +70,8 @@ impl Annotatable {
             Annotatable::Item(ref item) => item.span,
             Annotatable::TraitItem(ref trait_item) => trait_item.span,
             Annotatable::ImplItem(ref impl_item) => impl_item.span,
+            Annotatable::Stmt(ref stmt) => stmt.span,
+            Annotatable::Expr(ref expr) => expr.span,
         }
     }
 

src/libsyntax/ext/expand.rs

@@ -435,6 +435,12 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             Annotatable::ImplItem(item) => {
                 Annotatable::ImplItem(item.map(|item| cfg.fold_impl_item(item).pop().unwrap()))
             }
+            Annotatable::Stmt(stmt) => {
+                Annotatable::Stmt(stmt.map(|stmt| cfg.fold_stmt(stmt).pop().unwrap()))
+            }
+            Annotatable::Expr(expr) => {
+                Annotatable::Expr(cfg.fold_expr(expr))
+            }
         }
     }
 
@@ -503,6 +509,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     Annotatable::Item(item) => token::NtItem(item),
                     Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
                     Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
+                    Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
+                    Annotatable::Expr(expr) => token::NtExpr(expr),
                 })).into();
                 let tok_result = mac.expand(self.cx, attr.span, attr.tokens, item_tok);
                 self.parse_expansion(tok_result, kind, &attr.path, attr.span)
@@ -751,6 +759,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 Some(expansion)
             }
             Err(mut err) => {
+                err.set_span(span);
                 err.emit();
                 self.cx.trace_macros_diag();
                 kind.dummy(span)
@@ -796,7 +805,13 @@ impl<'a> Parser<'a> {
                 Expansion::Stmts(stmts)
             }
             ExpansionKind::Expr => Expansion::Expr(self.parse_expr()?),
-            ExpansionKind::OptExpr => Expansion::OptExpr(Some(self.parse_expr()?)),
+            ExpansionKind::OptExpr => {
+                if self.token != token::Eof {
+                    Expansion::OptExpr(Some(self.parse_expr()?))
+                } else {
+                    Expansion::OptExpr(None)
+                }
+            },
             ExpansionKind::Ty => Expansion::Ty(self.parse_ty()?),
             ExpansionKind::Pat => Expansion::Pat(self.parse_pat()?),
         })
@@ -904,6 +919,18 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
         let mut expr = self.cfg.configure_expr(expr).into_inner();
         expr.node = self.cfg.configure_expr_kind(expr.node);
 
+        let (attr, derives, expr) = self.classify_item(expr);
+
+        if attr.is_some() || !derives.is_empty() {
+            // collect the invoc regardless of whether or not attributes are permitted here
+            // expansion will eat the attribute so it won't error later
+            attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
+
+            // ExpansionKind::Expr requires the macro to emit an expression
+            return self.collect_attr(attr, derives, Annotatable::Expr(P(expr)), ExpansionKind::Expr)
+                .make_expr();
+        }
+
         if let ast::ExprKind::Mac(mac) = expr.node {
             self.check_attributes(&expr.attrs);
             self.collect_bang(mac, expr.span, ExpansionKind::Expr).make_expr()
@@ -916,6 +943,16 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
         let mut expr = configure!(self, expr).into_inner();
         expr.node = self.cfg.configure_expr_kind(expr.node);
 
+        let (attr, derives, expr) = self.classify_item(expr);
+
+        if attr.is_some() || !derives.is_empty() {
+            attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
+
+            return self.collect_attr(attr, derives, Annotatable::Expr(P(expr)),
+                                     ExpansionKind::OptExpr)
+                .make_opt_expr();
+        }
+
         if let ast::ExprKind::Mac(mac) = expr.node {
             self.check_attributes(&expr.attrs);
             self.collect_bang(mac, expr.span, ExpansionKind::OptExpr).make_opt_expr()
@@ -938,33 +975,47 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
     }
 
     fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
-        let stmt = match self.cfg.configure_stmt(stmt) {
+        let mut stmt = match self.cfg.configure_stmt(stmt) {
             Some(stmt) => stmt,
             None => return SmallVector::new(),
         };
 
-        let (mac, style, attrs) = if let StmtKind::Mac(mac) = stmt.node {
-            mac.into_inner()
-        } else {
-            // The placeholder expander gives ids to statements, so we avoid folding the id here.
-            let ast::Stmt { id, node, span } = stmt;
-            return noop_fold_stmt_kind(node, self).into_iter().map(|node| {
-                ast::Stmt { id: id, node: node, span: span }
-            }).collect()
-        };
+        // we'll expand attributes on expressions separately
+        if !stmt.is_expr() {
+            let (attr, derives, stmt_) = self.classify_item(stmt);
 
-        self.check_attributes(&attrs);
-        let mut placeholder = self.collect_bang(mac, stmt.span, ExpansionKind::Stmts).make_stmts();
-
-        // If this is a macro invocation with a semicolon, then apply that
-        // semicolon to the final statement produced by expansion.
-        if style == MacStmtStyle::Semicolon {
-            if let Some(stmt) = placeholder.pop() {
-                placeholder.push(stmt.add_trailing_semicolon());
+            if attr.is_some() || !derives.is_empty() {
+                return self.collect_attr(attr, derives,
+                                         Annotatable::Stmt(P(stmt_)), ExpansionKind::Stmts)
+                    .make_stmts();
             }
+
+            stmt = stmt_;
         }
 
-        placeholder
+        if let StmtKind::Mac(mac) = stmt.node {
+            let (mac, style, attrs) = mac.into_inner();
+            self.check_attributes(&attrs);
+            let mut placeholder = self.collect_bang(mac, stmt.span, ExpansionKind::Stmts)
+                                        .make_stmts();
+
+            // If this is a macro invocation with a semicolon, then apply that
+            // semicolon to the final statement produced by expansion.
+            if style == MacStmtStyle::Semicolon {
+                if let Some(stmt) = placeholder.pop() {
+                    placeholder.push(stmt.add_trailing_semicolon());
+                }
+            }
+
+            return placeholder;
+        }
+
+        // The placeholder expander gives ids to statements, so we avoid folding the id here.
+        let ast::Stmt { id, node, span } = stmt;
+        noop_fold_stmt_kind(node, self).into_iter().map(|node| {
+            ast::Stmt { id, node, span }
+        }).collect()
+
     }
 
     fn fold_block(&mut self, block: P<Block>) -> P<Block> {

src/libsyntax/feature_gate.rs

@@ -1250,7 +1250,7 @@ const EXPLAIN_BOX_SYNTAX: &'static str =
     "box expression syntax is experimental; you can call `Box::new` instead.";
 
 pub const EXPLAIN_STMT_ATTR_SYNTAX: &'static str =
-    "attributes on non-item statements and expressions are experimental.";
+    "attributes on expressions are experimental.";
 
 pub const EXPLAIN_ASM: &'static str =
     "inline assembly is not stable enough for use and is subject to change";

src/libsyntax/parse/parser.rs

@@ -4601,6 +4601,9 @@ impl<'a> Parser<'a> {
 
     /// Parse a statement, including the trailing semicolon.
     pub fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> {
+        // skip looking for a trailing semicolon when we have an interpolated statement
+        maybe_whole!(self, NtStmt, |x| Some(x));
+
         let mut stmt = match self.parse_stmt_without_recovery(macro_legacy_warnings)? {
             Some(stmt) => stmt,
             None => return Ok(None),

src/libsyntax_ext/deriving/custom.rs

@@ -54,7 +54,9 @@ impl MultiItemModifier for ProcMacroDerive {
         let item = match item {
             Annotatable::Item(item) => item,
             Annotatable::ImplItem(_) |
-            Annotatable::TraitItem(_) => {
+            Annotatable::TraitItem(_) |
+            Annotatable::Stmt(_) |
+            Annotatable::Expr(_) => {
                 ecx.span_err(span, "proc-macro derives may only be \
                                     applied to struct/enum items");
                 return Vec::new()

src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs

@@ -93,6 +93,8 @@ fn expand_into_foo_multi(cx: &mut ExtCtxt,
                 }
             })
         }
+        // these are covered in proc_macro/attr-stmt-expr.rs
+        Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item")
     }
 }
 
@@ -145,6 +147,8 @@ fn expand_duplicate(cx: &mut ExtCtxt,
             new_it.ident = copy_name;
             push(Annotatable::TraitItem(P(new_it)));
         }
+        // covered in proc_macro/attr-stmt-expr.rs
+        Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item")
     }
 }
 

src/test/compile-fail-fulldeps/proc-macro/attr-invalid-exprs.rs

@@ -0,0 +1,38 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr-stmt-expr.rs
+// ignore-stage1
+
+//! Attributes producing expressions in invalid locations
+
+#![feature(proc_macro, stmt_expr_attributes)]
+
+extern crate attr_stmt_expr;
+use attr_stmt_expr::{duplicate, no_output};
+
+fn main() {
+    let _ = #[no_output] "Hello, world!";
+    //~^ ERROR expected expression, found `<eof>`
+
+    let _ = #[duplicate] "Hello, world!";
+    //~^ ERROR macro expansion ignores token `,` and any following
+
+    let _ = {
+        #[no_output]
+        "Hello, world!"
+    };
+
+    let _ = {
+        #[duplicate]
+        //~^ ERROR macro expansion ignores token `,` and any following
+        "Hello, world!"
+    };
+}

src/test/compile-fail-fulldeps/proc-macro/attr-stmt-expr.rs

@@ -0,0 +1,38 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr-stmt-expr.rs
+// ignore-stage1
+
+#![feature(proc_macro)]
+
+extern crate attr_stmt_expr;
+use attr_stmt_expr::{expect_let, expect_print_stmt, expect_expr, expect_print_expr};
+
+fn print_str(string: &'static str) {
+    // macros are handled a bit differently
+    #[expect_print_expr]
+    //~^ ERROR attributes on expressions are experimental
+    //~| HELP add #![feature(stmt_expr_attributes)] to the crate attributes to enable
+    println!("{}", string)
+}
+
+fn main() {
+    #[expect_let]
+    let string = "Hello, world!";
+
+    #[expect_print_stmt]
+    println!("{}", string);
+
+    #[expect_expr]
+    //~^ ERROR attributes on expressions are experimental
+    //~| HELP add #![feature(stmt_expr_attributes)] to the crate attributes to enable
+    print_str("string")
+}

src/test/compile-fail-fulldeps/proc-macro/auxiliary/attr-stmt-expr.rs

@@ -0,0 +1,59 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "println!(\"{}\" , string);");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "print_str(\"string\")");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "println!(\"{}\" , string)");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn duplicate(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    format!("{}, {}", item, item).parse().unwrap()
+}
+
+#[proc_macro_attribute]
+pub fn no_output(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert!(!item.to_string().is_empty());
+    "".parse().unwrap()
+}

src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs

@@ -96,6 +96,8 @@ fn expand_into_foo_multi(cx: &mut ExtCtxt,
                 }
             })
         ],
+        // these are covered in proc_macro/attr-stmt-expr.rs
+        Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item"),
     }
 }
 
@@ -140,6 +142,8 @@ fn expand_duplicate(cx: &mut ExtCtxt,
             new_it.ident = copy_name;
             push(Annotatable::TraitItem(P(new_it)));
         }
+        // these are covered in proc_macro/attr-stmt-expr.rs
+        Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item")
     }
 }
 

src/test/run-pass-fulldeps/proc-macro/attr-stmt-expr.rs

@@ -0,0 +1,34 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr-stmt-expr.rs
+// ignore-stage1
+
+#![feature(proc_macro, stmt_expr_attributes)]
+
+extern crate attr_stmt_expr;
+use attr_stmt_expr::{expect_let, expect_print_stmt, expect_expr, expect_print_expr};
+
+fn print_str(string: &'static str) {
+    // macros are handled a bit differently
+    #[expect_print_expr]
+    println!("{}", string)
+}
+
+fn main() {
+    #[expect_let]
+    let string = "Hello, world!";
+
+    #[expect_print_stmt]
+    println!("{}", string);
+
+    #[expect_expr]
+    print_str("string")
+}

src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-stmt-expr.rs

@@ -0,0 +1,46 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "println!(\"{}\" , string);");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "print_str(\"string\")");
+    item
+}
+
+#[proc_macro_attribute]
+pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
+    assert!(attr.to_string().is_empty());
+    assert_eq!(item.to_string(), "println!(\"{}\" , string)");
+    item
+}

src/test/ui/feature-gate-stmt_expr_attributes.rs

@@ -9,6 +9,6 @@
 // except according to those terms.
 
 const X: i32 = #[allow(dead_code)] 8;
-//~^ ERROR attributes on non-item statements and expressions are experimental. (see issue #15701)
+//~^ ERROR attributes on expressions are experimental. (see issue #15701)
 
 fn main() {}

src/test/ui/feature-gate-stmt_expr_attributes.stderr

@@ -1,4 +1,4 @@
-error[E0658]: attributes on non-item statements and expressions are experimental. (see issue #15701)
+error[E0658]: attributes on expressions are experimental. (see issue #15701)
   --> $DIR/feature-gate-stmt_expr_attributes.rs:11:16
    |
 LL | const X: i32 = #[allow(dead_code)] 8;