nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 08:13:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stop using a special inner body for the coroutine by-move body for async closures

Browse Source
This commit is contained in:
Michael Goulet 2024-08-01 13:05:17 -04:00
parent 515395af0e
commit 4609841c07
40 changed files with 295 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
compiler/rustc_const_eval/src/interpret/call.rs
View File

@ -576,7 +576,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
| ty::InstanceKind::ReifyShim(..)
| ty::InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| ty::InstanceKind::CoroutineKindShim { .. }
| ty::InstanceKind::FnPtrShim(..)
| ty::InstanceKind::DropGlue(..)
| ty::InstanceKind::CloneShim(..)

11
compiler/rustc_hir/src/def.rs
View File

@ -133,6 +133,9 @@ pub enum DefKind {
/// we treat them all the same, and code which needs to distinguish them can match
/// or `hir::ClosureKind` or `type_of`.
Closure,
/// The definition of a synthetic coroutine body created by the lowering of a
/// coroutine-closure, such as an async closure.
SyntheticCoroutineBody,
}
impl DefKind {
@ -177,6 +180,7 @@ impl DefKind {
DefKind::Closure => "closure",
DefKind::ExternCrate => "extern crate",
DefKind::GlobalAsm => "global assembly block",
DefKind::SyntheticCoroutineBody => "synthetic mir body",
}
}
@ -236,7 +240,8 @@ impl DefKind {
| DefKind::ForeignMod
| DefKind::GlobalAsm
| DefKind::Impl { .. }
| DefKind::OpaqueTy => None,
| DefKind::OpaqueTy
| DefKind::SyntheticCoroutineBody => None,
}
}
@ -276,6 +281,7 @@ impl DefKind {
DefKind::GlobalAsm => DefPathData::GlobalAsm,
DefKind::Impl { .. } => DefPathData::Impl,
DefKind::Closure => DefPathData::Closure,
DefKind::SyntheticCoroutineBody => DefPathData::Closure,
}
}
@ -291,7 +297,8 @@ impl DefKind {
| DefKind::AssocFn
| DefKind::Ctor(..)
| DefKind::Closure
| DefKind::Static { .. } => true,
| DefKind::Static { .. }
| DefKind::SyntheticCoroutineBody => true,
DefKind::Mod
| DefKind::Struct
| DefKind::Union

3
compiler/rustc_hir_analysis/src/check/wfcheck.rs
View File

@ -2174,7 +2174,8 @@ fn lint_redundant_lifetimes<'tcx>(
| DefKind::Field
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::Closure => return,
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => return,
}
// The ordering of this lifetime map is a bit subtle.

4
compiler/rustc_hir_analysis/src/lib.rs
View File

@ -200,10 +200,6 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
}
});
// Freeze definitions as we don't add new ones at this point. This improves performance by
// allowing lock-free access to them.
tcx.untracked().definitions.freeze();
// FIXME: Remove this when we implement creating `DefId`s
// for anon constants during their parents' typeck.
// Typeck all body owners in parallel will produce queries

16
compiler/rustc_interface/src/passes.rs
View File

@ -784,7 +784,22 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
}
);
});
rustc_hir_analysis::check_crate(tcx);
sess.time("MIR_coroutine_by_move_body", || {
tcx.hir().par_body_owners(|def_id| {
if tcx.needs_coroutine_by_move_body_def_id(def_id) {
tcx.ensure_with_value().coroutine_by_move_body_def_id(def_id);
}
});
});
// Freeze definitions as we don't add new ones at this point.
// We need to wait until now since we synthesize a by-move body
// This improves performance by allowing lock-free access to them.
// FIXME(async_closures): We could force `coroutine_by_move_body_def_id`
// immediately after typeck, then freeze after that.
tcx.untracked().definitions.freeze();
sess.time("MIR_borrow_checking", || {
tcx.hir().par_body_owners(|def_id| {
// Run unsafety check because it's responsible for stealing and
@ -816,6 +831,7 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
);
}
});
sess.time("layout_testing", || layout_test::test_layout(tcx));
sess.time("abi_testing", || abi_test::test_abi(tcx));

39
compiler/rustc_metadata/src/rmeta/encoder.rs
View File

@ -872,7 +872,8 @@ fn should_encode_span(def_kind: DefKind) -> bool {
| DefKind::OpaqueTy
| DefKind::Field
| DefKind::Impl { .. }
| DefKind::Closure => true,
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => true,
DefKind::ForeignMod | DefKind::GlobalAsm => false,
}
}
@ -902,6 +903,7 @@ fn should_encode_attrs(def_kind: DefKind) -> bool {
// https://github.com/model-checking/kani and is not a performance
// or maintenance issue for us.
DefKind::Closure => true,
DefKind::SyntheticCoroutineBody => false,
DefKind::TyParam
| DefKind::ConstParam
| DefKind::Ctor(..)
@ -948,7 +950,8 @@ fn should_encode_expn_that_defined(def_kind: DefKind) -> bool {
| DefKind::Field
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::Closure => false,
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => false,
}
}
@ -984,7 +987,8 @@ fn should_encode_visibility(def_kind: DefKind) -> bool {
| DefKind::GlobalAsm
| DefKind::Impl { .. }
| DefKind::Closure
| DefKind::ExternCrate => false,
| DefKind::ExternCrate
| DefKind::SyntheticCoroutineBody => false,
}
}
@ -1019,7 +1023,8 @@ fn should_encode_stability(def_kind: DefKind) -> bool {
| DefKind::InlineConst
| DefKind::GlobalAsm
| DefKind::Closure
| DefKind::ExternCrate => false,
| DefKind::ExternCrate
| DefKind::SyntheticCoroutineBody => false,
}
}
@ -1061,6 +1066,8 @@ fn should_encode_mir(
}
// Coroutines require optimized MIR to compute layout.
DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => (false, true),
// FIXME: lol
DefKind::SyntheticCoroutineBody => (false, true),
// Full-fledged functions + closures
DefKind::AssocFn | DefKind::Fn | DefKind::Closure => {
let generics = tcx.generics_of(def_id);
@ -1109,7 +1116,8 @@ fn should_encode_variances<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, def_kind: Def
| DefKind::InlineConst
| DefKind::GlobalAsm
| DefKind::Closure
| DefKind::ExternCrate => false,
| DefKind::ExternCrate
| DefKind::SyntheticCoroutineBody => false,
DefKind::TyAlias => tcx.type_alias_is_lazy(def_id),
}
}
@ -1137,7 +1145,8 @@ fn should_encode_generics(def_kind: DefKind) -> bool {
| DefKind::Impl { .. }
| DefKind::Field
| DefKind::TyParam
| DefKind::Closure => true,
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => true,
DefKind::Mod
| DefKind::ForeignMod
| DefKind::ConstParam
@ -1168,7 +1177,8 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
| DefKind::Closure
| DefKind::ConstParam
| DefKind::AnonConst
| DefKind::InlineConst => true,
| DefKind::InlineConst
| DefKind::SyntheticCoroutineBody => true,
DefKind::OpaqueTy => {
let origin = tcx.opaque_type_origin(def_id);
@ -1240,7 +1250,8 @@ fn should_encode_fn_sig(def_kind: DefKind) -> bool {
| DefKind::Use
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::ExternCrate => false,
| DefKind::ExternCrate
| DefKind::SyntheticCoroutineBody => false,
}
}
@ -1277,7 +1288,8 @@ fn should_encode_constness(def_kind: DefKind) -> bool {
| DefKind::Use
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::ExternCrate => false,
| DefKind::ExternCrate
| DefKind::SyntheticCoroutineBody => false,
}
}
@ -1310,7 +1322,8 @@ fn should_encode_const(def_kind: DefKind) -> bool {
| DefKind::Use
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::ExternCrate => false,
| DefKind::ExternCrate
| DefKind::SyntheticCoroutineBody => false,
}
}
@ -1366,6 +1379,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
let def_span = tcx.def_span(local_id);
record!(self.tables.def_span[def_id] <- def_span);
}
// FIXME(async_closures): We should just use `tcx.attrs` rather than going
// through the HIR. Historically, though, this has been inefficient apparently.
// For now, it's kind of pointless to fix, because coroutine-closures' coroutine
// bodies have no attrs anyways.
if should_encode_attrs(def_kind) {
self.encode_attrs(local_id);
}
@ -1458,7 +1475,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
record!(self.tables.associated_type_for_effects[def_id] <- assoc_def_id);
}
}
if def_kind == DefKind::Closure
if let DefKind::Closure | DefKind::SyntheticCoroutineBody = def_kind
&& let Some(coroutine_kind) = self.tcx.coroutine_kind(def_id)
{
self.tables.coroutine_kind.set(def_id.index, Some(coroutine_kind))

1
compiler/rustc_metadata/src/rmeta/table.rs
View File

@ -171,6 +171,7 @@ fixed_size_enum! {
( Macro(MacroKind::Bang) )
( Macro(MacroKind::Attr) )
( Macro(MacroKind::Derive) )
( SyntheticCoroutineBody )
}
}

2
compiler/rustc_middle/src/hir/map/mod.rs
View File

@ -307,7 +307,7 @@ impl<'hir> Map<'hir> {
}
DefKind::InlineConst => BodyOwnerKind::Const { inline: true },
DefKind::Ctor(..) | DefKind::Fn | DefKind::AssocFn => BodyOwnerKind::Fn,
DefKind::Closure => BodyOwnerKind::Closure,
DefKind::Closure | DefKind::SyntheticCoroutineBody => BodyOwnerKind::Closure,
DefKind::Static { safety: _, mutability, nested: false } => {
BodyOwnerKind::Static(mutability)
}

17
compiler/rustc_middle/src/mir/mod.rs
View File

@ -267,18 +267,6 @@ pub struct CoroutineInfo<'tcx> {
/// Coroutine drop glue. This field is populated after the state transform pass.
pub coroutine_drop: Option<Body<'tcx>>,
/// The body of the coroutine, modified to take its upvars by move rather than by ref.
///
/// This is used by coroutine-closures, which must return a different flavor of coroutine
/// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which
/// is run right after building the initial MIR, and will only be populated for coroutines
/// which come out of the async closure desugaring.
///
/// This body should be processed in lockstep with the containing body -- any optimization
/// passes, etc, should be applied to this body as well. This is done automatically if
/// using `run_passes`.
pub by_move_body: Option<Body<'tcx>>,
/// The layout of a coroutine. This field is populated after the state transform pass.
pub coroutine_layout: Option<CoroutineLayout<'tcx>>,
@ -298,7 +286,6 @@ impl<'tcx> CoroutineInfo<'tcx> {
coroutine_kind,
yield_ty: Some(yield_ty),
resume_ty: Some(resume_ty),
by_move_body: None,
coroutine_drop: None,
coroutine_layout: None,
}
@ -665,10 +652,6 @@ impl<'tcx> Body<'tcx> {
self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref())
}
pub fn coroutine_by_move_body(&self) -> Option<&Body<'tcx>> {
self.coroutine.as_ref()?.by_move_body.as_ref()
}
#[inline]
pub fn coroutine_kind(&self) -> Option<CoroutineKind> {
self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind)

1
compiler/rustc_middle/src/mir/mono.rs
View File

@ -415,7 +415,6 @@ impl<'tcx> CodegenUnit<'tcx> {
| InstanceKind::Virtual(..)
| InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::CoroutineKindShim { .. }
| InstanceKind::DropGlue(..)
| InstanceKind::CloneShim(..)
| InstanceKind::ThreadLocalShim(..)

1
compiler/rustc_middle/src/mir/visit.rs
View File

@ -349,7 +349,6 @@ macro_rules! make_mir_visitor {
coroutine_closure_def_id: _def_id,
receiver_by_ref: _,
} |
ty::InstanceKind::CoroutineKindShim { coroutine_def_id: _def_id } |
ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, None) |
ty::InstanceKind::DropGlue(_def_id, None) => {}

15
compiler/rustc_middle/src/query/mod.rs
View File

@ -326,6 +326,7 @@ rustc_queries! {
query predicates_of(key: DefId) -> ty::GenericPredicates<'tcx> {
desc { |tcx| "computing predicates of `{}`", tcx.def_path_str(key) }
cache_on_disk_if { key.is_local() }
feedable
}
query opaque_types_defined_by(
@ -498,6 +499,7 @@ rustc_queries! {
/// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/construction.html
query mir_built(key: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
desc { |tcx| "building MIR for `{}`", tcx.def_path_str(key) }
feedable
}
/// Try to build an abstract representation of the given constant.
@ -742,6 +744,7 @@ rustc_queries! {
query constness(key: DefId) -> hir::Constness {
desc { |tcx| "checking if item is const: `{}`", tcx.def_path_str(key) }
separate_provide_extern
feedable
}
query asyncness(key: DefId) -> ty::Asyncness {
@ -760,10 +763,22 @@ rustc_queries! {
desc { |tcx| "checking if item is promotable: `{}`", tcx.def_path_str(key) }
}
/// The body of the coroutine, modified to take its upvars by move rather than by ref.
///
/// This is used by coroutine-closures, which must return a different flavor of coroutine
/// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which
/// is run right after building the initial MIR, and will only be populated for coroutines
/// which come out of the async closure desugaring.
query coroutine_by_move_body_def_id(def_id: DefId) -> DefId {
desc { |tcx| "looking up the coroutine by-move body for `{}`", tcx.def_path_str(def_id) }
separate_provide_extern
}
/// Returns `Some(coroutine_kind)` if the node pointed to by `def_id` is a coroutine.
query coroutine_kind(def_id: DefId) -> Option<hir::CoroutineKind> {
desc { |tcx| "looking up coroutine kind of `{}`", tcx.def_path_str(def_id) }
separate_provide_extern
feedable
}
query coroutine_for_closure(def_id: DefId) -> DefId {

18
compiler/rustc_middle/src/ty/context.rs
View File

@ -1579,6 +1579,12 @@ impl<'tcx> TyCtxt<'tcx> {
)
}
// Whether the body owner is synthetic, which in this case means it does not correspond to
// meaningful HIR. This is currently used to skip over MIR borrowck.
pub fn is_synthetic_mir(self, def_id: impl Into<DefId>) -> bool {
matches!(self.def_kind(def_id.into()), DefKind::SyntheticCoroutineBody)
}
/// Returns `true` if the node pointed to by `def_id` is a general coroutine that implements `Coroutine`.
/// This means it is neither an `async` or `gen` construct.
pub fn is_general_coroutine(self, def_id: DefId) -> bool {
@ -3168,6 +3174,18 @@ impl<'tcx> TyCtxt<'tcx> {
self.impl_trait_header(def_id).map_or(ty::ImplPolarity::Positive, |h| h.polarity)
}
pub fn needs_coroutine_by_move_body_def_id(self, def_id: LocalDefId) -> bool {
if let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
self.coroutine_kind(def_id)
&& let ty::Coroutine(_, args) = self.type_of(def_id).instantiate_identity().kind()
&& args.as_coroutine().kind_ty().to_opt_closure_kind() != Some(ty::ClosureKind::FnOnce)
{
true
} else {
false
}
}
/// Whether this is a trait implementation that has `#[diagnostic::do_not_recommend]`
pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true })

16
compiler/rustc_middle/src/ty/instance.rs
View File

@ -141,14 +141,6 @@ pub enum InstanceKind<'tcx> {
receiver_by_ref: bool,
},
/// `<[coroutine] as Future>::poll`, but for coroutines produced when `AsyncFnOnce`
/// is called on a coroutine-closure whose closure kind greater than `FnOnce`, or
/// similarly for `AsyncFnMut`.
///
/// This will select the body that is produced by the `ByMoveBody` transform, and thus
/// take and use all of its upvars by-move rather than by-ref.
CoroutineKindShim { coroutine_def_id: DefId },
/// Compiler-generated accessor for thread locals which returns a reference to the thread local
/// the `DefId` defines. This is used to export thread locals from dylibs on platforms lacking
/// native support.
@ -248,7 +240,6 @@ impl<'tcx> InstanceKind<'tcx> {
coroutine_closure_def_id: def_id,
receiver_by_ref: _,
}
| ty::InstanceKind::CoroutineKindShim { coroutine_def_id: def_id }
| InstanceKind::DropGlue(def_id, _)
| InstanceKind::CloneShim(def_id, _)
| InstanceKind::FnPtrAddrShim(def_id, _)
@ -270,7 +261,6 @@ impl<'tcx> InstanceKind<'tcx> {
| InstanceKind::Intrinsic(..)
| InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| ty::InstanceKind::CoroutineKindShim { .. }
| InstanceKind::DropGlue(..)
| InstanceKind::AsyncDropGlueCtorShim(..)
| InstanceKind::CloneShim(..)
@ -377,7 +367,6 @@ impl<'tcx> InstanceKind<'tcx> {
| InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => false,
InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::CoroutineKindShim { .. }
| InstanceKind::DropGlue(..)
| InstanceKind::AsyncDropGlueCtorShim(..)
| InstanceKind::Item(_)
@ -452,7 +441,6 @@ pub fn fmt_instance(
InstanceKind::FnPtrShim(_, ty) => write!(f, " - shim({ty})"),
InstanceKind::ClosureOnceShim { .. } => write!(f, " - shim"),
InstanceKind::ConstructCoroutineInClosureShim { .. } => write!(f, " - shim"),
InstanceKind::CoroutineKindShim { .. } => write!(f, " - shim"),
InstanceKind::DropGlue(_, None) => write!(f, " - shim(None)"),
InstanceKind::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"),
InstanceKind::CloneShim(_, ty) => write!(f, " - shim({ty})"),
@ -850,7 +838,9 @@ impl<'tcx> Instance<'tcx> {
Some(Instance { def: ty::InstanceKind::Item(coroutine_def_id), args })
} else {
Some(Instance {
def: ty::InstanceKind::CoroutineKindShim { coroutine_def_id },
def: ty::InstanceKind::Item(
tcx.coroutine_by_move_body_def_id(coroutine_def_id),
),
args,
})
}

4
compiler/rustc_middle/src/ty/mod.rs
View File

@ -1756,7 +1756,6 @@ impl<'tcx> TyCtxt<'tcx> {
| ty::InstanceKind::Virtual(..)
| ty::InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| ty::InstanceKind::CoroutineKindShim { .. }
| ty::InstanceKind::DropGlue(..)
| ty::InstanceKind::CloneShim(..)
| ty::InstanceKind::ThreadLocalShim(..)
@ -1874,7 +1873,8 @@ impl<'tcx> TyCtxt<'tcx> {
identity_kind_ty.to_opt_closure_kind(),
Some(ClosureKind::Fn | ClosureKind::FnMut)
);
mir.coroutine_by_move_body().unwrap().coroutine_layout_raw()
self.optimized_mir(self.coroutine_by_move_body_def_id(def_id))
.coroutine_layout_raw()
}
}
}

2
compiler/rustc_mir_transform/src/coroutine.rs
View File

@ -53,7 +53,7 @@
mod by_move_body;
use std::{iter, ops};
pub use by_move_body::ByMoveBody;
pub use by_move_body::coroutine_by_move_body_def_id;
use rustc_data_structures::fx::FxHashSet;
use rustc_errors::pluralize;
use rustc_hir as hir;

72
compiler/rustc_mir_transform/src/coroutine/by_move_body.rs
View File

@ -69,44 +69,34 @@
//! in case precise captures (edition 2021 closure capture rules) caused the inner coroutine
//! to split one field capture into two.
use rustc_data_structures::steal::Steal;
use rustc_data_structures::unord::UnordMap;
use rustc_hir as hir;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{DefId, LocalDefId};
use rustc_middle::bug;
use rustc_middle::hir::place::{Projection, ProjectionKind};
use rustc_middle::mir::visit::MutVisitor;
use rustc_middle::mir::{self, dump_mir, MirPass};
use rustc_middle::mir::{self, dump_mir};
use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt};
use rustc_span::symbol::kw;
use rustc_target::abi::{FieldIdx, VariantIdx};
use crate::pass_manager::validate_body;
pub fn coroutine_by_move_body_def_id<'tcx>(
tcx: TyCtxt<'tcx>,
coroutine_def_id: LocalDefId,
) -> DefId {
let body = tcx.mir_built(coroutine_def_id).borrow();
pub struct ByMoveBody;
impl<'tcx> MirPass<'tcx> for ByMoveBody {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) {
// We only need to generate by-move coroutine bodies for coroutines that come
// from coroutine-closures.
let Some(coroutine_def_id) = body.source.def_id().as_local() else {
return;
};
let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
tcx.coroutine_kind(coroutine_def_id)
else {
return;
bug!("should only be invoked on coroutine-closures");
};
// Also, let's skip processing any bodies with errors, since there's no guarantee
// the MIR body will be constructed well.
let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
if coroutine_ty.references_error() {
return;
}
// We don't need to generate a by-move coroutine if the coroutine body was
// produced by the `CoroutineKindShim`, since it's already by-move.
if matches!(body.source.instance, ty::InstanceKind::CoroutineKindShim { .. }) {
return;
}
let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") };
let args = args.as_coroutine();
@ -119,6 +109,10 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
else {
bug!();
};
if parent_args.references_error() {
return coroutine_def_id.to_def_id();
}
let parent_closure_args = parent_args.as_coroutine_closure();
let num_args = parent_closure_args
.coroutine_closure_sig()
@ -133,9 +127,8 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
|(parent_field_idx, parent_capture), (child_field_idx, child_capture)| {
// Store this set of additional projections (fields and derefs).
// We need to re-apply them later.
let mut child_precise_captures = child_capture.place.projections
[parent_capture.place.projections.len()..]
.to_vec();
let mut child_precise_captures =
child_capture.place.projections[parent_capture.place.projections.len()..].to_vec();
// If the parent capture is by-ref, then we need to apply an additional
// deref before applying any further projections to this place.
@ -196,7 +189,8 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
if coroutine_kind == ty::ClosureKind::FnOnce {
assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len());
return;
// The by-move body is just the body :)
return coroutine_def_id.to_def_id();
}
let by_move_coroutine_ty = tcx
@ -215,15 +209,27 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body);
dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(()));
// Let's just always validate this body.
validate_body(tcx, &mut by_move_body, "Initial coroutine_by_move body".to_string());
let body_def = tcx.create_def(coroutine_def_id, kw::Empty, DefKind::SyntheticCoroutineBody);
by_move_body.source =
mir::MirSource::from_instance(InstanceKind::Item(body_def.def_id().to_def_id()));
// FIXME: use query feeding to generate the body right here and then only store the `DefId` of the new body.
by_move_body.source = mir::MirSource::from_instance(InstanceKind::CoroutineKindShim {
coroutine_def_id: coroutine_def_id.to_def_id(),
});
body.coroutine.as_mut().unwrap().by_move_body = Some(by_move_body);
}
// Inherited from the by-ref coroutine.
body_def.codegen_fn_attrs(tcx.codegen_fn_attrs(coroutine_def_id).clone());
body_def.constness(tcx.constness(coroutine_def_id).clone());
body_def.coroutine_kind(tcx.coroutine_kind(coroutine_def_id).clone());
body_def.def_ident_span(tcx.def_ident_span(coroutine_def_id));
body_def.def_span(tcx.def_span(coroutine_def_id));
body_def.explicit_predicates_of(tcx.explicit_predicates_of(coroutine_def_id).clone());
body_def.generics_of(tcx.generics_of(coroutine_def_id).clone());
body_def.param_env(tcx.param_env(coroutine_def_id).clone());
body_def.predicates_of(tcx.predicates_of(coroutine_def_id).clone());
// The type of the coroutine is the `by_move_coroutine_ty`.
body_def.type_of(ty::EarlyBinder::bind(by_move_coroutine_ty));
body_def.mir_built(tcx.arena.alloc(Steal::new(by_move_body)));
body_def.def_id().to_def_id()
}
struct MakeByMoveBody<'tcx> {

1
compiler/rustc_mir_transform/src/inline.rs
View File

@ -341,7 +341,6 @@ impl<'tcx> Inliner<'tcx> {
| InstanceKind::FnPtrShim(..)
| InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::CoroutineKindShim { .. }
| InstanceKind::DropGlue(..)
| InstanceKind::CloneShim(..)
| InstanceKind::ThreadLocalShim(..)

1
compiler/rustc_mir_transform/src/inline/cycle.rs
View File

@ -88,7 +88,6 @@ pub(crate) fn mir_callgraph_reachable<'tcx>(
| InstanceKind::FnPtrShim(..)
| InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::CoroutineKindShim { .. }
| InstanceKind::ThreadLocalShim { .. }
| InstanceKind::CloneShim(..) => {}

40
compiler/rustc_mir_transform/src/lib.rs
View File

@ -135,6 +135,7 @@ pub fn provide(providers: &mut Providers) {
mir_inliner_callees: inline::cycle::mir_inliner_callees,
promoted_mir,
deduced_param_attrs: deduce_param_attrs::deduced_param_attrs,
coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id,
..providers.queries
};
}
@ -293,10 +294,6 @@ fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
&Lint(check_packed_ref::CheckPackedRef),
&Lint(check_const_item_mutation::CheckConstItemMutation),
&Lint(function_item_references::FunctionItemReferences),
// If this is an async closure's output coroutine, generate
// by-move and by-mut bodies if needed. We do this first so
// they can be optimized in lockstep with their parent bodies.
&coroutine::ByMoveBody,
// What we need to do constant evaluation.
&simplify::SimplifyCfg::Initial,
&rustc_peek::SanityCheck, // Just a lint
@ -329,8 +326,15 @@ fn mir_promoted(
| DefKind::AnonConst => tcx.mir_const_qualif(def),
_ => ConstQualifs::default(),
};
// has_ffi_unwind_calls query uses the raw mir, so make sure it is run.
// the `has_ffi_unwind_calls` query uses the raw mir, so make sure it is run.
tcx.ensure_with_value().has_ffi_unwind_calls(def);
// the `by_move_body` query uses the raw mir, so make sure it is run.
if tcx.needs_coroutine_by_move_body_def_id(def) {
tcx.ensure_with_value().coroutine_by_move_body_def_id(def);
}
let mut body = tcx.mir_built(def).steal();
if let Some(error_reported) = const_qualifs.tainted_by_errors {
body.tainted_by_errors = Some(error_reported);
@ -339,14 +343,6 @@ fn mir_promoted(
// Collect `required_consts` *before* promotion, so if there are any consts being promoted
// we still add them to the list in the outer MIR body.
RequiredConstsVisitor::compute_required_consts(&mut body);
// If this has an associated by-move async closure body, that doesn't get run through these
// passes itself, it gets "tagged along" by the pass manager. `RequiredConstsVisitor` is not
// a regular pass so we have to also apply it manually to the other body.
if let Some(coroutine) = body.coroutine.as_mut() {
if let Some(by_move_body) = coroutine.by_move_body.as_mut() {
RequiredConstsVisitor::compute_required_consts(by_move_body);
}
}
// What we need to run borrowck etc.
let promote_pass = promote_consts::PromoteTemps::default();
@ -398,7 +394,10 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
if tcx.is_coroutine(def.to_def_id()) {
tcx.ensure_with_value().mir_coroutine_witnesses(def);
}
let mir_borrowck = tcx.mir_borrowck(def);
// We only need to borrowck non-synthetic MIR.
let tainted_by_errors =
if !tcx.is_synthetic_mir(def) { tcx.mir_borrowck(def).tainted_by_errors } else { None };
let is_fn_like = tcx.def_kind(def).is_fn_like();
if is_fn_like {
@ -410,7 +409,8 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
let (body, _) = tcx.mir_promoted(def);
let mut body = body.steal();
if let Some(error_reported) = mir_borrowck.tainted_by_errors {
if let Some(error_reported) = tainted_by_errors {
body.tainted_by_errors = Some(error_reported);
}
@ -660,14 +660,6 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
// visited does not depend on the optimization level.
// We do not use `run_passes` for this as that might skip the pass if `injection_phase` is set.
mentioned_items::MentionedItems.run_pass(tcx, &mut body);
// If this has an associated by-move async closure body, that doesn't get run through these
// passes itself, it gets "tagged along" by the pass manager. Since we're not using the pass
// manager we have to do this by hand.
if let Some(coroutine) = body.coroutine.as_mut() {
if let Some(by_move_body) = coroutine.by_move_body.as_mut() {
mentioned_items::MentionedItems.run_pass(tcx, by_move_body);
}
}
// If `mir_drops_elaborated_and_const_checked` found that the current body has unsatisfiable
// predicates, it will shrink the MIR to a single `unreachable` terminator.
@ -690,7 +682,9 @@ fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_
return tcx.arena.alloc(IndexVec::new());
}
if !tcx.is_synthetic_mir(def) {
tcx.ensure_with_value().mir_borrowck(def);
}
let mut promoted = tcx.mir_promoted(def).1.steal();
for body in &mut promoted {

6
compiler/rustc_mir_transform/src/pass_manager.rs
View File

@ -182,12 +182,6 @@ fn run_passes_inner<'tcx>(
body.pass_count = 1;
}
if let Some(coroutine) = body.coroutine.as_mut() {
if let Some(by_move_body) = coroutine.by_move_body.as_mut() {
run_passes_inner(tcx, by_move_body, passes, phase_change, validate_each);
}
}
}
pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) {

12
compiler/rustc_mir_transform/src/shim.rs
View File

@ -78,15 +78,11 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
receiver_by_ref,
} => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
ty::InstanceKind::CoroutineKindShim { coroutine_def_id } => {
return tcx.optimized_mir(coroutine_def_id).coroutine_by_move_body().unwrap().clone();
}
ty::InstanceKind::DropGlue(def_id, ty) => {
// FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end
// of this function. Is this intentional?
if let Some(ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
let coroutine_body = tcx.optimized_mir(*coroutine_def_id);
if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
let coroutine_body = tcx.optimized_mir(coroutine_def_id);
let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
else {
@ -105,7 +101,9 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
ty::ClosureKind::FnOnce
);
coroutine_body.coroutine_by_move_body().unwrap().coroutine_drop().unwrap()
tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
.coroutine_drop()
.unwrap()
};
let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);

19
compiler/rustc_mir_transform/src/validate.rs
View File

@ -102,25 +102,6 @@ impl<'tcx> MirPass<'tcx> for Validator {
}
}
}
// Enforce that coroutine-closure layouts are identical.
if let Some(layout) = body.coroutine_layout_raw()
&& let Some(by_move_body) = body.coroutine_by_move_body()
&& let Some(by_move_layout) = by_move_body.coroutine_layout_raw()
{
// FIXME(async_closures): We could do other validation here?
if layout.variant_fields.len() != by_move_layout.variant_fields.len() {
cfg_checker.fail(
Location::START,
format!(
"Coroutine layout has different number of variant fields from \
by-move coroutine layout:\n\
layout: {layout:#?}\n\
by_move_layout: {by_move_layout:#?}",
),
);
}
}
}
}

1
compiler/rustc_monomorphize/src/collector.rs
View File

@ -920,7 +920,6 @@ fn visit_instance_use<'tcx>(
| ty::InstanceKind::ReifyShim(..)
| ty::InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| ty::InstanceKind::CoroutineKindShim { .. }
| ty::InstanceKind::Item(..)
| ty::InstanceKind::FnPtrShim(..)
| ty::InstanceKind::CloneShim(..)

2
compiler/rustc_monomorphize/src/partitioning.rs
View File

@ -626,7 +626,6 @@ fn characteristic_def_id_of_mono_item<'tcx>(
| ty::InstanceKind::FnPtrShim(..)
| ty::InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| ty::InstanceKind::CoroutineKindShim { .. }
| ty::InstanceKind::Intrinsic(..)
| ty::InstanceKind::DropGlue(..)
| ty::InstanceKind::Virtual(..)
@ -796,7 +795,6 @@ fn mono_item_visibility<'tcx>(
| InstanceKind::Intrinsic(..)
| InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::CoroutineKindShim { .. }
| InstanceKind::DropGlue(..)
| InstanceKind::AsyncDropGlueCtorShim(..)
| InstanceKind::CloneShim(..)

2
compiler/rustc_monomorphize/src/polymorphize.rs
View File

@ -127,7 +127,7 @@ fn mark_used_by_default_parameters<'tcx>(
unused_parameters: &mut UnusedGenericParams,
) {
match tcx.def_kind(def_id) {
DefKind::Closure => {
DefKind::Closure | DefKind::SyntheticCoroutineBody => {
for param in &generics.own_params {
debug!(?param, "(closure/gen)");
unused_parameters.mark_used(param.index);

3
compiler/rustc_privacy/src/lib.rs
View File

@ -626,7 +626,8 @@ impl<'tcx> EmbargoVisitor<'tcx> {
| DefKind::Field
| DefKind::GlobalAsm
| DefKind::Impl { .. }
| DefKind::Closure => (),
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => (),
}
}
}

1
compiler/rustc_resolve/src/build_reduced_graph.rs
View File

@ -260,6 +260,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::Closure
| DefKind::SyntheticCoroutineBody
| DefKind::Impl { .. },
_,
)

1
compiler/rustc_smir/src/rustc_smir/convert/ty.rs
View File

@ -851,7 +851,6 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> {
| ty::InstanceKind::FnPtrAddrShim(..)
| ty::InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| ty::InstanceKind::CoroutineKindShim { .. }
| ty::InstanceKind::ThreadLocalShim(..)
| ty::InstanceKind::DropGlue(..)
| ty::InstanceKind::CloneShim(..)

4
compiler/rustc_smir/src/rustc_smir/mod.rs
View File

@ -114,7 +114,9 @@ pub(crate) fn new_item_kind(kind: DefKind) -> ItemKind {
| DefKind::GlobalAsm => {
unreachable!("Not a valid item kind: {kind:?}");
}
DefKind::Closure | DefKind::AssocFn | DefKind::Fn => ItemKind::Fn,
DefKind::Closure | DefKind::AssocFn | DefKind::Fn | DefKind::SyntheticCoroutineBody => {
ItemKind::Fn
}
DefKind::Const | DefKind::InlineConst | DefKind::AssocConst | DefKind::AnonConst => {
ItemKind::Const
}

3
compiler/rustc_symbol_mangling/src/legacy.rs
View File

@ -92,9 +92,6 @@ pub(super) fn mangle<'tcx>(
.write_str(if receiver_by_ref { "{{by-move-shim}}" } else { "{{by-ref-shim}}" })
.unwrap();
}
ty::InstanceKind::CoroutineKindShim { .. } => {
printer.write_str("{{by-move-body-shim}}").unwrap();
}
_ => {}
}

1
compiler/rustc_symbol_mangling/src/v0.rs
View File

@ -56,7 +56,6 @@ pub(super) fn mangle<'tcx>(
ty::InstanceKind::ConstructCoroutineInClosureShim { receiver_by_ref: false, .. } => {
Some("by_ref")
}
ty::InstanceKind::CoroutineKindShim { .. } => Some("by_move_body"),
_ => None,
};

33
compiler/rustc_ty_utils/src/abi.rs
View File

@ -169,39 +169,6 @@ fn fn_sig_for_fn_abi<'tcx>(
kind: ty::BoundRegionKind::BrEnv,
};
let mut ty = ty;
// When this `Closure` comes from a `CoroutineKindShim`,
// make sure we respect the `target_kind` in that shim.
// FIXME(async_closures): This shouldn't be needed, and we should be populating
// a separate def-id for these bodies.
if let InstanceKind::CoroutineKindShim { .. } = instance.def {
// Grab the parent coroutine-closure. It has the same args for the purposes
// of instantiation, so this will be okay to do.
let ty::CoroutineClosure(_, coroutine_closure_args) = *tcx
.instantiate_and_normalize_erasing_regions(
args,
param_env,
tcx.type_of(tcx.parent(did)),
)
.kind()
else {
bug!("CoroutineKindShim comes from calling a coroutine-closure");
};
let coroutine_closure_args = coroutine_closure_args.as_coroutine_closure();
ty = tcx.instantiate_bound_regions_with_erased(
coroutine_closure_args.coroutine_closure_sig().map_bound(|sig| {
sig.to_coroutine_given_kind_and_upvars(
tcx,
coroutine_closure_args.parent_args(),
did,
ty::ClosureKind::FnOnce,
tcx.lifetimes.re_erased,
coroutine_closure_args.tupled_upvars_ty(),
coroutine_closure_args.coroutine_captures_by_ref_ty(),
)
}),
);
}
let env_ty = Ty::new_mut_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), ty);
let pin_did = tcx.require_lang_item(LangItem::Pin, None);

3
compiler/rustc_ty_utils/src/implied_bounds.rs
View File

@ -144,7 +144,8 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<'
| DefKind::Field
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::Closure => ty::List::empty(),
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => ty::List::empty(),
}
}

3
compiler/rustc_ty_utils/src/opaque_types.rs
View File

@ -348,7 +348,8 @@ fn opaque_types_defined_by<'tcx>(
| DefKind::Field
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::Impl { .. } => {}
| DefKind::Impl { .. }
| DefKind::SyntheticCoroutineBody => {}
// Closures and coroutines are type checked with their parent, so we need to allow all
// opaques from the closure signature *and* from the parent body.
DefKind::Closure | DefKind::InlineConst => {

2
compiler/rustc_ty_utils/src/sig_types.rs
View File

@ -87,7 +87,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
// These are not part of a public API, they can only appear as hidden types, and there
// the interesting parts are solely in the signature of the containing item's opaque type
// or dyn type.
DefKind::InlineConst | DefKind::Closure => {}
DefKind::InlineConst | DefKind::Closure | DefKind::SyntheticCoroutineBody => {}
DefKind::Impl { of_trait } => {
if of_trait {
let span = tcx.hir_node_by_def_id(item).expect_item().expect_impl().of_trait.unwrap().path.span;

3
src/librustdoc/formats/item_type.rs
View File

@ -162,7 +162,8 @@ impl ItemType {
| DefKind::LifetimeParam
| DefKind::GlobalAsm
| DefKind::Impl { .. }
| DefKind::Closure => Self::ForeignType,
| DefKind::Closure
| DefKind::SyntheticCoroutineBody => Self::ForeignType,
}
}

4
src/librustdoc/passes/collect_intra_doc_links.rs
View File

@ -1950,7 +1950,9 @@ fn resolution_failure(
| TraitAlias
| TyParam
| Static { .. } => "associated item",
Impl { .. } | GlobalAsm => unreachable!("not a path"),
Impl { .. } | GlobalAsm | SyntheticCoroutineBody => {
unreachable!("not a path")
}
}
} else {
"associated item"

12
tests/ui/lint/unused/lint-unused-variables.stderr
View File

@ -16,6 +16,12 @@ error: unused variable: `a`
LL | a: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_a`
error: unused variable: `a`
--> $DIR/lint-unused-variables.rs:68:9
|
LL | a: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_a`
error: unused variable: `b`
--> $DIR/lint-unused-variables.rs:14:5
|
@ -58,12 +64,6 @@ error: unused variable: `b`
LL | b: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_b`
error: unused variable: `a`
--> $DIR/lint-unused-variables.rs:68:9
|
LL | a: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_a`
error: unused variable: `b`
--> $DIR/lint-unused-variables.rs:74:9
|

12
tests/ui/rfcs/rfc-2565-param-attrs/param-attrs-cfg.stderr
View File

@ -16,6 +16,12 @@ error: unused variable: `a`
LL | #[cfg(something)] a: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_a`
error: unused variable: `a`
--> $DIR/param-attrs-cfg.rs:107:27
|
LL | #[cfg(something)] a: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_a`
error: unused variable: `b`
--> $DIR/param-attrs-cfg.rs:30:23
|
@ -100,12 +106,6 @@ error: unused variable: `c`
LL | #[cfg_attr(nothing, cfg(nothing))] c: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_c`
error: unused variable: `a`
--> $DIR/param-attrs-cfg.rs:107:27
|
LL | #[cfg(something)] a: i32,
| ^ help: if this is intentional, prefix it with an underscore: `_a`
error: unused variable: `b`
--> $DIR/param-attrs-cfg.rs:113:27
|