Auto merge of #18267 - lnicola:sync-from-rust, r=lnicola

minor: Sync from downstream

.github/ISSUE_TEMPLATE/blank_issue.md

@@ -1,4 +0,0 @@
----
-name: Blank Issue
-about: Create a blank issue.
----

.github/pull_request_template.md

@@ -7,6 +7,6 @@ tracking issue or there are none, feel free to ignore this.
 This PR will get automatically assigned to a reviewer. In case you would like
 a specific user to review your work, you can assign it to them by using
 
-    r​? <reviewer name>
+    r\? <reviewer name> (with the `\` removed)
 -->
 <!-- homu-ignore:end -->

.github/workflows/ci.yml

@@ -212,6 +212,16 @@ jobs:
         # erroring about invalid credentials instead.
         if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
 
+      - name: upload job metrics to DataDog
+        if: needs.calculate_matrix.outputs.run_type != 'pr'
+        env:
+          DATADOG_SITE: datadoghq.com
+          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+          DD_GITHUB_JOB_NAME: ${{ matrix.name }}
+        run: |
+          npm install -g @datadog/datadog-ci@^2.x.x
+          python3 src/ci/scripts/upload-build-metrics.py build/cpu-usage.csv
+
   # This job isused to tell bors the final status of the build, as there is no practical way to detect
   # when a workflow is successful listening to webhooks only in our current bors implementation (homu).
   outcome:

.github/workflows/dependencies.yml

@@ -61,9 +61,11 @@ jobs:
           rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN
           rustup default $TOOLCHAIN
 
-      - name: cargo update
+      - name: cargo update compiler & tools
         # Remove first line that always just says "Updating crates.io index"
-        run: cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
+        run: |
+          echo -e "\ncompiler & tools dependencies:" >> cargo_update.log
+          cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
       - name: cargo update library
         run: |
           echo -e "\nlibrary dependencies:" >> cargo_update.log

.gitignore

@@ -25,6 +25,7 @@ Session.vim
 .favorites.json
 .settings/
 .vs/
+.dir-locals.el
 
 ## Tool
 .valgrindrc
@@ -85,4 +86,13 @@ package.json
 ## Rustdoc GUI tests
 tests/rustdoc-gui/src/**.lock
 
+## direnv
+.envrc
+.direnv/
+
+## nix
+flake.nix
+flake.lock
+default.nix
+
 # Before adding new lines, see the comment at the top.

INSTALL.md

@@ -79,9 +79,23 @@ See [the rustc-dev-guide for more info][sysllvm].
    ./configure
    ```
 
-   If you plan to use `x.py install` to create an installation, it is
-   recommended that you set the `prefix` value in the `[install]` section to a
-   directory: `./configure --set install.prefix=<path>`
+   If you plan to use `x.py install` to create an installation, you can either
+   set `DESTDIR` environment variable to your custom directory path:
+
+   ```bash
+   export DESTDIR=<path>
+   ```
+
+   or set `prefix` and `sysconfdir` in the `[install]` section to your custom
+   directory path:
+
+   ```sh
+   ./configure --set install.prefix=<path> --set install.sysconfdir=<path>
+   ```
+
+   When the `DESTDIR` environment variable is present, the `prefix` and
+   `sysconfdir` values are combined with the path from the `DESTDIR`
+   environment variable.
 
 3. Build and install:
 

compiler/rustc_ast/src/ast.rs

@@ -511,7 +511,7 @@ pub enum MetaItemKind {
     /// List meta item.
     ///
     /// E.g., `#[derive(..)]`, where the field represents the `..`.
-    List(ThinVec<NestedMetaItem>),
+    List(ThinVec<MetaItemInner>),
 
     /// Name value meta item.
     ///
@@ -523,7 +523,7 @@ pub enum MetaItemKind {
 ///
 /// E.g., each of `Clone`, `Copy` in `#[derive(Clone, Copy)]`.
 #[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
-pub enum NestedMetaItem {
+pub enum MetaItemInner {
     /// A full MetaItem, for recursive meta items.
     MetaItem(MetaItem),
 
@@ -2278,7 +2278,7 @@ impl InlineAsmOptions {
     pub const COUNT: usize = Self::all().bits().count_ones() as usize;
 
     pub const GLOBAL_OPTIONS: Self = Self::ATT_SYNTAX.union(Self::RAW);
-    pub const NAKED_OPTIONS: Self = Self::ATT_SYNTAX.union(Self::RAW).union(Self::NORETURN);
+    pub const NAKED_OPTIONS: Self = Self::ATT_SYNTAX.union(Self::RAW);
 
     pub fn human_readable_names(&self) -> Vec<&'static str> {
         let mut options = vec![];
@@ -2434,6 +2434,32 @@ pub enum AsmMacro {
     NakedAsm,
 }
 
+impl AsmMacro {
+    pub const fn macro_name(self) -> &'static str {
+        match self {
+            AsmMacro::Asm => "asm",
+            AsmMacro::GlobalAsm => "global_asm",
+            AsmMacro::NakedAsm => "naked_asm",
+        }
+    }
+
+    pub const fn is_supported_option(self, option: InlineAsmOptions) -> bool {
+        match self {
+            AsmMacro::Asm => true,
+            AsmMacro::GlobalAsm => InlineAsmOptions::GLOBAL_OPTIONS.contains(option),
+            AsmMacro::NakedAsm => InlineAsmOptions::NAKED_OPTIONS.contains(option),
+        }
+    }
+
+    pub const fn diverges(self, options: InlineAsmOptions) -> bool {
+        match self {
+            AsmMacro::Asm => options.contains(InlineAsmOptions::NORETURN),
+            AsmMacro::GlobalAsm => true,
+            AsmMacro::NakedAsm => true,
+        }
+    }
+}
+
 /// Inline assembly.
 ///
 /// E.g., `asm!("NOP");`.

compiler/rustc_ast/src/attr/mod.rs

@@ -11,7 +11,7 @@ use thin_vec::{ThinVec, thin_vec};
 
 use crate::ast::{
     AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute, DUMMY_NODE_ID,
-    DelimArgs, Expr, ExprKind, LitKind, MetaItem, MetaItemKind, MetaItemLit, NestedMetaItem,
+    DelimArgs, Expr, ExprKind, LitKind, MetaItem, MetaItemInner, MetaItemKind, MetaItemLit,
     NormalAttr, Path, PathSegment, Safety,
 };
 use crate::ptr::P;
@@ -136,7 +136,7 @@ impl Attribute {
         }
     }
 
-    pub fn meta_item_list(&self) -> Option<ThinVec<NestedMetaItem>> {
+    pub fn meta_item_list(&self) -> Option<ThinVec<MetaItemInner>> {
         match &self.kind {
             AttrKind::Normal(normal) => normal.item.meta_item_list(),
             AttrKind::DocComment(..) => None,
@@ -223,7 +223,7 @@ impl AttrItem {
         self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
     }
 
-    fn meta_item_list(&self) -> Option<ThinVec<NestedMetaItem>> {
+    fn meta_item_list(&self) -> Option<ThinVec<MetaItemInner>> {
         match &self.args {
             AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => {
                 MetaItemKind::list_from_tokens(args.tokens.clone())
@@ -285,7 +285,7 @@ impl MetaItem {
         matches!(self.kind, MetaItemKind::Word)
     }
 
-    pub fn meta_item_list(&self) -> Option<&[NestedMetaItem]> {
+    pub fn meta_item_list(&self) -> Option<&[MetaItemInner]> {
         match &self.kind {
             MetaItemKind::List(l) => Some(&**l),
             _ => None,
@@ -393,11 +393,11 @@ impl MetaItem {
 }
 
 impl MetaItemKind {
-    fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> {
+    fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<MetaItemInner>> {
         let mut tokens = tokens.trees().peekable();
         let mut result = ThinVec::new();
         while tokens.peek().is_some() {
-            let item = NestedMetaItem::from_tokens(&mut tokens)?;
+            let item = MetaItemInner::from_tokens(&mut tokens)?;
             result.push(item);
             match tokens.next() {
                 None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
@@ -460,11 +460,11 @@ impl MetaItemKind {
     }
 }
 
-impl NestedMetaItem {
+impl MetaItemInner {
     pub fn span(&self) -> Span {
         match self {
-            NestedMetaItem::MetaItem(item) => item.span,
-            NestedMetaItem::Lit(lit) => lit.span,
+            MetaItemInner::MetaItem(item) => item.span,
+            MetaItemInner::Lit(lit) => lit.span,
         }
     }
 
@@ -488,7 +488,7 @@ impl NestedMetaItem {
     }
 
     /// Gets a list of inner meta items from a list `MetaItem` type.
-    pub fn meta_item_list(&self) -> Option<&[NestedMetaItem]> {
+    pub fn meta_item_list(&self) -> Option<&[MetaItemInner]> {
         self.meta_item().and_then(|meta_item| meta_item.meta_item_list())
     }
 
@@ -519,18 +519,28 @@ impl NestedMetaItem {
         self.meta_item().and_then(|meta_item| meta_item.value_str())
     }
 
-    /// Returns the `MetaItemLit` if `self` is a `NestedMetaItem::Literal`s.
+    /// Returns the `MetaItemLit` if `self` is a `MetaItemInner::Literal`s.
     pub fn lit(&self) -> Option<&MetaItemLit> {
         match self {
-            NestedMetaItem::Lit(lit) => Some(lit),
+            MetaItemInner::Lit(lit) => Some(lit),
             _ => None,
         }
     }
 
-    /// Returns the `MetaItem` if `self` is a `NestedMetaItem::MetaItem`.
+    /// Returns the `MetaItem` if `self` is a `MetaItemInner::MetaItem` or if it's
+    /// `MetaItemInner::Lit(MetaItemLit { kind: LitKind::Bool(_), .. })`.
+    pub fn meta_item_or_bool(&self) -> Option<&MetaItemInner> {
+        match self {
+            MetaItemInner::MetaItem(_item) => Some(self),
+            MetaItemInner::Lit(MetaItemLit { kind: LitKind::Bool(_), .. }) => Some(self),
+            _ => None,
+        }
+    }
+
+    /// Returns the `MetaItem` if `self` is a `MetaItemInner::MetaItem`.
     pub fn meta_item(&self) -> Option<&MetaItem> {
         match self {
-            NestedMetaItem::MetaItem(item) => Some(item),
+            MetaItemInner::MetaItem(item) => Some(item),
             _ => None,
         }
     }
@@ -540,22 +550,22 @@ impl NestedMetaItem {
         self.meta_item().is_some()
     }
 
-    fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
+    fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItemInner>
     where
         I: Iterator<Item = &'a TokenTree>,
     {
         match tokens.peek() {
             Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
                 tokens.next();
-                return Some(NestedMetaItem::Lit(lit));
+                return Some(MetaItemInner::Lit(lit));
             }
             Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
                 tokens.next();
-                return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
+                return MetaItemInner::from_tokens(&mut inner_tokens.trees().peekable());
             }
             _ => {}
         }
-        MetaItem::from_tokens(tokens).map(NestedMetaItem::MetaItem)
+        MetaItem::from_tokens(tokens).map(MetaItemInner::MetaItem)
     }
 }
 
@@ -666,6 +676,6 @@ pub fn contains_name(attrs: &[Attribute], name: Symbol) -> bool {
     find_by_name(attrs, name).is_some()
 }
 
-pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
+pub fn list_contains_name(items: &[MetaItemInner], name: Symbol) -> bool {
     items.iter().any(|item| item.has_name(name))
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -83,7 +83,7 @@ pub trait MutVisitor: Sized {
         walk_crate(self, c)
     }
 
-    fn visit_meta_list_item(&mut self, list_item: &mut NestedMetaItem) {
+    fn visit_meta_list_item(&mut self, list_item: &mut MetaItemInner) {
         walk_meta_list_item(self, list_item);
     }
 
@@ -659,10 +659,10 @@ fn walk_macro_def<T: MutVisitor>(vis: &mut T, macro_def: &mut MacroDef) {
     visit_delim_args(vis, body);
 }
 
-fn walk_meta_list_item<T: MutVisitor>(vis: &mut T, li: &mut NestedMetaItem) {
+fn walk_meta_list_item<T: MutVisitor>(vis: &mut T, li: &mut MetaItemInner) {
     match li {
-        NestedMetaItem::MetaItem(mi) => vis.visit_meta_item(mi),
-        NestedMetaItem::Lit(_lit) => {}
+        MetaItemInner::MetaItem(mi) => vis.visit_meta_item(mi),
+        MetaItemInner::Lit(_lit) => {}
     }
 }
 

compiler/rustc_ast_lowering/src/index.rs

@@ -226,6 +226,14 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> {
         });
     }
 
+    fn visit_opaque_ty(&mut self, opaq: &'hir OpaqueTy<'hir>) {
+        self.insert(opaq.span, opaq.hir_id, Node::OpaqueTy(opaq));
+
+        self.with_parent(opaq.hir_id, |this| {
+            intravisit::walk_opaque_ty(this, opaq);
+        });
+    }
+
     fn visit_anon_const(&mut self, constant: &'hir AnonConst) {
         // FIXME: use real span?
         self.insert(DUMMY_SP, constant.hir_id, Node::AnonConst(constant));

compiler/rustc_ast_lowering/src/item.rs

@@ -286,7 +286,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                                 parent: this.local_def_id(id),
                                 in_assoc_ty: false,
                             },
-                            fn_kind: None,
                         }),
                     },
                 );
@@ -983,7 +982,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                                     parent: this.local_def_id(i.id),
                                     in_assoc_ty: true,
                                 },
-                                fn_kind: None,
                             });
                             hir::ImplItemKind::Type(ty)
                         }

compiler/rustc_ast_lowering/src/lib.rs

@@ -288,12 +288,7 @@ enum ImplTraitContext {
     /// Example: `fn foo() -> impl Debug`, where `impl Debug` is conceptually
     /// equivalent to a new opaque type like `type T = impl Debug; fn foo() -> T`.
     ///
-    OpaqueTy {
-        origin: hir::OpaqueTyOrigin,
-        /// Only used to change the lifetime capture rules, since
-        /// RPITIT captures all in scope, RPIT does not.
-        fn_kind: Option<FnDeclKind>,
-    },
+    OpaqueTy { origin: hir::OpaqueTyOrigin },
     /// `impl Trait` is unstably accepted in this position.
     FeatureGated(ImplTraitPosition, Symbol),
     /// `impl Trait` is not accepted in this position.
@@ -1404,14 +1399,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             TyKind::ImplTrait(def_node_id, bounds) => {
                 let span = t.span;
                 match itctx {
-                    ImplTraitContext::OpaqueTy { origin, fn_kind } => self.lower_opaque_impl_trait(
-                        span,
-                        origin,
-                        *def_node_id,
-                        bounds,
-                        fn_kind,
-                        itctx,
-                    ),
+                    ImplTraitContext::OpaqueTy { origin } => {
+                        self.lower_opaque_impl_trait(span, origin, *def_node_id, bounds, itctx)
+                    }
                     ImplTraitContext::Universal => {
                         if let Some(span) = bounds.iter().find_map(|bound| match *bound {
                             ast::GenericBound::Use(_, span) => Some(span),
@@ -1513,7 +1503,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         origin: hir::OpaqueTyOrigin,
         opaque_ty_node_id: NodeId,
         bounds: &GenericBounds,
-        fn_kind: Option<FnDeclKind>,
         itctx: ImplTraitContext,
     ) -> hir::TyKind<'hir> {
         // Make sure we know that some funky desugaring has been going on here.
@@ -1555,11 +1544,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         .map(|(ident, id, _)| Lifetime { id, ident })
                         .collect()
                 }
-                hir::OpaqueTyOrigin::FnReturn(..) => {
-                    if matches!(
-                        fn_kind.expect("expected RPITs to be lowered with a FnKind"),
-                        FnDeclKind::Impl | FnDeclKind::Trait
-                    ) || self.tcx.features().lifetime_capture_rules_2024
+                hir::OpaqueTyOrigin::FnReturn { in_trait_or_impl, .. } => {
+                    if in_trait_or_impl.is_some()
+                        || self.tcx.features().lifetime_capture_rules_2024
                         || span.at_least_rust_2024()
                     {
                         // return-position impl trait in trait was decided to capture all
@@ -1576,16 +1563,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         lifetime_collector::lifetimes_in_bounds(self.resolver, bounds)
                     }
                 }
-                hir::OpaqueTyOrigin::AsyncFn(..) => {
+                hir::OpaqueTyOrigin::AsyncFn { .. } => {
                     unreachable!("should be using `lower_async_fn_ret_ty`")
                 }
             }
         };
         debug!(?captured_lifetimes_to_duplicate);
 
-        match fn_kind {
-            // Deny `use<>` on RPITIT in trait/trait-impl for now.
-            Some(FnDeclKind::Trait | FnDeclKind::Impl) => {
+        // Feature gate for RPITIT + use<..>
+        match origin {
+            rustc_hir::OpaqueTyOrigin::FnReturn { in_trait_or_impl: Some(_), .. } => {
                 if let Some(span) = bounds.iter().find_map(|bound| match *bound {
                     ast::GenericBound::Use(_, span) => Some(span),
                     _ => None,
@@ -1593,20 +1580,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     self.tcx.dcx().emit_err(errors::NoPreciseCapturesOnRpitit { span });
                 }
             }
-            None
-            | Some(
-                FnDeclKind::Fn
-                | FnDeclKind::Inherent
-                | FnDeclKind::ExternFn
-                | FnDeclKind::Closure
-                | FnDeclKind::Pointer,
-            ) => {}
+            _ => {}
         }
 
         self.lower_opaque_inner(
             opaque_ty_node_id,
             origin,
-            matches!(fn_kind, Some(FnDeclKind::Trait)),
             captured_lifetimes_to_duplicate,
             span,
             opaque_ty_span,
@@ -1618,14 +1597,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         &mut self,
         opaque_ty_node_id: NodeId,
         origin: hir::OpaqueTyOrigin,
-        in_trait: bool,
         captured_lifetimes_to_duplicate: FxIndexSet<Lifetime>,
         span: Span,
         opaque_ty_span: Span,
         lower_item_bounds: impl FnOnce(&mut Self) -> &'hir [hir::GenericBound<'hir>],
     ) -> hir::TyKind<'hir> {
         let opaque_ty_def_id = self.local_def_id(opaque_ty_node_id);
-        debug!(?opaque_ty_def_id);
+        let opaque_ty_hir_id = self.lower_node_id(opaque_ty_node_id);
+        debug!(?opaque_ty_def_id, ?opaque_ty_hir_id);
 
         // Map from captured (old) lifetime to synthetic (new) lifetime.
         // Used to resolve lifetimes in the bounds of the opaque.
@@ -1698,7 +1677,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
         }
 
-        self.with_hir_id_owner(opaque_ty_node_id, |this| {
+        let opaque_ty_def = self.with_def_id_parent(opaque_ty_def_id, |this| {
             // Install the remapping from old to new (if any). This makes sure that
             // any lifetimes that would have resolved to the def-id of captured
             // lifetimes are remapped to the new *synthetic* lifetimes of the opaque.
@@ -1736,7 +1715,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
             let lifetime_mapping = self.arena.alloc_slice(&synthesized_lifetime_args);
 
-            let opaque_ty_item = hir::OpaqueTy {
+            trace!("registering opaque type with id {:#?}", opaque_ty_def_id);
+            let opaque_ty_def = hir::OpaqueTy {
+                hir_id: opaque_ty_hir_id,
+                def_id: opaque_ty_def_id,
                 generics: this.arena.alloc(hir::Generics {
                     params: generic_params,
                     predicates: &[],
@@ -1747,20 +1729,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 bounds,
                 origin,
                 lifetime_mapping,
-                in_trait,
-            };
-
-            // Generate an `type Foo = impl Trait;` declaration.
-            trace!("registering opaque type with id {:#?}", opaque_ty_def_id);
-            let opaque_ty_item = hir::Item {
-                owner_id: hir::OwnerId { def_id: opaque_ty_def_id },
-                ident: Ident::empty(),
-                kind: hir::ItemKind::OpaqueTy(this.arena.alloc(opaque_ty_item)),
-                vis_span: this.lower_span(span.shrink_to_lo()),
                 span: this.lower_span(opaque_ty_span),
             };
-
-            hir::OwnerNode::Item(this.arena.alloc(opaque_ty_item))
+            this.arena.alloc(opaque_ty_def)
         });
 
         let generic_args = self.arena.alloc_from_iter(
@@ -1773,11 +1744,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         // Foo = impl Trait` is, internally, created as a child of the
         // async fn, so the *type parameters* are inherited. It's
         // only the lifetime parameters that we must supply.
-        hir::TyKind::OpaqueDef(
-            hir::ItemId { owner_id: hir::OwnerId { def_id: opaque_ty_def_id } },
-            generic_args,
-            in_trait,
-        )
+        hir::TyKind::OpaqueDef(opaque_ty_def, generic_args)
     }
 
     fn lower_precise_capturing_args(
@@ -1864,12 +1831,23 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             None => match &decl.output {
                 FnRetTy::Ty(ty) => {
                     let itctx = match kind {
-                        FnDeclKind::Fn
-                        | FnDeclKind::Inherent
-                        | FnDeclKind::Trait
-                        | FnDeclKind::Impl => ImplTraitContext::OpaqueTy {
-                            origin: hir::OpaqueTyOrigin::FnReturn(self.local_def_id(fn_node_id)),
-                            fn_kind: Some(kind),
+                        FnDeclKind::Fn | FnDeclKind::Inherent => ImplTraitContext::OpaqueTy {
+                            origin: hir::OpaqueTyOrigin::FnReturn {
+                                parent: self.local_def_id(fn_node_id),
+                                in_trait_or_impl: None,
+                            },
+                        },
+                        FnDeclKind::Trait => ImplTraitContext::OpaqueTy {
+                            origin: hir::OpaqueTyOrigin::FnReturn {
+                                parent: self.local_def_id(fn_node_id),
+                                in_trait_or_impl: Some(hir::RpitContext::Trait),
+                            },
+                        },
+                        FnDeclKind::Impl => ImplTraitContext::OpaqueTy {
+                            origin: hir::OpaqueTyOrigin::FnReturn {
+                                parent: self.local_def_id(fn_node_id),
+                                in_trait_or_impl: Some(hir::RpitContext::TraitImpl),
+                            },
                         },
                         FnDeclKind::ExternFn => {
                             ImplTraitContext::Disallowed(ImplTraitPosition::ExternFnReturn)
@@ -1951,10 +1929,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             .map(|(ident, id, _)| Lifetime { id, ident })
             .collect();
 
+        let in_trait_or_impl = match fn_kind {
+            FnDeclKind::Trait => Some(hir::RpitContext::Trait),
+            FnDeclKind::Impl => Some(hir::RpitContext::TraitImpl),
+            FnDeclKind::Fn | FnDeclKind::Inherent => None,
+            FnDeclKind::ExternFn | FnDeclKind::Closure | FnDeclKind::Pointer => unreachable!(),
+        };
+
         let opaque_ty_ref = self.lower_opaque_inner(
             opaque_ty_node_id,
-            hir::OpaqueTyOrigin::AsyncFn(fn_def_id),
-            matches!(fn_kind, FnDeclKind::Trait),
+            hir::OpaqueTyOrigin::AsyncFn { parent: fn_def_id, in_trait_or_impl },
             captured_lifetimes,
             span,
             opaque_ty_span,
@@ -1964,8 +1948,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     coro,
                     opaque_ty_span,
                     ImplTraitContext::OpaqueTy {
-                        origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id),
-                        fn_kind: Some(fn_kind),
+                        origin: hir::OpaqueTyOrigin::FnReturn {
+                            parent: fn_def_id,
+                            in_trait_or_impl,
+                        },
                     },
                 );
                 arena_vec![this; bound]

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -63,7 +63,7 @@ impl TraitOrTraitImpl {
 }
 
 struct AstValidator<'a> {
-    session: &'a Session,
+    sess: &'a Session,
     features: &'a Features,
 
     /// The span of the `extern` in an `extern { ... }` block, if any.
@@ -267,7 +267,7 @@ impl<'a> AstValidator<'a> {
     }
 
     fn dcx(&self) -> DiagCtxtHandle<'a> {
-        self.session.dcx()
+        self.sess.dcx()
     }
 
     fn visibility_not_permitted(&self, vis: &Visibility, note: errors::VisibilityNotPermittedNote) {
@@ -359,7 +359,7 @@ impl<'a> AstValidator<'a> {
             in_impl: matches!(parent, TraitOrTraitImpl::TraitImpl { .. }),
             const_context_label: parent_constness,
             remove_const_sugg: (
-                self.session.source_map().span_extend_while_whitespace(span),
+                self.sess.source_map().span_extend_while_whitespace(span),
                 match parent_constness {
                     Some(_) => rustc_errors::Applicability::MachineApplicable,
                     None => rustc_errors::Applicability::MaybeIncorrect,
@@ -472,7 +472,7 @@ impl<'a> AstValidator<'a> {
 
     fn check_defaultness(&self, span: Span, defaultness: Defaultness) {
         if let Defaultness::Default(def_span) = defaultness {
-            let span = self.session.source_map().guess_head_span(span);
+            let span = self.sess.source_map().guess_head_span(span);
             self.dcx().emit_err(errors::ForbiddenDefault { span, def_span });
         }
     }
@@ -480,7 +480,7 @@ impl<'a> AstValidator<'a> {
     /// If `sp` ends with a semicolon, returns it as a `Span`
     /// Otherwise, returns `sp.shrink_to_hi()`
     fn ending_semi_or_hi(&self, sp: Span) -> Span {
-        let source_map = self.session.source_map();
+        let source_map = self.sess.source_map();
         let end = source_map.end_point(sp);
 
         if source_map.span_to_snippet(end).is_ok_and(|s| s == ";") {
@@ -552,7 +552,7 @@ impl<'a> AstValidator<'a> {
     }
 
     fn current_extern_span(&self) -> Span {
-        self.session.source_map().guess_head_span(self.extern_mod.unwrap())
+        self.sess.source_map().guess_head_span(self.extern_mod.unwrap())
     }
 
     /// An `fn` in `extern { ... }` cannot have qualifiers, e.g. `async fn`.
@@ -648,7 +648,7 @@ impl<'a> AstValidator<'a> {
         if ident.name.as_str().is_ascii() {
             return;
         }
-        let span = self.session.source_map().guess_head_span(item_span);
+        let span = self.sess.source_map().guess_head_span(item_span);
         self.dcx().emit_err(errors::NoMangleAscii { span });
     }
 
@@ -753,7 +753,7 @@ impl<'a> AstValidator<'a> {
                     self.dcx().emit_err(errors::PatternFnPointer { span });
                 });
                 if let Extern::Implicit(_) = bfty.ext {
-                    let sig_span = self.session.source_map().next_point(ty.span.shrink_to_lo());
+                    let sig_span = self.sess.source_map().next_point(ty.span.shrink_to_lo());
                     self.maybe_lint_missing_abi(sig_span, ty.id);
                 }
             }
@@ -795,7 +795,7 @@ impl<'a> AstValidator<'a> {
         // FIXME(davidtwco): This is a hack to detect macros which produce spans of the
         // call site which do not have a macro backtrace. See #61963.
         if self
-            .session
+            .sess
             .source_map()
             .span_to_snippet(span)
             .is_ok_and(|snippet| !snippet.starts_with("#["))
@@ -885,7 +885,7 @@ fn validate_generic_param_order(dcx: DiagCtxtHandle<'_>, generics: &[GenericPara
 
 impl<'a> Visitor<'a> for AstValidator<'a> {
     fn visit_attribute(&mut self, attr: &Attribute) {
-        validate_attr::check_attr(&self.session.psess, attr);
+        validate_attr::check_attr(&self.sess.psess, attr);
     }
 
     fn visit_ty(&mut self, ty: &'a Ty) {
@@ -1192,7 +1192,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 } else if where_clauses.after.has_where_token {
                     self.dcx().emit_err(errors::WhereClauseAfterTypeAlias {
                         span: where_clauses.after.span,
-                        help: self.session.is_nightly_build(),
+                        help: self.sess.is_nightly_build(),
                     });
                 }
             }
@@ -1328,7 +1328,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     (BoundKind::SuperTraits, BoundConstness::Never, BoundPolarity::Maybe(_))
                         if !self.features.more_maybe_bounds =>
                     {
-                        self.session
+                        self.sess
                             .create_feature_err(
                                 errors::OptionalTraitSupertrait {
                                     span: trait_ref.span,
@@ -1341,7 +1341,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     (BoundKind::TraitObject, BoundConstness::Never, BoundPolarity::Maybe(_))
                         if !self.features.more_maybe_bounds =>
                     {
-                        self.session
+                        self.sess
                             .create_feature_err(
                                 errors::OptionalTraitObject { span: trait_ref.span },
                                 sym::more_maybe_bounds,
@@ -1752,13 +1752,13 @@ fn deny_equality_constraints(
 }
 
 pub fn check_crate(
-    session: &Session,
+    sess: &Session,
     features: &Features,
     krate: &Crate,
     lints: &mut LintBuffer,
 ) -> bool {
     let mut validator = AstValidator {
-        session,
+        sess,
         features,
         extern_mod: None,
         outer_trait_or_trait_impl: None,

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -4,9 +4,9 @@ use rustc_ast::{NodeId, PatKind, attr, token};
 use rustc_feature::{AttributeGate, BUILTIN_ATTRIBUTE_MAP, BuiltinAttribute, Features, GateIssue};
 use rustc_session::Session;
 use rustc_session::parse::{feature_err, feature_err_issue, feature_warn};
-use rustc_span::Span;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::sym;
+use rustc_span::{Span, Symbol};
 use rustc_target::spec::abi;
 use thin_vec::ThinVec;
 
@@ -186,9 +186,9 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
         }
         // Check unstable flavors of the `#[doc]` attribute.
         if attr.has_name(sym::doc) {
-            for nested_meta in attr.meta_item_list().unwrap_or_default() {
+            for meta_item_inner in attr.meta_item_list().unwrap_or_default() {
                 macro_rules! gate_doc { ($($s:literal { $($name:ident => $feature:ident)* })*) => {
-                    $($(if nested_meta.has_name(sym::$name) {
+                    $($(if meta_item_inner.has_name(sym::$name) {
                         let msg = concat!("`#[doc(", stringify!($name), ")]` is ", $s);
                         gate!(self, $feature, attr.span, msg);
                     })*)*
@@ -483,6 +483,8 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
     maybe_stage_features(sess, features, krate);
     check_incompatible_features(sess, features);
+    check_new_solver_banned_features(sess, features);
+
     let mut visitor = PostExpansionVisitor { sess, features };
 
     let spans = sess.psess.gated_spans.spans.borrow();
@@ -662,3 +664,22 @@ fn check_incompatible_features(sess: &Session, features: &Features) {
         }
     }
 }
+
+fn check_new_solver_banned_features(sess: &Session, features: &Features) {
+    if !sess.opts.unstable_opts.next_solver.is_some_and(|n| n.globally) {
+        return;
+    }
+
+    // Ban GCE with the new solver, because it does not implement GCE correctly.
+    if let Some(&(_, gce_span, _)) = features
+        .declared_lang_features
+        .iter()
+        .find(|&&(feat, _, _)| feat == sym::generic_const_exprs)
+    {
+        sess.dcx().emit_err(errors::IncompatibleFeatures {
+            spans: vec![gce_span],
+            f1: Symbol::intern("-Znext-solver=globally"),
+            f2: sym::generic_const_exprs,
+        });
+    }
+}

compiler/rustc_ast_pretty/src/pprust/mod.rs

@@ -67,7 +67,7 @@ pub fn vis_to_string(v: &ast::Visibility) -> String {
     State::new().vis_to_string(v)
 }
 
-pub fn meta_list_item_to_string(li: &ast::NestedMetaItem) -> String {
+pub fn meta_list_item_to_string(li: &ast::MetaItemInner) -> String {
     State::new().meta_list_item_to_string(li)
 }
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -2006,10 +2006,10 @@ impl<'a> State<'a> {
         self.print_attribute_inline(attr, false)
     }
 
-    fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) {
+    fn print_meta_list_item(&mut self, item: &ast::MetaItemInner) {
         match item {
-            ast::NestedMetaItem::MetaItem(mi) => self.print_meta_item(mi),
-            ast::NestedMetaItem::Lit(lit) => self.print_meta_item_lit(lit),
+            ast::MetaItemInner::MetaItem(mi) => self.print_meta_item(mi),
+            ast::MetaItemInner::Lit(lit) => self.print_meta_item_lit(lit),
         }
     }
 
@@ -2054,7 +2054,7 @@ impl<'a> State<'a> {
         Self::to_string(|s| s.print_path_segment(p, false))
     }
 
-    pub(crate) fn meta_list_item_to_string(&self, li: &ast::NestedMetaItem) -> String {
+    pub(crate) fn meta_list_item_to_string(&self, li: &ast::MetaItemInner) -> String {
         Self::to_string(|s| s.print_meta_list_item(li))
     }
 

compiler/rustc_attr/messages.ftl

@@ -107,6 +107,8 @@ attr_unknown_version_literal =
 attr_unstable_cfg_target_compact =
     compact `cfg(target(..))` is experimental and subject to change
 
+attr_unsupported_literal_cfg_boolean =
+    literal in `cfg` predicate value must be a boolean
 attr_unsupported_literal_cfg_string =
     literal in `cfg` predicate value must be a string
 attr_unsupported_literal_deprecated_kv_pair =

compiler/rustc_attr/src/builtin.rs

@@ -4,7 +4,7 @@ use std::num::NonZero;
 
 use rustc_abi::Align;
 use rustc_ast::{
-    self as ast, Attribute, LitKind, MetaItem, MetaItemKind, MetaItemLit, NestedMetaItem, NodeId,
+    self as ast, Attribute, LitKind, MetaItem, MetaItemInner, MetaItemKind, MetaItemLit, NodeId,
     attr,
 };
 use rustc_ast_pretty::pprust;
@@ -18,7 +18,7 @@ use rustc_session::parse::feature_err;
 use rustc_session::{RustcVersion, Session};
 use rustc_span::Span;
 use rustc_span::hygiene::Transparency;
-use rustc_span::symbol::{Symbol, sym};
+use rustc_span::symbol::{Symbol, kw, sym};
 
 use crate::fluent_generated;
 use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
@@ -36,6 +36,7 @@ pub fn is_builtin_attr(attr: &Attribute) -> bool {
 pub(crate) enum UnsupportedLiteralReason {
     Generic,
     CfgString,
+    CfgBoolean,
     DeprecatedString,
     DeprecatedKvPair,
 }
@@ -80,6 +81,10 @@ impl Stability {
     pub fn is_stable(&self) -> bool {
         self.level.is_stable()
     }
+
+    pub fn stable_since(&self) -> Option<StableSince> {
+        self.level.stable_since()
+    }
 }
 
 /// Represents the `#[rustc_const_unstable]` and `#[rustc_const_stable]` attributes.
@@ -170,6 +175,12 @@ impl StabilityLevel {
     pub fn is_stable(&self) -> bool {
         matches!(self, StabilityLevel::Stable { .. })
     }
+    pub fn stable_since(&self) -> Option<StableSince> {
+        match *self {
+            StabilityLevel::Stable { since, .. } => Some(since),
+            StabilityLevel::Unstable { .. } => None,
+        }
+    }
 }
 
 #[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
@@ -523,7 +534,7 @@ pub struct Condition {
 
 /// Tests if a cfg-pattern matches the cfg set
 pub fn cfg_matches(
-    cfg: &ast::MetaItem,
+    cfg: &ast::MetaItemInner,
     sess: &Session,
     lint_node_id: NodeId,
     features: Option<&Features>,
@@ -594,22 +605,53 @@ pub fn parse_version(s: Symbol) -> Option<RustcVersion> {
 /// Evaluate a cfg-like condition (with `any` and `all`), using `eval` to
 /// evaluate individual items.
 pub fn eval_condition(
-    cfg: &ast::MetaItem,
+    cfg: &ast::MetaItemInner,
     sess: &Session,
     features: Option<&Features>,
     eval: &mut impl FnMut(Condition) -> bool,
 ) -> bool {
     let dcx = sess.dcx();
+
+    let cfg = match cfg {
+        ast::MetaItemInner::MetaItem(meta_item) => meta_item,
+        ast::MetaItemInner::Lit(MetaItemLit { kind: LitKind::Bool(b), .. }) => {
+            if let Some(features) = features {
+                // we can't use `try_gate_cfg` as symbols don't differentiate between `r#true`
+                // and `true`, and we want to keep the former working without feature gate
+                gate_cfg(
+                    &((
+                        if *b { kw::True } else { kw::False },
+                        sym::cfg_boolean_literals,
+                        |features: &Features| features.cfg_boolean_literals,
+                    )),
+                    cfg.span(),
+                    sess,
+                    features,
+                );
+            }
+            return *b;
+        }
+        _ => {
+            dcx.emit_err(session_diagnostics::UnsupportedLiteral {
+                span: cfg.span(),
+                reason: UnsupportedLiteralReason::CfgBoolean,
+                is_bytestr: false,
+                start_point_span: sess.source_map().start_point(cfg.span()),
+            });
+            return false;
+        }
+    };
+
     match &cfg.kind {
         ast::MetaItemKind::List(mis) if cfg.name_or_empty() == sym::version => {
             try_gate_cfg(sym::version, cfg.span, sess, features);
             let (min_version, span) = match &mis[..] {
-                [NestedMetaItem::Lit(MetaItemLit { kind: LitKind::Str(sym, ..), span, .. })] => {
+                [MetaItemInner::Lit(MetaItemLit { kind: LitKind::Str(sym, ..), span, .. })] => {
                     (sym, span)
                 }
                 [
-                    NestedMetaItem::Lit(MetaItemLit { span, .. })
-                    | NestedMetaItem::MetaItem(MetaItem { span, .. }),
+                    MetaItemInner::Lit(MetaItemLit { span, .. })
+                    | MetaItemInner::MetaItem(MetaItem { span, .. }),
                 ] => {
                     dcx.emit_err(session_diagnostics::ExpectedVersionLiteral { span: *span });
                     return false;
@@ -635,7 +677,7 @@ pub fn eval_condition(
         }
         ast::MetaItemKind::List(mis) => {
             for mi in mis.iter() {
-                if !mi.is_meta_item() {
+                if mi.meta_item_or_bool().is_none() {
                     dcx.emit_err(session_diagnostics::UnsupportedLiteral {
                         span: mi.span(),
                         reason: UnsupportedLiteralReason::Generic,
@@ -653,23 +695,19 @@ pub fn eval_condition(
                     .iter()
                     // We don't use any() here, because we want to evaluate all cfg condition
                     // as eval_condition can (and does) extra checks
-                    .fold(false, |res, mi| {
-                        res | eval_condition(mi.meta_item().unwrap(), sess, features, eval)
-                    }),
+                    .fold(false, |res, mi| res | eval_condition(mi, sess, features, eval)),
                 sym::all => mis
                     .iter()
                     // We don't use all() here, because we want to evaluate all cfg condition
                     // as eval_condition can (and does) extra checks
-                    .fold(true, |res, mi| {
-                        res & eval_condition(mi.meta_item().unwrap(), sess, features, eval)
-                    }),
+                    .fold(true, |res, mi| res & eval_condition(mi, sess, features, eval)),
                 sym::not => {
                     let [mi] = mis.as_slice() else {
                         dcx.emit_err(session_diagnostics::ExpectedOneCfgPattern { span: cfg.span });
                         return false;
                     };
 
-                    !eval_condition(mi.meta_item().unwrap(), sess, features, eval)
+                    !eval_condition(mi, sess, features, eval)
                 }
                 sym::target => {
                     if let Some(features) = features
@@ -690,7 +728,12 @@ pub fn eval_condition(
                             seg.ident.name = Symbol::intern(&format!("target_{}", seg.ident.name));
                         }
 
-                        res & eval_condition(&mi, sess, features, eval)
+                        res & eval_condition(
+                            &ast::MetaItemInner::MetaItem(mi),
+                            sess,
+                            features,
+                            eval,
+                        )
                     })
                 }
                 _ => {
@@ -830,7 +873,7 @@ pub fn find_deprecation(
 
                 for meta in list {
                     match meta {
-                        NestedMetaItem::MetaItem(mi) => match mi.name_or_empty() {
+                        MetaItemInner::MetaItem(mi) => match mi.name_or_empty() {
                             sym::since => {
                                 if !get(mi, &mut since) {
                                     continue 'outer;
@@ -869,7 +912,7 @@ pub fn find_deprecation(
                                 continue 'outer;
                             }
                         },
-                        NestedMetaItem::Lit(lit) => {
+                        MetaItemInner::Lit(lit) => {
                             sess.dcx().emit_err(session_diagnostics::UnsupportedLiteral {
                                 span: lit.span,
                                 reason: UnsupportedLiteralReason::DeprecatedKvPair,
@@ -1234,7 +1277,7 @@ pub fn parse_confusables(attr: &Attribute) -> Option<Vec<Symbol>> {
     let mut candidates = Vec::new();
 
     for meta in metas {
-        let NestedMetaItem::Lit(meta_lit) = meta else {
+        let MetaItemInner::Lit(meta_lit) = meta else {
             return None;
         };
         candidates.push(meta_lit.symbol);

compiler/rustc_attr/src/session_diagnostics.rs

@@ -206,6 +206,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for UnsupportedLiteral {
         let mut diag = Diag::new(dcx, level, match self.reason {
             UnsupportedLiteralReason::Generic => fluent::attr_unsupported_literal_generic,
             UnsupportedLiteralReason::CfgString => fluent::attr_unsupported_literal_cfg_string,
+            UnsupportedLiteralReason::CfgBoolean => fluent::attr_unsupported_literal_cfg_boolean,
             UnsupportedLiteralReason::DeprecatedString => {
                 fluent::attr_unsupported_literal_deprecated_string
             }

compiler/rustc_borrowck/src/consumers.rs

@@ -1,7 +1,5 @@
 //! This file provides API for compiler consumers.
 
-use std::rc::Rc;
-
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::{Body, Promoted};
@@ -65,10 +63,10 @@ pub struct BodyWithBorrowckFacts<'tcx> {
     /// The mir bodies of promoteds.
     pub promoted: IndexVec<Promoted, Body<'tcx>>,
     /// The set of borrows occurring in `body` with data about them.
-    pub borrow_set: Rc<BorrowSet<'tcx>>,
+    pub borrow_set: BorrowSet<'tcx>,
     /// Context generated during borrowck, intended to be passed to
     /// [`calculate_borrows_out_of_scope_at_location`].
-    pub region_inference_context: Rc<RegionInferenceContext<'tcx>>,
+    pub region_inference_context: RegionInferenceContext<'tcx>,
     /// The table that maps Polonius points to locations in the table.
     /// Populated when using [`ConsumerOptions::PoloniusInputFacts`]
     /// or [`ConsumerOptions::PoloniusOutputFacts`].
@@ -79,7 +77,7 @@ pub struct BodyWithBorrowckFacts<'tcx> {
     pub input_facts: Option<Box<PoloniusInput>>,
     /// Polonius output facts. Populated when using
     /// [`ConsumerOptions::PoloniusOutputFacts`].
-    pub output_facts: Option<Rc<PoloniusOutput>>,
+    pub output_facts: Option<Box<PoloniusOutput>>,
 }
 
 /// This function computes borrowck facts for the given body. The [`ConsumerOptions`]

compiler/rustc_borrowck/src/dataflow.rs

@@ -1,5 +1,3 @@
-use std::fmt;
-
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::graph;
 use rustc_index::bit_set::BitSet;
@@ -425,10 +423,6 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> {
         Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location }
     }
 
-    pub fn location(&self, idx: BorrowIndex) -> &Location {
-        &self.borrow_set[idx].reserve_location
-    }
-
     /// Add all borrows to the kill set, if those borrows are out of scope at `location`.
     /// That means they went out of a nonlexical scope
     fn kill_loans_out_of_scope_at_location(
@@ -615,8 +609,4 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
     }
 }
 
-impl DebugWithContext<Borrows<'_, '_>> for BorrowIndex {
-    fn fmt_with(&self, ctxt: &Borrows<'_, '_>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", ctxt.location(*self))
-    }
-}
+impl<C> DebugWithContext<C> for BorrowIndex {}

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -708,9 +708,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         // for the branching codepaths that aren't covered, to point at them.
         let map = self.infcx.tcx.hir();
         let body = map.body_owned_by(self.mir_def_id());
-        let mut visitor =
-            ConditionVisitor { tcx: self.infcx.tcx, spans: &spans, name: &name, errors: vec![] };
+        let mut visitor = ConditionVisitor { tcx: self.infcx.tcx, spans, name, errors: vec![] };
         visitor.visit_body(&body);
+        let spans = visitor.spans;
 
         let mut show_assign_sugg = false;
         let isnt_initialized = if let InitializationRequiringAction::PartialAssignment
@@ -4465,20 +4465,20 @@ impl<'hir> Visitor<'hir> for BreakFinder {
 
 /// Given a set of spans representing statements initializing the relevant binding, visit all the
 /// function expressions looking for branching code paths that *do not* initialize the binding.
-struct ConditionVisitor<'b, 'tcx> {
+struct ConditionVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
-    spans: &'b [Span],
-    name: &'b str,
+    spans: Vec<Span>,
+    name: String,
     errors: Vec<(Span, String)>,
 }
 
-impl<'b, 'v, 'tcx> Visitor<'v> for ConditionVisitor<'b, 'tcx> {
+impl<'v, 'tcx> Visitor<'v> for ConditionVisitor<'tcx> {
     fn visit_expr(&mut self, ex: &'v hir::Expr<'v>) {
         match ex.kind {
             hir::ExprKind::If(cond, body, None) => {
                 // `if` expressions with no `else` that initialize the binding might be missing an
                 // `else` arm.
-                if ReferencedStatementsVisitor(self.spans).visit_expr(body).is_break() {
+                if ReferencedStatementsVisitor(&self.spans).visit_expr(body).is_break() {
                     self.errors.push((
                         cond.span,
                         format!(
@@ -4495,8 +4495,8 @@ impl<'b, 'v, 'tcx> Visitor<'v> for ConditionVisitor<'b, 'tcx> {
             hir::ExprKind::If(cond, body, Some(other)) => {
                 // `if` expressions where the binding is only initialized in one of the two arms
                 // might be missing a binding initialization.
-                let a = ReferencedStatementsVisitor(self.spans).visit_expr(body).is_break();
-                let b = ReferencedStatementsVisitor(self.spans).visit_expr(other).is_break();
+                let a = ReferencedStatementsVisitor(&self.spans).visit_expr(body).is_break();
+                let b = ReferencedStatementsVisitor(&self.spans).visit_expr(other).is_break();
                 match (a, b) {
                     (true, true) | (false, false) => {}
                     (true, false) => {
@@ -4536,7 +4536,7 @@ impl<'b, 'v, 'tcx> Visitor<'v> for ConditionVisitor<'b, 'tcx> {
                 // arms might be missing an initialization.
                 let results: Vec<bool> = arms
                     .iter()
-                    .map(|arm| ReferencedStatementsVisitor(self.spans).visit_arm(arm).is_break())
+                    .map(|arm| ReferencedStatementsVisitor(&self.spans).visit_arm(arm).is_break())
                     .collect();
                 if results.iter().any(|x| *x) && !results.iter().all(|x| *x) {
                     for (arm, seen) in arms.iter().zip(results) {

compiler/rustc_borrowck/src/diagnostics/find_use.rs

@@ -1,5 +1,4 @@
 use std::collections::VecDeque;
-use std::rc::Rc;
 
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::mir::visit::{MirVisitable, PlaceContext, Visitor};
@@ -11,7 +10,7 @@ use crate::region_infer::{Cause, RegionInferenceContext};
 
 pub(crate) fn find<'tcx>(
     body: &Body<'tcx>,
-    regioncx: &Rc<RegionInferenceContext<'tcx>>,
+    regioncx: &RegionInferenceContext<'tcx>,
     tcx: TyCtxt<'tcx>,
     region_vid: RegionVid,
     start_point: Location,
@@ -23,7 +22,7 @@ pub(crate) fn find<'tcx>(
 
 struct UseFinder<'a, 'tcx> {
     body: &'a Body<'tcx>,
-    regioncx: &'a Rc<RegionInferenceContext<'tcx>>,
+    regioncx: &'a RegionInferenceContext<'tcx>,
     tcx: TyCtxt<'tcx>,
     region_vid: RegionVid,
     start_point: Location,

compiler/rustc_borrowck/src/diagnostics/region_name.rs

@@ -830,20 +830,14 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
     ///
     /// [`OpaqueDef`]: hir::TyKind::OpaqueDef
     fn get_future_inner_return_ty(&self, hir_ty: &'tcx hir::Ty<'tcx>) -> &'tcx hir::Ty<'tcx> {
-        let hir = self.infcx.tcx.hir();
-
-        let hir::TyKind::OpaqueDef(id, _, _) = hir_ty.kind else {
+        let hir::TyKind::OpaqueDef(opaque_ty, _) = hir_ty.kind else {
             span_bug!(
                 hir_ty.span,
                 "lowered return type of async fn is not OpaqueDef: {:?}",
                 hir_ty
             );
         };
-        let opaque_ty = hir.item(id);
-        if let hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-            bounds: [hir::GenericBound::Trait(trait_ref, _)],
-            ..
-        }) = opaque_ty.kind
+        if let hir::OpaqueTy { bounds: [hir::GenericBound::Trait(trait_ref, _)], .. } = opaque_ty
             && let Some(segment) = trait_ref.trait_ref.path.segments.last()
             && let Some(args) = segment.args
             && let [constraint] = args.constraints

compiler/rustc_borrowck/src/facts.rs

@@ -1,7 +1,7 @@
 use std::error::Error;
 use std::fmt::Debug;
 use std::fs::{self, File};
-use std::io::{BufWriter, Write};
+use std::io::Write;
 use std::path::Path;
 
 use polonius_engine::{AllFacts as PoloniusFacts, Atom};
@@ -127,7 +127,7 @@ impl<'w> FactWriter<'w> {
         T: FactRow,
     {
         let file = &self.dir.join(file_name);
-        let mut file = BufWriter::new(File::create(file)?);
+        let mut file = File::create_buffered(file)?;
         for row in rows {
             row.write(&mut file, self.location_table)?;
         }

compiler/rustc_borrowck/src/lib.rs

@@ -5,7 +5,7 @@
 #![doc(rust_logo)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]
-#![feature(control_flow_enum)]
+#![feature(file_buffered)]
 #![feature(let_chains)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
@@ -19,7 +19,6 @@ use std::cell::RefCell;
 use std::collections::BTreeMap;
 use std::marker::PhantomData;
 use std::ops::Deref;
-use std::rc::Rc;
 
 use consumers::{BodyWithBorrowckFacts, ConsumerOptions};
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
@@ -200,8 +199,7 @@ fn do_mir_borrowck<'tcx>(
         .into_results_cursor(body);
 
     let locals_are_invalidated_at_exit = tcx.hir().body_owner_kind(def).is_fn_or_closure();
-    let borrow_set =
-        Rc::new(BorrowSet::build(tcx, body, locals_are_invalidated_at_exit, &move_data));
+    let borrow_set = BorrowSet::build(tcx, body, locals_are_invalidated_at_exit, &move_data);
 
     // Compute non-lexical lifetimes.
     let nll::NllOutput {
@@ -245,8 +243,6 @@ fn do_mir_borrowck<'tcx>(
     // usage significantly on some benchmarks.
     drop(flow_inits);
 
-    let regioncx = Rc::new(regioncx);
-
     let flow_borrows = Borrows::new(tcx, body, &regioncx, &borrow_set)
         .into_engine(tcx, body)
         .pass_name("borrowck")
@@ -288,10 +284,10 @@ fn do_mir_borrowck<'tcx>(
             access_place_error_reported: Default::default(),
             reservation_error_reported: Default::default(),
             uninitialized_error_reported: Default::default(),
-            regioncx: regioncx.clone(),
+            regioncx: &regioncx,
             used_mut: Default::default(),
             used_mut_upvars: SmallVec::new(),
-            borrow_set: Rc::clone(&borrow_set),
+            borrow_set: &borrow_set,
             upvars: &[],
             local_names: IndexVec::from_elem(None, &promoted_body.local_decls),
             region_names: RefCell::default(),
@@ -329,10 +325,10 @@ fn do_mir_borrowck<'tcx>(
         access_place_error_reported: Default::default(),
         reservation_error_reported: Default::default(),
         uninitialized_error_reported: Default::default(),
-        regioncx: Rc::clone(&regioncx),
+        regioncx: &regioncx,
         used_mut: Default::default(),
         used_mut_upvars: SmallVec::new(),
-        borrow_set: Rc::clone(&borrow_set),
+        borrow_set: &borrow_set,
         upvars: tcx.closure_captures(def),
         local_names,
         region_names: RefCell::default(),
@@ -569,10 +565,10 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
     used_mut_upvars: SmallVec<[FieldIdx; 8]>,
     /// Region inference context. This contains the results from region inference and lets us e.g.
     /// find out which CFG points are contained in each borrow region.
-    regioncx: Rc<RegionInferenceContext<'tcx>>,
+    regioncx: &'a RegionInferenceContext<'tcx>,
 
     /// The set of borrows extracted from the MIR
-    borrow_set: Rc<BorrowSet<'tcx>>,
+    borrow_set: &'a BorrowSet<'tcx>,
 
     /// Information about upvars not necessarily preserved in types or MIR
     upvars: &'tcx [&'tcx ty::CapturedPlace<'tcx>],
@@ -588,7 +584,7 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
     next_region_name: RefCell<usize>,
 
     /// Results of Polonius analysis.
-    polonius_output: Option<Rc<PoloniusOutput>>,
+    polonius_output: Option<Box<PoloniusOutput>>,
 
     diags: diags::BorrowckDiags<'infcx, 'tcx>,
     move_errors: Vec<MoveError<'tcx>>,
@@ -742,6 +738,7 @@ impl<'a, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'a, 'tcx, R>
             }
 
             TerminatorKind::InlineAsm {
+                asm_macro: _,
                 template: _,
                 operands,
                 options: _,
@@ -799,9 +796,8 @@ impl<'a, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'a, 'tcx, R>
             TerminatorKind::Yield { value: _, resume: _, resume_arg: _, drop: _ } => {
                 if self.movable_coroutine {
                     // Look for any active borrows to locals
-                    let borrow_set = self.borrow_set.clone();
                     for i in state.borrows.iter() {
-                        let borrow = &borrow_set[i];
+                        let borrow = &self.borrow_set[i];
                         self.check_for_local_borrow(borrow, span);
                     }
                 }
@@ -815,9 +811,8 @@ impl<'a, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'a, 'tcx, R>
                 // Often, the storage will already have been killed by an explicit
                 // StorageDead, but we don't always emit those (notably on unwind paths),
                 // so this "extra check" serves as a kind of backup.
-                let borrow_set = self.borrow_set.clone();
                 for i in state.borrows.iter() {
-                    let borrow = &borrow_set[i];
+                    let borrow = &self.borrow_set[i];
                     self.check_for_invalidation_at_exit(loc, borrow, span);
                 }
             }
@@ -1036,13 +1031,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
         state: &BorrowckDomain<'a, 'tcx>,
     ) -> bool {
         let mut error_reported = false;
-        let borrow_set = Rc::clone(&self.borrow_set);
 
         // Use polonius output if it has been enabled.
         let mut polonius_output;
         let borrows_in_scope = if let Some(polonius) = &self.polonius_output {
             let location = self.location_table.start_index(location);
-            polonius_output = BitSet::new_empty(borrow_set.len());
+            polonius_output = BitSet::new_empty(self.borrow_set.len());
             for &idx in polonius.errors_at(location) {
                 polonius_output.insert(idx);
             }
@@ -1056,7 +1050,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
             self.infcx.tcx,
             self.body,
             (sd, place_span.0),
-            &borrow_set,
+            self.borrow_set,
             |borrow_index| borrows_in_scope.contains(borrow_index),
             |this, borrow_index, borrow| match (rw, borrow.kind) {
                 // Obviously an activation is compatible with its own
@@ -1579,9 +1573,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
         // Two-phase borrow support: For each activation that is newly
         // generated at this statement, check if it interferes with
         // another borrow.
-        let borrow_set = self.borrow_set.clone();
-        for &borrow_index in borrow_set.activations_at_location(location) {
-            let borrow = &borrow_set[borrow_index];
+        for &borrow_index in self.borrow_set.activations_at_location(location) {
+            let borrow = &self.borrow_set[borrow_index];
 
             // only mutable borrows should be 2-phase
             assert!(match borrow.kind {

compiler/rustc_borrowck/src/nll.rs

@@ -42,7 +42,7 @@ pub(crate) struct NllOutput<'tcx> {
     pub regioncx: RegionInferenceContext<'tcx>,
     pub opaque_type_values: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
     pub polonius_input: Option<Box<AllFacts>>,
-    pub polonius_output: Option<Rc<PoloniusOutput>>,
+    pub polonius_output: Option<Box<PoloniusOutput>>,
     pub opt_closure_req: Option<ClosureRegionRequirements<'tcx>>,
     pub nll_errors: RegionErrors<'tcx>,
 }
@@ -98,7 +98,7 @@ pub(crate) fn compute_regions<'a, 'tcx>(
 
     let universal_regions = Rc::new(universal_regions);
 
-    let elements = &Rc::new(DenseLocationMap::new(body));
+    let elements = Rc::new(DenseLocationMap::new(body));
 
     // Run the MIR type-checker.
     let MirTypeckResults { constraints, universal_region_relations, opaque_type_values } =
@@ -107,13 +107,13 @@ pub(crate) fn compute_regions<'a, 'tcx>(
             param_env,
             body,
             promoted,
-            &universal_regions,
+            universal_regions.clone(),
             location_table,
             borrow_set,
             &mut all_facts,
             flow_inits,
             move_data,
-            elements,
+            elements.clone(),
             upvars,
         );
 
@@ -165,7 +165,7 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         universe_causes,
         type_tests,
         liveness_constraints,
-        elements,
+        elements.clone(),
     );
 
     // If requested: dump NLL facts, and run legacy polonius analysis.
@@ -184,7 +184,7 @@ pub(crate) fn compute_regions<'a, 'tcx>(
             let algorithm = Algorithm::from_str(&algorithm).unwrap();
             debug!("compute_regions: using polonius algorithm {:?}", algorithm);
             let _prof_timer = infcx.tcx.prof.generic_activity("polonius_analysis");
-            Some(Rc::new(Output::compute(all_facts, algorithm, false)))
+            Some(Box::new(Output::compute(all_facts, algorithm, false)))
         } else {
             None
         }

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -169,6 +169,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
                 }
             }
             TerminatorKind::InlineAsm {
+                asm_macro: _,
                 template: _,
                 operands,
                 options: _,

compiler/rustc_borrowck/src/region_infer/mod.rs

@@ -407,7 +407,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
         type_tests: Vec<TypeTest<'tcx>>,
         liveness_constraints: LivenessValues,
-        elements: &Rc<DenseLocationMap>,
+        elements: Rc<DenseLocationMap>,
     ) -> Self {
         debug!("universal_regions: {:#?}", universal_regions);
         debug!("outlives constraints: {:#?}", outlives_constraints);
@@ -430,7 +430,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         }
 
         let mut scc_values =
-            RegionValues::new(elements, universal_regions.len(), &placeholder_indices);
+            RegionValues::new(elements, universal_regions.len(), placeholder_indices);
 
         for region in liveness_constraints.regions() {
             let scc = constraint_sccs.scc(region);
@@ -637,7 +637,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         &mut self,
         infcx: &InferCtxt<'tcx>,
         body: &Body<'tcx>,
-        polonius_output: Option<Rc<PoloniusOutput>>,
+        polonius_output: Option<Box<PoloniusOutput>>,
     ) -> (Option<ClosureRegionRequirements<'tcx>>, RegionErrors<'tcx>) {
         let mir_def_id = body.source.def_id();
         self.propagate_constraints();
@@ -663,7 +663,9 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             self.check_polonius_subset_errors(
                 outlives_requirements.as_mut(),
                 &mut errors_buffer,
-                polonius_output.expect("Polonius output is unavailable despite `-Z polonius`"),
+                polonius_output
+                    .as_ref()
+                    .expect("Polonius output is unavailable despite `-Z polonius`"),
             );
         } else {
             self.check_universal_regions(outlives_requirements.as_mut(), &mut errors_buffer);
@@ -1411,7 +1413,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         &self,
         mut propagated_outlives_requirements: Option<&mut Vec<ClosureOutlivesRequirement<'tcx>>>,
         errors_buffer: &mut RegionErrors<'tcx>,
-        polonius_output: Rc<PoloniusOutput>,
+        polonius_output: &PoloniusOutput,
     ) {
         debug!(
             "check_polonius_subset_errors: {} subset_errors",

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -329,8 +329,8 @@ fn check_opaque_type_well_formed<'tcx>(
 ) -> Result<Ty<'tcx>, ErrorGuaranteed> {
     // Only check this for TAIT. RPIT already supports `tests/ui/impl-trait/nested-return-type2.rs`
     // on stable and we'd break that.
-    let opaque_ty_hir = tcx.hir().expect_item(def_id);
-    let OpaqueTyOrigin::TyAlias { .. } = opaque_ty_hir.expect_opaque_ty().origin else {
+    let opaque_ty_hir = tcx.hir().expect_opaque_ty(def_id);
+    let OpaqueTyOrigin::TyAlias { .. } = opaque_ty_hir.origin else {
         return Ok(definition_ty);
     };
     let param_env = tcx.param_env(def_id);
@@ -503,8 +503,8 @@ impl<'tcx> LazyOpaqueTyEnv<'tcx> {
         let &Self { tcx, def_id, .. } = self;
         let origin = tcx.opaque_type_origin(def_id);
         let parent = match origin {
-            hir::OpaqueTyOrigin::FnReturn(parent)
-            | hir::OpaqueTyOrigin::AsyncFn(parent)
+            hir::OpaqueTyOrigin::FnReturn { parent, .. }
+            | hir::OpaqueTyOrigin::AsyncFn { parent, .. }
             | hir::OpaqueTyOrigin::TyAlias { parent, .. } => parent,
         };
         let param_env = tcx.param_env(parent);

compiler/rustc_borrowck/src/region_infer/values.rs

@@ -275,15 +275,16 @@ impl<N: Idx> RegionValues<N> {
     /// Each of the regions in num_region_variables will be initialized with an
     /// empty set of points and no causal information.
     pub(crate) fn new(
-        elements: &Rc<DenseLocationMap>,
+        elements: Rc<DenseLocationMap>,
         num_universal_regions: usize,
-        placeholder_indices: &Rc<PlaceholderIndices>,
+        placeholder_indices: Rc<PlaceholderIndices>,
     ) -> Self {
+        let num_points = elements.num_points();
         let num_placeholders = placeholder_indices.len();
         Self {
-            elements: elements.clone(),
-            points: SparseIntervalMatrix::new(elements.num_points()),
-            placeholder_indices: placeholder_indices.clone(),
+            elements,
+            points: SparseIntervalMatrix::new(num_points),
+            placeholder_indices,
             free_regions: SparseBitMatrix::new(num_universal_regions),
             placeholders: SparseBitMatrix::new(num_placeholders),
         }

compiler/rustc_borrowck/src/renumber.rs

@@ -11,7 +11,7 @@ use crate::BorrowckInferCtxt;
 /// Replaces all free regions appearing in the MIR with fresh
 /// inference variables, returning the number of variables created.
 #[instrument(skip(infcx, body, promoted), level = "debug")]
-pub fn renumber_mir<'tcx>(
+pub(crate) fn renumber_mir<'tcx>(
     infcx: &BorrowckInferCtxt<'tcx>,
     body: &mut Body<'tcx>,
     promoted: &mut IndexSlice<Promoted, Body<'tcx>>,

compiler/rustc_borrowck/src/type_check/constraint_conversion.rs

@@ -97,7 +97,7 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
     /// that we computed for the closure. This has the effect of adding new outlives obligations
     /// to existing region variables in `closure_args`.
     #[instrument(skip(self), level = "debug")]
-    pub fn apply_closure_requirements(
+    pub(crate) fn apply_closure_requirements(
         &mut self,
         closure_requirements: &ClosureRegionRequirements<'tcx>,
         closure_def_id: DefId,

compiler/rustc_borrowck/src/type_check/free_region_relations.rs

@@ -54,7 +54,7 @@ pub(crate) fn create<'tcx>(
     infcx: &InferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     implicit_region_bound: ty::Region<'tcx>,
-    universal_regions: &Rc<UniversalRegions<'tcx>>,
+    universal_regions: Rc<UniversalRegions<'tcx>>,
     constraints: &mut MirTypeckRegionConstraints<'tcx>,
 ) -> CreateResult<'tcx> {
     UniversalRegionRelationsBuilder {
@@ -62,7 +62,7 @@ pub(crate) fn create<'tcx>(
         param_env,
         implicit_region_bound,
         constraints,
-        universal_regions: universal_regions.clone(),
+        universal_regions,
         region_bound_pairs: Default::default(),
         outlives: Default::default(),
         inverse_outlives: Default::default(),

compiler/rustc_borrowck/src/type_check/liveness/mod.rs

@@ -1,5 +1,3 @@
-use std::rc::Rc;
-
 use itertools::{Either, Itertools};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir::visit::{TyContext, Visitor};
@@ -33,7 +31,7 @@ mod trace;
 pub(super) fn generate<'a, 'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     body: &Body<'tcx>,
-    elements: &Rc<DenseLocationMap>,
+    elements: &DenseLocationMap,
     flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
 ) {

compiler/rustc_borrowck/src/type_check/liveness/trace.rs

@@ -1,5 +1,3 @@
-use std::rc::Rc;
-
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_index::bit_set::BitSet;
 use rustc_index::interval::IntervalSet;
@@ -40,7 +38,7 @@ use crate::type_check::{NormalizeLocation, TypeChecker};
 pub(super) fn trace<'a, 'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     body: &Body<'tcx>,
-    elements: &Rc<DenseLocationMap>,
+    elements: &DenseLocationMap,
     flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     relevant_live_locals: Vec<Local>,

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -121,13 +121,13 @@ pub(crate) fn type_check<'a, 'tcx>(
     param_env: ty::ParamEnv<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexSlice<Promoted, Body<'tcx>>,
-    universal_regions: &Rc<UniversalRegions<'tcx>>,
+    universal_regions: Rc<UniversalRegions<'tcx>>,
     location_table: &LocationTable,
     borrow_set: &BorrowSet<'tcx>,
     all_facts: &mut Option<AllFacts>,
     flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
-    elements: &Rc<DenseLocationMap>,
+    elements: Rc<DenseLocationMap>,
     upvars: &[&ty::CapturedPlace<'tcx>],
 ) -> MirTypeckResults<'tcx> {
     let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body);
@@ -150,14 +150,14 @@ pub(crate) fn type_check<'a, 'tcx>(
         infcx,
         param_env,
         implicit_region_bound,
-        universal_regions,
+        universal_regions.clone(),
         &mut constraints,
     );
 
     debug!(?normalized_inputs_and_output);
 
     let mut borrowck_context = BorrowCheckContext {
-        universal_regions,
+        universal_regions: &universal_regions,
         location_table,
         borrow_set,
         all_facts,
@@ -181,10 +181,10 @@ pub(crate) fn type_check<'a, 'tcx>(
     verifier.visit_body(body);
 
     checker.typeck_mir(body);
-    checker.equate_inputs_and_outputs(body, universal_regions, &normalized_inputs_and_output);
+    checker.equate_inputs_and_outputs(body, &universal_regions, &normalized_inputs_and_output);
     checker.check_signature_annotation(body);
 
-    liveness::generate(&mut checker, body, elements, flow_inits, move_data);
+    liveness::generate(&mut checker, body, &elements, flow_inits, move_data);
 
     translate_outlives_facts(&mut checker);
     let opaque_type_values = infcx.take_opaque_types();
@@ -2437,7 +2437,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 
                                     debug!(?src_tty, ?dst_tty, ?src_obj, ?dst_obj);
 
-                                    self.eq_types(
+                                    self.sub_types(
                                         src_obj,
                                         dst_obj,
                                         location.to_locations(),

compiler/rustc_builtin_macros/messages.ftl

@@ -12,9 +12,9 @@ builtin_macros_asm_duplicate_arg = duplicate argument named `{$name}`
 builtin_macros_asm_expected_comma = expected token: `,`
     .label = expected `,`
 
-builtin_macros_asm_expected_other = expected operand, {$is_global_asm ->
-    [true] options
-    *[false] clobber_abi, options
+builtin_macros_asm_expected_other = expected operand, {$is_inline_asm ->
+    [false] options
+    *[true] clobber_abi, options
     }, or additional template string
 
 builtin_macros_asm_expected_string_literal = expected string literal
@@ -51,6 +51,15 @@ builtin_macros_asm_sym_no_path = expected a path for argument to `sym`
 
 builtin_macros_asm_underscore_input = _ cannot be used for input operands
 
+builtin_macros_asm_unsupported_clobber_abi = `clobber_abi` cannot be used with `{$macro_name}!`
+
+builtin_macros_asm_unsupported_operand = the `{$symbol}` operand cannot be used with `{$macro_name}!`
+    .label = the `{$symbol}` operand is not meaningful for global-scoped inline assembly, remove it
+
+builtin_macros_asm_unsupported_option = the `{$symbol}` option cannot be used with `{$macro_name}!`
+    .label = the `{$symbol}` option is not meaningful for global-scoped inline assembly
+    .suggestion = remove this option
+
 builtin_macros_assert_missing_comma = unexpected string literal
     .suggestion = try adding a comma
 
@@ -194,15 +203,6 @@ builtin_macros_format_unused_args = multiple unused formatting arguments
 
 builtin_macros_format_use_positional = consider using a positional formatting argument instead
 
-builtin_macros_global_asm_clobber_abi = `clobber_abi` cannot be used with `global_asm!`
-
-builtin_macros_global_asm_unsupported_operand = the `{$symbol}` operand cannot be used with `global_asm!`
-    .label = the `{$symbol}` operand is not meaningful for global-scoped inline assembly, remove it
-
-builtin_macros_global_asm_unsupported_option = the `{$symbol}` option cannot be used with `global_asm!`
-    .label = the `{$symbol}` option is not meaningful for global-scoped inline assembly
-    .suggestion = remove this option
-
 builtin_macros_invalid_crate_attribute = invalid crate attribute
 
 builtin_macros_multiple_default_attrs = multiple `#[default]` attributes
@@ -235,6 +235,8 @@ builtin_macros_non_exhaustive_default = default variant must be exhaustive
     .label = declared `#[non_exhaustive]` here
     .help = consider a manual implementation of `Default`
 
+builtin_macros_non_generic_pointee = the `#[pointee]` attribute may only be used on generic parameters
+
 builtin_macros_non_unit_default = the `#[default]` attribute may only be used on unit enum variants
     .help = consider a manual implementation of `Default`
 

compiler/rustc_builtin_macros/src/asm.rs

@@ -37,15 +37,23 @@ pub struct AsmArgs {
 /// - `Ok(true)` if the current token matches the keyword, and was expected
 /// - `Ok(false)` if the current token does not match the keyword
 /// - `Err(_)` if the current token matches the keyword, but was not expected
-fn eat_operand_keyword<'a>(p: &mut Parser<'a>, symbol: Symbol, expect: bool) -> PResult<'a, bool> {
-    if expect {
+fn eat_operand_keyword<'a>(
+    p: &mut Parser<'a>,
+    symbol: Symbol,
+    asm_macro: AsmMacro,
+) -> PResult<'a, bool> {
+    if matches!(asm_macro, AsmMacro::Asm) {
         Ok(p.eat_keyword(symbol))
     } else {
         let span = p.token.span;
         if p.eat_keyword_noexpect(symbol) {
             // in gets printed as `r#in` otherwise
             let symbol = if symbol == kw::In { "in" } else { symbol.as_str() };
-            Err(p.dcx().create_err(errors::GlobalAsmUnsupportedOperand { span, symbol }))
+            Err(p.dcx().create_err(errors::AsmUnsupportedOperand {
+                span,
+                symbol,
+                macro_name: asm_macro.macro_name(),
+            }))
         } else {
             Ok(false)
         }
@@ -56,10 +64,10 @@ fn parse_args<'a>(
     ecx: &ExtCtxt<'a>,
     sp: Span,
     tts: TokenStream,
-    is_global_asm: bool,
+    asm_macro: AsmMacro,
 ) -> PResult<'a, AsmArgs> {
     let mut p = ecx.new_parser_from_tts(tts);
-    parse_asm_args(&mut p, sp, is_global_asm)
+    parse_asm_args(&mut p, sp, asm_macro)
 }
 
 // Primarily public for rustfmt consumption.
@@ -67,7 +75,7 @@ fn parse_args<'a>(
 pub fn parse_asm_args<'a>(
     p: &mut Parser<'a>,
     sp: Span,
-    is_global_asm: bool,
+    asm_macro: AsmMacro,
 ) -> PResult<'a, AsmArgs> {
     let dcx = p.dcx();
 
@@ -110,7 +118,7 @@ pub fn parse_asm_args<'a>(
 
         // Parse options
         if p.eat_keyword(sym::options) {
-            parse_options(p, &mut args, is_global_asm)?;
+            parse_options(p, &mut args, asm_macro)?;
             allow_templates = false;
             continue;
         }
@@ -129,7 +137,7 @@ pub fn parse_asm_args<'a>(
         };
 
         let mut explicit_reg = false;
-        let op = if eat_operand_keyword(p, kw::In, !is_global_asm)? {
+        let op = if eat_operand_keyword(p, kw::In, asm_macro)? {
             let reg = parse_reg(p, &mut explicit_reg)?;
             if p.eat_keyword(kw::Underscore) {
                 let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
@@ -137,15 +145,15 @@ pub fn parse_asm_args<'a>(
             }
             let expr = p.parse_expr()?;
             ast::InlineAsmOperand::In { reg, expr }
-        } else if eat_operand_keyword(p, sym::out, !is_global_asm)? {
+        } else if eat_operand_keyword(p, sym::out, asm_macro)? {
             let reg = parse_reg(p, &mut explicit_reg)?;
             let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) };
             ast::InlineAsmOperand::Out { reg, expr, late: false }
-        } else if eat_operand_keyword(p, sym::lateout, !is_global_asm)? {
+        } else if eat_operand_keyword(p, sym::lateout, asm_macro)? {
             let reg = parse_reg(p, &mut explicit_reg)?;
             let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) };
             ast::InlineAsmOperand::Out { reg, expr, late: true }
-        } else if eat_operand_keyword(p, sym::inout, !is_global_asm)? {
+        } else if eat_operand_keyword(p, sym::inout, asm_macro)? {
             let reg = parse_reg(p, &mut explicit_reg)?;
             if p.eat_keyword(kw::Underscore) {
                 let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
@@ -159,7 +167,7 @@ pub fn parse_asm_args<'a>(
             } else {
                 ast::InlineAsmOperand::InOut { reg, expr, late: false }
             }
-        } else if eat_operand_keyword(p, sym::inlateout, !is_global_asm)? {
+        } else if eat_operand_keyword(p, sym::inlateout, asm_macro)? {
             let reg = parse_reg(p, &mut explicit_reg)?;
             if p.eat_keyword(kw::Underscore) {
                 let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
@@ -173,7 +181,7 @@ pub fn parse_asm_args<'a>(
             } else {
                 ast::InlineAsmOperand::InOut { reg, expr, late: true }
             }
-        } else if eat_operand_keyword(p, sym::label, !is_global_asm)? {
+        } else if eat_operand_keyword(p, sym::label, asm_macro)? {
             let block = p.parse_block()?;
             ast::InlineAsmOperand::Label { block }
         } else if p.eat_keyword(kw::Const) {
@@ -205,7 +213,7 @@ pub fn parse_asm_args<'a>(
                 _ => {
                     let err = dcx.create_err(errors::AsmExpectedOther {
                         span: template.span,
-                        is_global_asm,
+                        is_inline_asm: matches!(asm_macro, AsmMacro::Asm),
                     });
                     return Err(err);
                 }
@@ -301,20 +309,25 @@ pub fn parse_asm_args<'a>(
         dcx.emit_err(errors::AsmMayUnwind { labels_sp });
     }
 
-    if args.clobber_abis.len() > 0 {
-        if is_global_asm {
-            let err = dcx.create_err(errors::GlobalAsmClobberAbi {
-                spans: args.clobber_abis.iter().map(|(_, span)| *span).collect(),
-            });
+    if !args.clobber_abis.is_empty() {
+        match asm_macro {
+            AsmMacro::GlobalAsm | AsmMacro::NakedAsm => {
+                let err = dcx.create_err(errors::AsmUnsupportedClobberAbi {
+                    spans: args.clobber_abis.iter().map(|(_, span)| *span).collect(),
+                    macro_name: asm_macro.macro_name(),
+                });
 
-            // Bail out now since this is likely to confuse later stages
-            return Err(err);
-        }
-        if !regclass_outputs.is_empty() {
-            dcx.emit_err(errors::AsmClobberNoReg {
-                spans: regclass_outputs,
-                clobbers: args.clobber_abis.iter().map(|(_, span)| *span).collect(),
-            });
+                // Bail out now since this is likely to confuse later stages
+                return Err(err);
+            }
+            AsmMacro::Asm => {
+                if !regclass_outputs.is_empty() {
+                    dcx.emit_err(errors::AsmClobberNoReg {
+                        spans: regclass_outputs,
+                        clobbers: args.clobber_abis.iter().map(|(_, span)| *span).collect(),
+                    });
+                }
+            }
         }
     }
 
@@ -335,10 +348,15 @@ fn err_duplicate_option(p: &Parser<'_>, symbol: Symbol, span: Span) {
 ///
 /// This function must be called immediately after the option token is parsed.
 /// Otherwise, the suggestion will be incorrect.
-fn err_unsupported_option(p: &Parser<'_>, symbol: Symbol, span: Span) {
+fn err_unsupported_option(p: &Parser<'_>, asm_macro: AsmMacro, symbol: Symbol, span: Span) {
     // Tool-only output
     let full_span = if p.token == token::Comma { span.to(p.token.span) } else { span };
-    p.dcx().emit_err(errors::GlobalAsmUnsupportedOption { span, symbol, full_span });
+    p.dcx().emit_err(errors::AsmUnsupportedOption {
+        span,
+        symbol,
+        full_span,
+        macro_name: asm_macro.macro_name(),
+    });
 }
 
 /// Try to set the provided option in the provided `AsmArgs`.
@@ -349,12 +367,12 @@ fn err_unsupported_option(p: &Parser<'_>, symbol: Symbol, span: Span) {
 fn try_set_option<'a>(
     p: &Parser<'a>,
     args: &mut AsmArgs,
-    is_global_asm: bool,
+    asm_macro: AsmMacro,
     symbol: Symbol,
     option: ast::InlineAsmOptions,
 ) {
-    if is_global_asm && !ast::InlineAsmOptions::GLOBAL_OPTIONS.contains(option) {
-        err_unsupported_option(p, symbol, p.prev_token.span);
+    if !asm_macro.is_supported_option(option) {
+        err_unsupported_option(p, asm_macro, symbol, p.prev_token.span);
     } else if args.options.contains(option) {
         err_duplicate_option(p, symbol, p.prev_token.span);
     } else {
@@ -365,7 +383,7 @@ fn try_set_option<'a>(
 fn parse_options<'a>(
     p: &mut Parser<'a>,
     args: &mut AsmArgs,
-    is_global_asm: bool,
+    asm_macro: AsmMacro,
 ) -> PResult<'a, ()> {
     let span_start = p.prev_token.span;
 
@@ -386,15 +404,14 @@ fn parse_options<'a>(
 
         'blk: {
             for (symbol, option) in OPTIONS {
-                let kw_matched =
-                    if !is_global_asm || ast::InlineAsmOptions::GLOBAL_OPTIONS.contains(option) {
-                        p.eat_keyword(symbol)
-                    } else {
-                        p.eat_keyword_noexpect(symbol)
-                    };
+                let kw_matched = if asm_macro.is_supported_option(option) {
+                    p.eat_keyword(symbol)
+                } else {
+                    p.eat_keyword_noexpect(symbol)
+                };
 
                 if kw_matched {
-                    try_set_option(p, args, is_global_asm, symbol, option);
+                    try_set_option(p, args, asm_macro, symbol, option);
                     break 'blk;
                 }
             }
@@ -483,7 +500,7 @@ fn parse_reg<'a>(
 
 fn expand_preparsed_asm(
     ecx: &mut ExtCtxt<'_>,
-    asm_macro: ast::AsmMacro,
+    asm_macro: AsmMacro,
     args: AsmArgs,
 ) -> ExpandResult<Result<ast::InlineAsm, ErrorGuaranteed>, ()> {
     let mut template = vec![];
@@ -507,6 +524,7 @@ fn expand_preparsed_asm(
 
         let msg = "asm template must be a string literal";
         let template_sp = template_expr.span;
+        let template_is_mac_call = matches!(template_expr.kind, ast::ExprKind::MacCall(_));
         let (template_str, template_style, template_span) = {
             let ExpandResult::Ready(mac) = expr_to_spanned_string(ecx, template_expr, msg) else {
                 return ExpandResult::Retry(());
@@ -596,7 +614,14 @@ fn expand_preparsed_asm(
 
         if !parser.errors.is_empty() {
             let err = parser.errors.remove(0);
-            let err_sp = template_span.from_inner(InnerSpan::new(err.span.start, err.span.end));
+            let err_sp = if template_is_mac_call {
+                // If the template is a macro call we can't reliably point to the error's
+                // span so just use the template's span as the error span (fixes #129503)
+                template_span
+            } else {
+                template_span.from_inner(InnerSpan::new(err.span.start, err.span.end))
+            };
+
             let msg = format!("invalid asm template string: {}", err.description);
             let mut e = ecx.dcx().struct_span_err(err_sp, msg);
             e.span_label(err_sp, err.label + " in asm template string");
@@ -789,7 +814,7 @@ pub(super) fn expand_asm<'cx>(
     sp: Span,
     tts: TokenStream,
 ) -> MacroExpanderResult<'cx> {
-    ExpandResult::Ready(match parse_args(ecx, sp, tts, false) {
+    ExpandResult::Ready(match parse_args(ecx, sp, tts, AsmMacro::Asm) {
         Ok(args) => {
             let ExpandResult::Ready(mac) = expand_preparsed_asm(ecx, AsmMacro::Asm, args) else {
                 return ExpandResult::Retry(());
@@ -818,29 +843,20 @@ pub(super) fn expand_naked_asm<'cx>(
     sp: Span,
     tts: TokenStream,
 ) -> MacroExpanderResult<'cx> {
-    ExpandResult::Ready(match parse_args(ecx, sp, tts, false) {
+    ExpandResult::Ready(match parse_args(ecx, sp, tts, AsmMacro::NakedAsm) {
         Ok(args) => {
             let ExpandResult::Ready(mac) = expand_preparsed_asm(ecx, AsmMacro::NakedAsm, args)
             else {
                 return ExpandResult::Retry(());
             };
             let expr = match mac {
-                Ok(mut inline_asm) => {
-                    // for future compatibility, we always set the NORETURN option.
-                    //
-                    // When we turn `asm!` into `naked_asm!` with this implementation, we can drop
-                    // the `options(noreturn)`, which makes the upgrade smooth when `naked_asm!`
-                    // starts disallowing the `noreturn` option in the future
-                    inline_asm.options |= ast::InlineAsmOptions::NORETURN;
-
-                    P(ast::Expr {
-                        id: ast::DUMMY_NODE_ID,
-                        kind: ast::ExprKind::InlineAsm(P(inline_asm)),
-                        span: sp,
-                        attrs: ast::AttrVec::new(),
-                        tokens: None,
-                    })
-                }
+                Ok(inline_asm) => P(ast::Expr {
+                    id: ast::DUMMY_NODE_ID,
+                    kind: ast::ExprKind::InlineAsm(P(inline_asm)),
+                    span: sp,
+                    attrs: ast::AttrVec::new(),
+                    tokens: None,
+                }),
                 Err(guar) => DummyResult::raw_expr(sp, Some(guar)),
             };
             MacEager::expr(expr)
@@ -857,7 +873,7 @@ pub(super) fn expand_global_asm<'cx>(
     sp: Span,
     tts: TokenStream,
 ) -> MacroExpanderResult<'cx> {
-    ExpandResult::Ready(match parse_args(ecx, sp, tts, true) {
+    ExpandResult::Ready(match parse_args(ecx, sp, tts, AsmMacro::GlobalAsm) {
         Ok(args) => {
             let ExpandResult::Ready(mac) = expand_preparsed_asm(ecx, AsmMacro::GlobalAsm, args)
             else {

compiler/rustc_builtin_macros/src/cfg.rs

@@ -6,7 +6,6 @@ use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::PResult;
 use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacEager, MacroExpanderResult};
-use rustc_parse::parser::attr::AllowLeadingUnsafe;
 use rustc_span::Span;
 use {rustc_ast as ast, rustc_attr as attr};
 
@@ -36,14 +35,18 @@ pub(crate) fn expand_cfg(
     })
 }
 
-fn parse_cfg<'a>(cx: &ExtCtxt<'a>, span: Span, tts: TokenStream) -> PResult<'a, ast::MetaItem> {
+fn parse_cfg<'a>(
+    cx: &ExtCtxt<'a>,
+    span: Span,
+    tts: TokenStream,
+) -> PResult<'a, ast::MetaItemInner> {
     let mut p = cx.new_parser_from_tts(tts);
 
     if p.token == token::Eof {
         return Err(cx.dcx().create_err(errors::RequiresCfgPattern { span }));
     }
 
-    let cfg = p.parse_meta_item(AllowLeadingUnsafe::Yes)?;
+    let cfg = p.parse_meta_item_inner()?;
 
     let _ = p.eat(&token::Comma);
 

compiler/rustc_builtin_macros/src/derive.rs

@@ -1,5 +1,5 @@
 use rustc_ast as ast;
-use rustc_ast::{GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
+use rustc_ast::{GenericParamKind, ItemKind, MetaItemInner, MetaItemKind, StmtKind};
 use rustc_expand::base::{
     Annotatable, DeriveResolution, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier,
 };
@@ -49,9 +49,9 @@ impl MultiItemModifier for Expander {
                 let mut resolutions = match &meta_item.kind {
                     MetaItemKind::List(list) => {
                         list.iter()
-                            .filter_map(|nested_meta| match nested_meta {
-                                NestedMetaItem::MetaItem(meta) => Some(meta),
-                                NestedMetaItem::Lit(lit) => {
+                            .filter_map(|meta_item_inner| match meta_item_inner {
+                                MetaItemInner::MetaItem(meta) => Some(meta),
+                                MetaItemInner::Lit(lit) => {
                                     // Reject `#[derive("Debug")]`.
                                     report_unexpected_meta_item_lit(sess, lit);
                                     None

compiler/rustc_builtin_macros/src/deriving/smart_ptr.rs

@@ -13,6 +13,8 @@ use rustc_span::symbol::{Ident, sym};
 use rustc_span::{Span, Symbol};
 use thin_vec::{ThinVec, thin_vec};
 
+use crate::errors;
+
 macro_rules! path {
     ($span:expr, $($part:ident)::*) => { vec![$(Ident::new(sym::$part, $span),)*] }
 }
@@ -25,6 +27,8 @@ pub(crate) fn expand_deriving_smart_ptr(
     push: &mut dyn FnMut(Annotatable),
     _is_const: bool,
 ) {
+    item.visit_with(&mut DetectNonGenericPointeeAttr { cx });
+
     let (name_ident, generics) = if let Annotatable::Item(aitem) = item
         && let ItemKind::Struct(struct_data, g) = &aitem.kind
     {
@@ -310,7 +314,7 @@ pub(crate) fn expand_deriving_smart_ptr(
     // Add the impl blocks for `DispatchFromDyn` and `CoerceUnsized`.
     let gen_args = vec![GenericArg::Type(alt_self_type.clone())];
     add_impl_block(impl_generics.clone(), sym::DispatchFromDyn, gen_args.clone());
-    add_impl_block(impl_generics.clone(), sym::CoerceUnsized, gen_args.clone());
+    add_impl_block(impl_generics.clone(), sym::CoerceUnsized, gen_args);
 }
 
 fn contains_maybe_sized_bound_on_pointee(predicates: &[WherePredicate], pointee: Symbol) -> bool {
@@ -396,3 +400,63 @@ impl<'a> ast::mut_visit::MutVisitor for TypeSubstitution<'a> {
         }
     }
 }
+
+struct DetectNonGenericPointeeAttr<'a, 'b> {
+    cx: &'a ExtCtxt<'b>,
+}
+
+impl<'a, 'b> rustc_ast::visit::Visitor<'a> for DetectNonGenericPointeeAttr<'a, 'b> {
+    fn visit_attribute(&mut self, attr: &'a rustc_ast::Attribute) -> Self::Result {
+        if attr.has_name(sym::pointee) {
+            self.cx.dcx().emit_err(errors::NonGenericPointee { span: attr.span });
+        }
+    }
+
+    fn visit_generic_param(&mut self, param: &'a rustc_ast::GenericParam) -> Self::Result {
+        let mut error_on_pointee = AlwaysErrorOnGenericParam { cx: self.cx };
+
+        match &param.kind {
+            GenericParamKind::Type { default } => {
+                // The `default` may end up containing a block expression.
+                // The problem is block expressions  may define structs with generics.
+                // A user may attach a #[pointee] attribute to one of these generics
+                // We want to catch that. The simple solution is to just
+                // always raise a `NonGenericPointee` error when this happens.
+                //
+                // This solution does reject valid rust programs but,
+                // such a code would have to, in order:
+                // - Define a smart pointer struct.
+                // - Somewhere in this struct definition use a type with a const generic argument.
+                // - Calculate this const generic in a expression block.
+                // - Define a new smart pointer type in this block.
+                // - Have this smart pointer type have more than 1 generic type.
+                // In this case, the inner smart pointer derive would be complaining that it
+                // needs a pointer attribute. Meanwhile, the outer macro would be complaining
+                // that we attached a #[pointee] to a generic type argument while helpfully
+                // informing the user that #[pointee] can only be attached to generic pointer arguments
+                rustc_ast::visit::visit_opt!(error_on_pointee, visit_ty, default);
+            }
+
+            GenericParamKind::Const { .. } | GenericParamKind::Lifetime => {
+                rustc_ast::visit::walk_generic_param(&mut error_on_pointee, param);
+            }
+        }
+    }
+
+    fn visit_ty(&mut self, t: &'a rustc_ast::Ty) -> Self::Result {
+        let mut error_on_pointee = AlwaysErrorOnGenericParam { cx: self.cx };
+        error_on_pointee.visit_ty(t)
+    }
+}
+
+struct AlwaysErrorOnGenericParam<'a, 'b> {
+    cx: &'a ExtCtxt<'b>,
+}
+
+impl<'a, 'b> rustc_ast::visit::Visitor<'a> for AlwaysErrorOnGenericParam<'a, 'b> {
+    fn visit_attribute(&mut self, attr: &'a rustc_ast::Attribute) -> Self::Result {
+        if attr.has_name(sym::pointee) {
+            self.cx.dcx().emit_err(errors::NonGenericPointee { span: attr.span });
+        }
+    }
+}

compiler/rustc_builtin_macros/src/errors.rs

@@ -751,7 +751,7 @@ pub(crate) struct AsmExpectedOther {
     #[primary_span]
     #[label(builtin_macros_asm_expected_other)]
     pub(crate) span: Span,
-    pub(crate) is_global_asm: bool,
+    pub(crate) is_inline_asm: bool,
 }
 
 #[derive(Diagnostic)]
@@ -799,13 +799,6 @@ pub(crate) struct AsmMayUnwind {
     pub(crate) labels_sp: Vec<Span>,
 }
 
-#[derive(Diagnostic)]
-#[diag(builtin_macros_global_asm_clobber_abi)]
-pub(crate) struct GlobalAsmClobberAbi {
-    #[primary_span]
-    pub(crate) spans: Vec<Span>,
-}
-
 pub(crate) struct AsmClobberNoReg {
     pub(crate) spans: Vec<Span>,
     pub(crate) clobbers: Vec<Span>,
@@ -841,23 +834,33 @@ pub(crate) struct AsmOptAlreadyprovided {
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_global_asm_unsupported_option)]
-pub(crate) struct GlobalAsmUnsupportedOption {
+#[diag(builtin_macros_asm_unsupported_option)]
+pub(crate) struct AsmUnsupportedOption {
     #[primary_span]
     #[label]
     pub(crate) span: Span,
     pub(crate) symbol: Symbol,
     #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")]
     pub(crate) full_span: Span,
+    pub(crate) macro_name: &'static str,
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_global_asm_unsupported_operand)]
-pub(crate) struct GlobalAsmUnsupportedOperand<'a> {
+#[diag(builtin_macros_asm_unsupported_operand)]
+pub(crate) struct AsmUnsupportedOperand<'a> {
     #[primary_span]
     #[label]
     pub(crate) span: Span,
     pub(crate) symbol: &'a str,
+    pub(crate) macro_name: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_asm_unsupported_clobber_abi)]
+pub(crate) struct AsmUnsupportedClobberAbi {
+    #[primary_span]
+    pub(crate) spans: Vec<Span>,
+    pub(crate) macro_name: &'static str,
 }
 
 #[derive(Diagnostic)]
@@ -937,3 +940,10 @@ pub(crate) struct NakedFunctionTestingAttribute {
     #[label]
     pub testing_span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_non_generic_pointee)]
+pub(crate) struct NonGenericPointee {
+    #[primary_span]
+    pub span: Span,
+}

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -726,6 +726,12 @@ pub macro global_asm() {
     /* compiler built-in */
 }
 
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro naked_asm() {
+    /* compiler built-in */
+}
+
 pub static A_STATIC: u8 = 42;
 
 #[lang = "panic_location"]

compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs

@@ -390,7 +390,7 @@ global_asm! {
 #[naked]
 extern "C" fn naked_test() {
     unsafe {
-        asm!("ret", options(noreturn));
+        naked_asm!("ret");
     }
 }
 

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -168,7 +168,7 @@ fn main() {
 
     foo(I64X2([0, 0]));
 
-    transmute_fat_pointer();
+    transmute_wide_pointer();
 
     rust_call_abi();
 
@@ -192,7 +192,7 @@ type TwoPtrs = i64;
 #[cfg(target_pointer_width = "64")]
 type TwoPtrs = i128;
 
-fn transmute_fat_pointer() -> TwoPtrs {
+fn transmute_wide_pointer() -> TwoPtrs {
     unsafe { transmute::<_, TwoPtrs>("true !") }
 }
 

compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch

@@ -82,19 +82,6 @@ index d9de37e..8293fce 100644
  #[cfg(target_has_atomic_load_store = "ptr")]
  macro_rules! atomic_int_ptr_sized {
      ( $($target_pointer_width:literal $align:literal)* ) => { $(
-diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs
-index 58b9ba4..91bbd0a 100644
---- a/library/core/src/cell.rs
-+++ b/library/core/src/cell.rs
-@@ -2246,8 +2246,6 @@ unsafe_cell_primitive_into_inner! {
-     u32 "32"
-     i64 "64"
-     u64 "64"
--    i128 "128"
--    u128 "128"
-     isize "ptr"
-     usize "ptr"
- }
 --
 2.26.2.7.g19db9cfb68
 

compiler/rustc_codegen_cranelift/src/base.rs

@@ -8,6 +8,7 @@ use rustc_ast::InlineAsmOptions;
 use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization;
 use rustc_index::IndexVec;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::mir::InlineAsmMacro;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::FnAbiOf;
@@ -57,6 +58,7 @@ pub(crate) fn codegen_fn<'tcx>(
 
         match &mir.basic_blocks[START_BLOCK].terminator().kind {
             TerminatorKind::InlineAsm {
+                asm_macro: InlineAsmMacro::NakedAsm,
                 template,
                 operands,
                 options,
@@ -498,6 +500,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 "tail calls are not yet supported in `rustc_codegen_cranelift` backend"
             ),
             TerminatorKind::InlineAsm {
+                asm_macro: _,
                 template,
                 operands,
                 options,
@@ -713,17 +716,17 @@ fn codegen_stmt<'tcx>(
                     let from_ty = operand.layout().ty;
                     let to_ty = fx.monomorphize(to_ty);
 
-                    fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
+                    fn is_wide_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
                         ty.builtin_deref(true)
                             .is_some_and(|pointee_ty| has_ptr_meta(fx.tcx, pointee_ty))
                     }
 
-                    if is_fat_ptr(fx, from_ty) {
-                        if is_fat_ptr(fx, to_ty) {
-                            // fat-ptr -> fat-ptr
+                    if is_wide_ptr(fx, from_ty) {
+                        if is_wide_ptr(fx, to_ty) {
+                            // wide-ptr -> wide-ptr
                             lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout));
                         } else {
-                            // fat-ptr -> thin-ptr
+                            // wide-ptr -> thin-ptr
                             let (ptr, _extra) = operand.load_scalar_pair(fx);
                             lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout))
                         }

compiler/rustc_codegen_cranelift/src/common.rs

@@ -101,7 +101,7 @@ fn clif_pair_type_from_ty<'tcx>(
     })
 }
 
-/// Is a pointer to this type a fat ptr?
+/// Is a pointer to this type a wide ptr?
 pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
     if ty.is_sized(tcx, ParamEnv::reveal_all()) {
         return false;

compiler/rustc_codegen_cranelift/src/debuginfo/types.rs

@@ -139,7 +139,7 @@ impl DebugContext {
 
             pointer_type_id
         } else {
-            // FIXME implement debuginfo for fat pointers
+            // FIXME implement debuginfo for wide pointers
             self.placeholder_for_type(tcx, type_dbg, ptr_type)
         }
     }

compiler/rustc_codegen_cranelift/src/unsize.rs

@@ -2,6 +2,7 @@
 //!
 //! [`PointerCoercion::Unsize`]: `rustc_middle::ty::adjustment::PointerCoercion::Unsize`
 
+use rustc_codegen_ssa::base::validate_trivial_unsize;
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 
 use crate::base::codegen_panic_nounwind;
@@ -34,7 +35,10 @@ pub(crate) fn unsized_info<'tcx>(
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
-                // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
+                debug_assert!(
+                    validate_trivial_unsize(fx.tcx, data_a, data_b),
+                    "NOP unsize vtable changed principal trait ref: {data_a} -> {data_b}"
+                );
                 return old_info;
             }
 

compiler/rustc_codegen_gcc/.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
         ]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # `rustup show` installs from rust-toolchain.toml
     - name: Setup rust toolchain
@@ -113,13 +113,13 @@ jobs:
   duplicates:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: python tools/check_intrinsics_duplicates.py
 
   build_system:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Test build system
         run: |
           cd build_system

compiler/rustc_codegen_gcc/.github/workflows/failures.yml

@@ -31,7 +31,7 @@ jobs:
             env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests' GCC_EXEC_PREFIX=/usr/lib/gcc/"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # `rustup show` installs from rust-toolchain.toml
     - name: Setup rust toolchain

compiler/rustc_codegen_gcc/.github/workflows/gcc12.yml

@@ -33,7 +33,7 @@ jobs:
         ]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # `rustup show` installs from rust-toolchain.toml
     - name: Setup rust toolchain

compiler/rustc_codegen_gcc/.github/workflows/m68k.yml

@@ -36,7 +36,7 @@ jobs:
         ]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # `rustup show` installs from rust-toolchain.toml
     - name: Setup rust toolchain
@@ -85,14 +85,14 @@ jobs:
     - name: Build sample project with target defined as JSON spec
       run: |
         ./y.sh prepare --only-libcore --cross
-        ./y.sh build --sysroot --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
+        ./y.sh build --sysroot --features compiler_builtins/no-f16-f128 --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
         ./y.sh cargo build --manifest-path=./tests/hello-world/Cargo.toml --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
         ./y.sh clean all
 
     - name: Build
       run: |
         ./y.sh prepare --only-libcore --cross
-        ./y.sh build --sysroot --target-triple m68k-unknown-linux-gnu
+        ./y.sh build --sysroot --features compiler_builtins/no-f16-f128 --target-triple m68k-unknown-linux-gnu
         CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu cargo test
         ./y.sh clean all
 
@@ -107,4 +107,4 @@ jobs:
 
     - name: Run tests
       run: |
-        ./y.sh test --release --clean --build-sysroot ${{ matrix.commands }}
+        ./y.sh test --release --clean --build-sysroot --sysroot-features compiler_builtins/no-f16-f128 ${{ matrix.commands }}

compiler/rustc_codegen_gcc/.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
         ]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # `rustup show` installs from rust-toolchain.toml
     - name: Setup rust toolchain

compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml

@@ -13,7 +13,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     strategy:
       fail-fast: false
@@ -24,7 +24,7 @@ jobs:
         ]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # `rustup show` installs from rust-toolchain.toml
     - name: Setup rust toolchain
@@ -36,6 +36,13 @@ jobs:
     - name: Install packages
       run: sudo apt-get install ninja-build ripgrep
 
+    # TODO: remove when we have binutils version 2.43 in the repo.
+    - name: Install more recent binutils
+      run: |
+        echo "deb http://archive.ubuntu.com/ubuntu oracular main universe" | sudo tee /etc/apt/sources.list.d/oracular-copies.list
+        sudo apt-get update
+        sudo apt-get install binutils
+
     - name: Install Intel Software Development Emulator
       if: ${{ matrix.cargo_runner }}
       run: |
@@ -96,4 +103,5 @@ jobs:
       run: |
         # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro.
         # TODO: remove --skip test_mm512_stream_ps when stdarch is updated in rustc.
-        STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_mm512_stream_ps
+        # TODO: remove --skip test_tile_ when it's implemented.
+        STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features --cfg stdarch_intel_sde" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_mm512_stream_ps --skip test_tile_

compiler/rustc_codegen_gcc/Cargo.lock

@@ -28,18 +28,18 @@ dependencies = [
 
 [[package]]
 name = "gccjit"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e0ba949ebee07c5cc21f02cb48f28f2c8db7fcbc15fdc5120476a6c43b4636"
+checksum = "4bb376e98c82d9284c3a17fc1d6bf9bc921055418950238d7a553c27a7e1f6ab"
 dependencies = [
  "gccjit_sys",
 ]
 
 [[package]]
 name = "gccjit_sys"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5bbf85e12c2593772329a9d4e8310271f6706e6045ce4f41b041dd34fba6603"
+checksum = "93b4b1be553b5df790bf25ca2a1d6add81727dc29f8d5c8742468ed306d621d1"
 dependencies = [
  "libc",
 ]

compiler/rustc_codegen_gcc/Cargo.toml

@@ -22,7 +22,9 @@ master = ["gccjit/master"]
 default = ["master"]
 
 [dependencies]
-gccjit = "2.1"
+gccjit = "2.2"
+#gccjit = { git = "https://github.com/rust-lang/gccjit.rs" }
+
 # Local copy.
 #gccjit = { path = "../gccjit.rs" }
 

compiler/rustc_codegen_gcc/build_system/build_sysroot/Cargo.lock

@@ -4,12 +4,12 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
+checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
 dependencies = [
  "compiler_builtins",
- "gimli",
+ "gimli 0.29.0",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
 ]
@@ -52,7 +52,7 @@ dependencies = [
 name = "compiler_builtins"
 version = "0.1.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f11973008a8cf741fe6d22f339eba21fd0ca81e2760a769ba8243ed6c21edd7e"
+checksum = "92afe7344b64cccf3662ca26d5d1c0828ab826f04206b97d856e3625e390e4b5"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -97,9 +97,20 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.28.1"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "gimli"
+version = "0.30.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e1d97fbe9722ba9bbd0c97051c2956e726562b61f86a25a4360398a40edfc9"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -120,9 +131,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.9"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -131,18 +142,18 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 dependencies = [
  "rustc-std-workspace-core",
 ]
 
 [[package]]
 name = "memchr"
-version = "2.7.2"
+version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -150,9 +161,9 @@ dependencies = [
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.2"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
 dependencies = [
  "adler",
  "compiler_builtins",
@@ -162,9 +173,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.32.2"
+version = "0.36.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
+checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"
 dependencies = [
  "compiler_builtins",
  "memchr",
@@ -205,9 +216,9 @@ dependencies = [
 
 [[package]]
 name = "r-efi"
-version = "4.4.0"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c47196f636c4cc0634b73b0405323d177753c2e15e866952c64ea22902567a34"
+checksum = "e9e935efc5854715dfc0a4c9ef18dc69dee0ec3bf9cc3ab740db831c0fdd86a3"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -226,9 +237,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-demangle"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -310,16 +321,14 @@ dependencies = [
  "core",
  "getopts",
  "libc",
- "panic_abort",
- "panic_unwind",
  "std",
 ]
 
 [[package]]
 name = "unicode-width"
-version = "0.1.12"
+version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
+checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -339,12 +348,12 @@ dependencies = [
 
 [[package]]
 name = "unwinding"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37a19a21a537f635c16c7576f22d0f2f7d63353c1337ad4ce0d8001c7952a25b"
+checksum = "dc55842d0db6329a669d55a623c674b02d677b16bfb2d24857d4089d41eba882"
 dependencies = [
  "compiler_builtins",
- "gimli",
+ "gimli 0.30.0",
  "rustc-std-workspace-core",
 ]
 
@@ -370,9 +379,9 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
  "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
@@ -386,48 +395,48 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
 [[package]]
 name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

compiler/rustc_codegen_gcc/build_system/build_sysroot/Cargo.toml

@@ -17,6 +17,22 @@ rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-c
 rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" }
 rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" }
 
+# For compiler-builtins we always use a high number of codegen units.
+# The goal here is to place every single intrinsic into its own object
+# file to avoid symbol clashes with the system libgcc if possible. Note
+# that this number doesn't actually produce this many object files, we
+# just don't create more than this number of object files.
+#
+# It's a bit of a bummer that we have to pass this here, unfortunately.
+# Ideally this would be specified through an env var to Cargo so Cargo
+# knows how many CGUs are for this specific crate, but for now
+# per-crate configuration isn't specifiable in the environment.
+[profile.dev.package.compiler_builtins]
+codegen-units = 10000
+
+[profile.release.package.compiler_builtins]
+codegen-units = 10000
+
 [profile.release]
 debug = "limited"
 #lto = "fat" # TODO(antoyo): re-enable when the failing LTO tests regarding proc-macros are fixed.

compiler/rustc_codegen_gcc/build_system/src/build.rs

@@ -24,16 +24,6 @@ impl BuildArg {
 
         while let Some(arg) = args.next() {
             match arg.as_str() {
-                "--features" => {
-                    if let Some(arg) = args.next() {
-                        build_arg.flags.push("--features".to_string());
-                        build_arg.flags.push(arg.as_str().into());
-                    } else {
-                        return Err(
-                            "Expected a value after `--features`, found nothing".to_string()
-                        );
-                    }
-                }
                 "--sysroot" => {
                     build_arg.build_sysroot = true;
                 }
@@ -56,7 +46,6 @@ impl BuildArg {
             r#"
 `build` command help:
 
-    --features [arg]       : Add a new feature [arg]
     --sysroot              : Build with sysroot"#
         );
         ConfigInfo::show_usage();
@@ -142,14 +131,11 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
         rustflags.push_str(" -Csymbol-mangling-version=v0");
     }
 
-    let mut args: Vec<&dyn AsRef<OsStr>> = vec![
-        &"cargo",
-        &"build",
-        &"--target",
-        &config.target,
-        &"--features",
-        &"compiler-builtins-no-f16-f128",
-    ];
+    let mut args: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"build", &"--target", &config.target];
+    for feature in &config.features {
+        args.push(&"--features");
+        args.push(feature);
+    }
 
     if config.no_default_features {
         rustflags.push_str(" -Csymbol-mangling-version=v0");

compiler/rustc_codegen_gcc/build_system/src/config.rs

@@ -98,7 +98,7 @@ impl ConfigFile {
     }
 }
 
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Clone)]
 pub struct ConfigInfo {
     pub target: String,
     pub target_triple: String,
@@ -123,6 +123,7 @@ pub struct ConfigInfo {
     pub no_download: bool,
     pub no_default_features: bool,
     pub backend: Option<String>,
+    pub features: Vec<String>,
 }
 
 impl ConfigInfo {
@@ -133,6 +134,13 @@ impl ConfigInfo {
         args: &mut impl Iterator<Item = String>,
     ) -> Result<bool, String> {
         match arg {
+            "--features" => {
+                if let Some(arg) = args.next() {
+                    self.features.push(arg);
+                } else {
+                    return Err("Expected a value after `--features`, found nothing".to_string());
+                }
+            }
             "--target" => {
                 if let Some(arg) = args.next() {
                     self.target = arg;
@@ -443,6 +451,7 @@ impl ConfigInfo {
     pub fn show_usage() {
         println!(
             "\
+    --features [arg]       : Add a new feature [arg]
     --target-triple [arg]  : Set the target triple to [arg]
     --target [arg]         : Set the target to [arg]
     --out-dir              : Location where the files will be generated

compiler/rustc_codegen_gcc/build_system/src/test.rs

@@ -92,6 +92,7 @@ struct TestArg {
     current_part: Option<usize>,
     sysroot_panic_abort: bool,
     config_info: ConfigInfo,
+    sysroot_features: Vec<String>,
 }
 
 impl TestArg {
@@ -127,6 +128,14 @@ impl TestArg {
                 "--sysroot-panic-abort" => {
                     test_arg.sysroot_panic_abort = true;
                 }
+                "--sysroot-features" => match args.next() {
+                    Some(feature) if !feature.is_empty() => {
+                        test_arg.sysroot_features.push(feature);
+                    }
+                    _ => {
+                        return Err(format!("Expected an argument after `{}`, found nothing", arg));
+                    }
+                },
                 "--help" => {
                     show_usage();
                     return Ok(None);
@@ -250,7 +259,9 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> {
 fn build_sysroot(env: &Env, args: &TestArg) -> Result<(), String> {
     // FIXME: create a function "display_if_not_quiet" or something along the line.
     println!("[BUILD] sysroot");
-    build::build_sysroot(env, &args.config_info)?;
+    let mut config = args.config_info.clone();
+    config.features.extend(args.sysroot_features.iter().cloned());
+    build::build_sysroot(env, &config)?;
     Ok(())
 }
 
@@ -626,7 +637,8 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
         "https://github.com/BurntSushi/memchr",
         "https://github.com/dtolnay/itoa",
         "https://github.com/rust-lang/cfg-if",
-        "https://github.com/rust-lang-nursery/lazy-static.rs",
+        //"https://github.com/rust-lang-nursery/lazy-static.rs", // TODO: re-enable when the
+        //failing test is fixed upstream.
         //"https://github.com/marshallpierce/rust-base64", // FIXME: one test is OOM-killed.
         // TODO: ignore the base64 test that is OOM-killed.
         "https://github.com/time-rs/time",

compiler/rustc_codegen_gcc/doc/debugging-gcc-lto.md

@@ -1,3 +0,0 @@
-# How to debug GCC LTO
-
-Run do the command with `-v -save-temps` and then extract the `lto1` line from the output and run that under the debugger.

compiler/rustc_codegen_gcc/doc/debugging.md

@@ -0,0 +1,38 @@
+# Debugging
+
+## How to debug GCC LTO
+
+Run do the command with `-v -save-temps` and then extract the `lto1` line from the output and run that under the debugger.
+
+## How to debug stdarch tests that cannot be ran locally
+
+First, run the tests normally:
+
+----
+cd build/build_sysroot/sysroot_src/library/stdarch/
+STDARCH_TEST_EVERYTHING=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="sde -future -rtm_mode full --" TARGET=x86_64-unknown-linux-gnu ../../../../../y.sh cargo test
+----
+
+It will show the command it ran, something like this:
+
+----
+  process didn't exit successfully: `sde -future -rtm_mode full -- /home/user/projects/rustc_codegen_gcc/build/build_sysroot/sysroot_src/library/stdarch/target/debug/deps/core_arch-fd2d75f89baae5c6` (signal: 11, SIGSEGV: invalid memory reference)
+----
+
+Then add the `-debug` flag to it:
+
+----
+sde -debug -future -rtm_mode full -- /home/user/projects/rustc_codegen_gcc/build/build_sysroot/sysroot_src/library/stdarch/target/debug/deps/core_arch-fd2d75f89baae5c6
+----
+
+To see the symbols in `gdb`, specify the executable in your command:
+
+----
+gdb /home/user/projects/rustc_codegen_gcc/build/build_sysroot/sysroot_src/library/stdarch/target/debug/deps/core_arch-fd2d75f89baae5c6
+----
+
+and then write the `gdb` command that `sde` tells you to use, something like:
+
+----
+target remote :51299
+----

compiler/rustc_codegen_gcc/libgccjit.version

@@ -1 +1 @@
-341be3b7d7ac6976cfed8ed59da3573c040d0776
+e744a9459d33864067214741daf5c5bc2a7b88c6

compiler/rustc_codegen_gcc/patches/0022-core-Disable-not-compiling-tests.patch

@@ -1,26 +1,24 @@
-From f6befc4bb51d84f5f1cf35938a168c953d421350 Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Sun, 24 Nov 2019 15:10:23 +0100
+From 18793c6109890493ceb3ff36549849a36e3d8022 Mon Sep 17 00:00:00 2001
+From: None <none@example.com>
+Date: Sun, 1 Sep 2024 11:42:17 -0400
 Subject: [PATCH] [core] Disable not compiling tests
 
 ---
- library/core/tests/Cargo.toml         | 8 ++++++++
- library/core/tests/num/flt2dec/mod.rs | 1 -
- library/core/tests/num/int_macros.rs  | 2 ++
- library/core/tests/num/uint_macros.rs | 2 ++
- library/core/tests/ptr.rs             | 2 ++
- library/core/tests/slice.rs           | 2 ++
- 6 files changed, 16 insertions(+), 1 deletion(-)
+ library/core/tests/Cargo.toml | 14 ++++++++++++++
+ library/core/tests/lib.rs     |  1 +
+ 2 files changed, 15 insertions(+)
  create mode 100644 library/core/tests/Cargo.toml
 
 diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml
 new file mode 100644
-index 0000000..46fd999
+index 0000000..ca326ac
 --- /dev/null
 +++ b/library/core/tests/Cargo.toml
-@@ -0,0 +1,12 @@
+@@ -0,0 +1,14 @@
++[workspace]
++
 +[package]
-+name = "core"
++name = "coretests"
 +version = "0.0.0"
 +edition = "2021"
 +
@@ -32,11 +30,14 @@ index 0000000..46fd999
 +rand = { version = "0.8.5", default-features = false }
 +rand_xorshift = { version = "0.3.0", default-features = false }
 diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
-index 42a26ae..5ac1042 100644
+index 1e336bf..5800ebb 100644
 --- a/library/core/tests/lib.rs
 +++ b/library/core/tests/lib.rs
-@@ -1,3 +1,4 @@
+@@ -1,4 +1,5 @@
+ // tidy-alphabetical-start
 +#![cfg(test)]
- #![feature(alloc_layout_extra)]
- #![feature(array_chunks)]
- #![feature(array_ptr_get)]
+ #![cfg_attr(bootstrap, feature(offset_of_nested))]
+ #![cfg_attr(target_has_atomic = "128", feature(integer_atomics))]
+ #![cfg_attr(test, feature(cfg_match))]
+--
+2.46.0

compiler/rustc_codegen_gcc/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch

@@ -11,15 +11,15 @@ diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
 index b71786c..cf484d5 100644
 --- a/library/core/tests/lib.rs
 +++ b/library/core/tests/lib.rs
-@@ -95,7 +95,6 @@
- #![feature(never_type)]
- #![feature(unwrap_infallible)]
+@@ -87,7 +87,6 @@
+ #![feature(numfmt)]
+ #![feature(pattern)]
  #![feature(pointer_is_aligned_to)]
 -#![feature(portable_simd)]
  #![feature(ptr_metadata)]
- #![feature(unsized_tuple_coercion)]
- #![feature(const_option)]
-@@ -157,7 +156,6 @@ mod pin;
+ #![feature(slice_from_ptr_range)]
+ #![feature(slice_internals)]
+@@ -155,7 +154,6 @@ mod pin;
  mod pin_macro;
  mod ptr;
  mod result;

compiler/rustc_codegen_gcc/rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2024-07-02"
+channel = "nightly-2024-08-11"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

compiler/rustc_codegen_gcc/src/allocator.rs

@@ -104,10 +104,17 @@ fn create_wrapper_function(
         false,
     );
 
-    if tcx.sess.default_hidden_visibility() {
-        #[cfg(feature = "master")]
-        func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
+    #[cfg(feature = "master")]
+    match tcx.sess.default_visibility() {
+        rustc_target::spec::SymbolVisibility::Hidden => {
+            func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden))
+        }
+        rustc_target::spec::SymbolVisibility::Protected => {
+            func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Protected))
+        }
+        rustc_target::spec::SymbolVisibility::Interposable => {}
     }
+
     if tcx.sess.must_emit_unwind_tables() {
         // TODO(antoyo): emit unwind tables.
     }

compiler/rustc_codegen_gcc/src/asm.rs

@@ -682,6 +682,11 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
+            InlineAsmRegClass::S390x(
+                S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg,
+            ) => {
+                unreachable!("clobber-only")
+            }
             InlineAsmRegClass::Err => unreachable!(),
         },
     };
@@ -757,6 +762,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
             S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
         ) => cx.type_i32(),
         InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
+        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
+            unreachable!("clobber-only")
+        }
         InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
         InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),

compiler/rustc_codegen_gcc/src/base.rs

@@ -128,8 +128,19 @@ pub fn compile_codegen_unit(
         // NOTE: Rust relies on LLVM doing wrapping on overflow.
         context.add_command_line_option("-fwrapv");
 
+        if let Some(model) = tcx.sess.code_model() {
+            use rustc_target::spec::CodeModel;
+
+            context.add_command_line_option(match model {
+                CodeModel::Tiny => "-mcmodel=tiny",
+                CodeModel::Small => "-mcmodel=small",
+                CodeModel::Kernel => "-mcmodel=kernel",
+                CodeModel::Medium => "-mcmodel=medium",
+                CodeModel::Large => "-mcmodel=large",
+            });
+        }
+
         if tcx.sess.relocation_model() == rustc_target::spec::RelocModel::Static {
-            context.add_command_line_option("-mcmodel=kernel");
             context.add_command_line_option("-fno-pie");
         }
 

compiler/rustc_codegen_gcc/src/builder.rs

@@ -39,9 +39,6 @@ use crate::type_of::LayoutGccExt;
 // TODO(antoyo)
 type Funclet = ();
 
-// TODO(antoyo): remove this variable.
-static mut RETURN_VALUE_COUNT: usize = 0;
-
 enum ExtremumOperation {
     Max,
     Min,
@@ -50,13 +47,18 @@ enum ExtremumOperation {
 pub struct Builder<'a: 'gcc, 'gcc, 'tcx> {
     pub cx: &'a CodegenCx<'gcc, 'tcx>,
     pub block: Block<'gcc>,
-    stack_var_count: Cell<usize>,
     pub location: Option<Location<'gcc>>,
+    value_counter: Cell<u64>,
 }
 
 impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     fn with_cx(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Self {
-        Builder { cx, block, stack_var_count: Cell::new(0), location: None }
+        Builder { cx, block, location: None, value_counter: Cell::new(0) }
+    }
+
+    fn next_value_counter(&self) -> u64 {
+        self.value_counter.set(self.value_counter.get() + 1);
+        self.value_counter.get()
     }
 
     fn atomic_extremum(
@@ -138,7 +140,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     ) -> RValue<'gcc> {
         let size = get_maybe_pointer_size(src);
         let compare_exchange =
-            self.context.get_builtin_function(&format!("__atomic_compare_exchange_{}", size));
+            self.context.get_builtin_function(format!("__atomic_compare_exchange_{}", size));
         let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
         let failure_order = self.context.new_rvalue_from_int(self.i32_type, failure_order.to_gcc());
         let weak = self.context.new_rvalue_from_int(self.bool_type, weak as i32);
@@ -270,10 +272,12 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                             actual_val.dereference(self.location).to_rvalue()
                         }
                     } else {
+                        // FIXME: this condition seems wrong: it will pass when both types are not
+                        // a vector.
                         assert!(
                             (!expected_ty.is_vector() || actual_ty.is_vector())
                                 && (expected_ty.is_vector() || !actual_ty.is_vector()),
-                            "{:?} ({}) -> {:?} ({}), index: {:?}[{}]",
+                            "{:?} (is vector: {}) -> {:?} (is vector: {}), Function: {:?}[{}]",
                             actual_ty,
                             actual_ty.is_vector(),
                             expected_ty,
@@ -283,6 +287,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                         );
                         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
                         // TODO: remove bitcast now that vector types can be compared?
+                        // ==> We use bitcast to avoid having to do many manual casts from e.g. __m256i to __v32qi (in
+                        // the case of _mm256_aesenc_epi128).
                         self.bitcast(actual_val, expected_ty)
                     }
                 } else {
@@ -325,11 +331,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         let void_type = self.context.new_type::<()>();
         let current_func = self.block.get_function();
         if return_type != void_type {
-            unsafe { RETURN_VALUE_COUNT += 1 };
             let result = current_func.new_local(
                 self.location,
                 return_type,
-                &format!("returnValue{}", unsafe { RETURN_VALUE_COUNT }),
+                format!("returnValue{}", self.next_value_counter()),
             );
             self.block.add_assignment(
                 self.location,
@@ -341,7 +346,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             self.block
                 .add_eval(self.location, self.cx.context.new_call(self.location, func, &args));
             // Return dummy value when not having return value.
-            self.context.new_rvalue_from_long(self.isize_type, 0)
+            self.context.new_rvalue_zero(self.isize_type)
         }
     }
 
@@ -367,6 +372,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         let args = {
             let function_address_names = self.function_address_names.borrow();
             let original_function_name = function_address_names.get(&func_ptr);
+            func_ptr = llvm::adjust_function(self.context, &func_name, func_ptr, args);
             llvm::adjust_intrinsic_arguments(
                 self,
                 gcc_func,
@@ -385,7 +391,6 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         let current_func = self.block.get_function();
 
         if return_type != void_type {
-            unsafe { RETURN_VALUE_COUNT += 1 };
             let return_value = self.cx.context.new_call_through_ptr(self.location, func_ptr, &args);
             let return_value = llvm::adjust_intrinsic_return_value(
                 self,
@@ -398,7 +403,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             let result = current_func.new_local(
                 self.location,
                 return_value.get_type(),
-                &format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }),
+                format!("ptrReturnValue{}", self.next_value_counter()),
             );
             self.block.add_assignment(self.location, result, return_value);
             result.to_rvalue()
@@ -422,17 +427,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 self.cx.context.new_call_through_ptr(self.location, func_ptr, &args),
             );
             // Return dummy value when not having return value.
-            let result = current_func.new_local(
-                self.location,
-                self.isize_type,
-                "dummyValueThatShouldNeverBeUsed",
-            );
-            self.block.add_assignment(
-                self.location,
-                result,
-                self.context.new_rvalue_from_long(self.isize_type, 0),
-            );
-            result.to_rvalue()
+            self.context.new_rvalue_zero(self.isize_type)
         }
     }
 
@@ -447,11 +442,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         let return_type = self.context.new_type::<bool>();
         let current_func = self.block.get_function();
         // TODO(antoyo): return the new_call() directly? Since the overflow function has no side-effects.
-        unsafe { RETURN_VALUE_COUNT += 1 };
         let result = current_func.new_local(
             self.location,
             return_type,
-            &format!("overflowReturnValue{}", unsafe { RETURN_VALUE_COUNT }),
+            format!("overflowReturnValue{}", self.next_value_counter()),
         );
         self.block.add_assignment(
             self.location,
@@ -930,9 +924,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     fn alloca(&mut self, size: Size, align: Align) -> RValue<'gcc> {
         let ty = self.cx.type_array(self.cx.type_i8(), size.bytes()).get_aligned(align.bytes());
         // TODO(antoyo): It might be better to return a LValue, but fixing the rustc API is non-trivial.
-        self.stack_var_count.set(self.stack_var_count.get() + 1);
         self.current_func()
-            .new_local(self.location, ty, &format!("stack_var_{}", self.stack_var_count.get()))
+            .new_local(self.location, ty, format!("stack_var_{}", self.next_value_counter()))
             .get_address(self.location)
     }
 
@@ -955,11 +948,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         };
         let ptr = self.context.new_cast(self.location, ptr, aligned_type.make_pointer());
         let deref = ptr.dereference(self.location).to_rvalue();
-        unsafe { RETURN_VALUE_COUNT += 1 };
         let loaded_value = function.new_local(
             self.location,
             aligned_type,
-            &format!("loadedValue{}", unsafe { RETURN_VALUE_COUNT }),
+            format!("loadedValue{}", self.next_value_counter()),
         );
         block.add_assignment(self.location, loaded_value, deref);
         loaded_value.to_rvalue()
@@ -980,7 +972,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // TODO(antoyo): use ty.
         // TODO(antoyo): handle alignment.
         let atomic_load =
-            self.context.get_builtin_function(&format!("__atomic_load_{}", size.bytes()));
+            self.context.get_builtin_function(format!("__atomic_load_{}", size.bytes()));
         let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
 
         let volatile_const_void_ptr_type =
@@ -1136,7 +1128,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     ) {
         // TODO(antoyo): handle alignment.
         let atomic_store =
-            self.context.get_builtin_function(&format!("__atomic_store_{}", size.bytes()));
+            self.context.get_builtin_function(format!("__atomic_store_{}", size.bytes()));
         let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
         let volatile_const_void_ptr_type =
             self.context.new_type::<()>().make_volatile().make_pointer();

compiler/rustc_codegen_gcc/src/consts.rs

@@ -9,6 +9,7 @@ use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}
 use rustc_middle::mir::interpret::{
     self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint,
 };
+use rustc_middle::mir::mono::Linkage;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Instance};
 use rustc_middle::{bug, span_bug};
@@ -258,7 +259,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
 
             if !self.tcx.is_reachable_non_generic(def_id) {
                 #[cfg(feature = "master")]
-                global.add_string_attribute(VarAttribute::Visibility(Visibility::Hidden));
+                global.add_attribute(VarAttribute::Visibility(Visibility::Hidden));
             }
 
             global
@@ -386,6 +387,11 @@ fn check_and_apply_linkage<'gcc, 'tcx>(
         let global1 =
             cx.declare_global_with_linkage(sym, cx.type_i8(), base::global_linkage_to_gcc(linkage));
 
+        if linkage == Linkage::ExternalWeak {
+            #[cfg(feature = "master")]
+            global1.add_attribute(VarAttribute::Weak);
+        }
+
         // Declare an internal global `extern_with_linkage_foo` which
         // is initialized with the address of `foo`.  If `foo` is
         // discarded during linking (for example, if `foo` has weak

compiler/rustc_codegen_gcc/src/context.rs

@@ -24,6 +24,7 @@ use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCA
 use crate::callee::get_fn;
 use crate::common::SignType;
 
+#[cfg_attr(not(feature = "master"), allow(dead_code))]
 pub struct CodegenCx<'gcc, 'tcx> {
     pub codegen_unit: &'tcx CodegenUnit<'tcx>,
     pub context: &'gcc Context<'gcc>,
@@ -226,48 +227,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             "__builtin_umul_overflow",
             "__builtin_usubll_overflow",
             "__builtin_usub_overflow",
-            "sqrtf",
-            "sqrt",
             "__builtin_powif",
             "__builtin_powi",
-            "sinf",
-            "sin",
-            "cosf",
-            "cos",
-            "powf",
-            "pow",
-            "expf",
-            "exp",
-            "exp2f",
-            "exp2",
-            "logf",
-            "log",
-            "log10f",
-            "log10",
-            "log2f",
-            "log2",
-            "fmaf",
-            "fma",
             "fabsf",
             "fabs",
-            "fminf",
-            "fmin",
-            "fmaxf",
-            "fmax",
             "copysignf",
             "copysign",
-            "floorf",
-            "floor",
-            "ceilf",
-            "ceil",
-            "truncf",
-            "trunc",
-            "rintf",
-            "rint",
             "nearbyintf",
             "nearbyint",
-            "roundf",
-            "round",
         ];
 
         for builtin in builtins.iter() {

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -55,7 +55,7 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
 }
 
 /// Generate the `debug_context` in an MIR Body.
-/// # Souce of Origin
+/// # Source of Origin
 /// Copied from `create_scope_map.rs` of rustc_codegen_llvm
 fn compute_mir_scopes<'gcc, 'tcx>(
     cx: &CodegenCx<'gcc, 'tcx>,
@@ -90,7 +90,7 @@ fn compute_mir_scopes<'gcc, 'tcx>(
 /// Update the `debug_context`, adding new scope to it,
 /// if it's not added as is denoted in `instantiated`.
 ///
-/// # Souce of Origin
+/// # Source of Origin
 /// Copied from `create_scope_map.rs` of rustc_codegen_llvm
 /// FIXME(tempdragon/?): Add Scope Support Here.
 fn make_mir_scope<'gcc, 'tcx>(

compiler/rustc_codegen_gcc/src/declare.rs

@@ -168,7 +168,15 @@ fn declare_raw_fn<'gcc>(
     variadic: bool,
 ) -> Function<'gcc> {
     if name.starts_with("llvm.") {
-        let intrinsic = llvm::intrinsic(name, cx);
+        let intrinsic = match name {
+            "llvm.fma.f16" => {
+                // fma is not a target builtin, but a normal builtin, so we handle it differently
+                // here.
+                cx.context.get_builtin_function("fma")
+            }
+            _ => llvm::intrinsic(name, cx),
+        };
+
         cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic);
         return intrinsic;
     }

compiler/rustc_codegen_gcc/src/int.rs

@@ -733,7 +733,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
 
         // TODO(antoyo): check if it's faster to use string literals and a
         // match instead of format!.
-        let bswap = self.cx.context.get_builtin_function(&format!("__builtin_bswap{}", width));
+        let bswap = self.cx.context.get_builtin_function(format!("__builtin_bswap{}", width));
         // FIXME(antoyo): this cast should not be necessary. Remove
         // when having proper sized integer types.
         let param_type = bswap.get_param(0).to_rvalue().get_type();

compiler/rustc_codegen_gcc/src/intrinsic/archs.rs

@@ -31,8 +31,11 @@ match name {
     "llvm.AMDGPU.trig.preop.v2f64" => "__builtin_amdgpu_trig_preop",
     "llvm.AMDGPU.trig.preop.v4f32" => "__builtin_amdgpu_trig_preop",
     // aarch64
+    "llvm.aarch64.chkfeat" => "__builtin_arm_chkfeat",
     "llvm.aarch64.dmb" => "__builtin_arm_dmb",
     "llvm.aarch64.dsb" => "__builtin_arm_dsb",
+    "llvm.aarch64.gcspopm" => "__builtin_arm_gcspopm",
+    "llvm.aarch64.gcsss" => "__builtin_arm_gcsss",
     "llvm.aarch64.isb" => "__builtin_arm_isb",
     "llvm.aarch64.prefetch" => "__builtin_arm_prefetch",
     "llvm.aarch64.sve.aesd" => "__builtin_sve_svaesd_u8",
@@ -80,7 +83,6 @@ match name {
     "llvm.amdgcn.dot4.f32.fp8.fp8" => "__builtin_amdgcn_dot4_f32_fp8_fp8",
     "llvm.amdgcn.ds.add.gs.reg.rtn" => "__builtin_amdgcn_ds_add_gs_reg_rtn",
     "llvm.amdgcn.ds.bpermute" => "__builtin_amdgcn_ds_bpermute",
-    "llvm.amdgcn.ds.fadd.v2bf16" => "__builtin_amdgcn_ds_atomic_fadd_v2bf16",
     "llvm.amdgcn.ds.gws.barrier" => "__builtin_amdgcn_ds_gws_barrier",
     "llvm.amdgcn.ds.gws.init" => "__builtin_amdgcn_ds_gws_init",
     "llvm.amdgcn.ds.gws.sema.br" => "__builtin_amdgcn_ds_gws_sema_br",
@@ -96,6 +98,7 @@ match name {
     "llvm.amdgcn.fdot2.f16.f16" => "__builtin_amdgcn_fdot2_f16_f16",
     "llvm.amdgcn.fdot2.f32.bf16" => "__builtin_amdgcn_fdot2_f32_bf16",
     "llvm.amdgcn.fmul.legacy" => "__builtin_amdgcn_fmul_legacy",
+    "llvm.amdgcn.global.load.lds" => "__builtin_amdgcn_global_load_lds",
     "llvm.amdgcn.groupstaticsize" => "__builtin_amdgcn_groupstaticsize",
     "llvm.amdgcn.iglp.opt" => "__builtin_amdgcn_iglp_opt",
     "llvm.amdgcn.implicit.buffer.ptr" => "__builtin_amdgcn_implicit_buffer_ptr",
@@ -154,16 +157,11 @@ match name {
     "llvm.amdgcn.mqsad.u32.u8" => "__builtin_amdgcn_mqsad_u32_u8",
     "llvm.amdgcn.msad.u8" => "__builtin_amdgcn_msad_u8",
     "llvm.amdgcn.perm" => "__builtin_amdgcn_perm",
-    "llvm.amdgcn.permlane16" => "__builtin_amdgcn_permlane16",
     "llvm.amdgcn.permlane16.var" => "__builtin_amdgcn_permlane16_var",
-    "llvm.amdgcn.permlane64" => "__builtin_amdgcn_permlane64",
-    "llvm.amdgcn.permlanex16" => "__builtin_amdgcn_permlanex16",
     "llvm.amdgcn.permlanex16.var" => "__builtin_amdgcn_permlanex16_var",
     "llvm.amdgcn.qsad.pk.u16.u8" => "__builtin_amdgcn_qsad_pk_u16_u8",
     "llvm.amdgcn.queue.ptr" => "__builtin_amdgcn_queue_ptr",
     "llvm.amdgcn.rcp.legacy" => "__builtin_amdgcn_rcp_legacy",
-    "llvm.amdgcn.readfirstlane" => "__builtin_amdgcn_readfirstlane",
-    "llvm.amdgcn.readlane" => "__builtin_amdgcn_readlane",
     "llvm.amdgcn.rsq.legacy" => "__builtin_amdgcn_rsq_legacy",
     "llvm.amdgcn.s.barrier" => "__builtin_amdgcn_s_barrier",
     "llvm.amdgcn.s.barrier.init" => "__builtin_amdgcn_s_barrier_init",
@@ -192,6 +190,8 @@ match name {
     "llvm.amdgcn.s.setreg" => "__builtin_amdgcn_s_setreg",
     "llvm.amdgcn.s.sleep" => "__builtin_amdgcn_s_sleep",
     "llvm.amdgcn.s.sleep.var" => "__builtin_amdgcn_s_sleep_var",
+    "llvm.amdgcn.s.ttracedata" => "__builtin_amdgcn_s_ttracedata",
+    "llvm.amdgcn.s.ttracedata.imm" => "__builtin_amdgcn_s_ttracedata_imm",
     "llvm.amdgcn.s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready",
     "llvm.amdgcn.s.waitcnt" => "__builtin_amdgcn_s_waitcnt",
     "llvm.amdgcn.s.wakeup.barrier" => "__builtin_amdgcn_s_wakeup_barrier",
@@ -227,7 +227,6 @@ match name {
     "llvm.amdgcn.workgroup.id.x" => "__builtin_amdgcn_workgroup_id_x",
     "llvm.amdgcn.workgroup.id.y" => "__builtin_amdgcn_workgroup_id_y",
     "llvm.amdgcn.workgroup.id.z" => "__builtin_amdgcn_workgroup_id_z",
-    "llvm.amdgcn.writelane" => "__builtin_amdgcn_writelane",
     // arm
     "llvm.arm.cdp" => "__builtin_arm_cdp",
     "llvm.arm.cdp2" => "__builtin_arm_cdp2",
@@ -4536,10 +4535,18 @@ match name {
     "llvm.nvvm.div.rz.d" => "__nvvm_div_rz_d",
     "llvm.nvvm.div.rz.f" => "__nvvm_div_rz_f",
     "llvm.nvvm.div.rz.ftz.f" => "__nvvm_div_rz_ftz_f",
+    "llvm.nvvm.e4m3x2.to.f16x2.rn" => "__nvvm_e4m3x2_to_f16x2_rn",
+    "llvm.nvvm.e4m3x2.to.f16x2.rn.relu" => "__nvvm_e4m3x2_to_f16x2_rn_relu",
+    "llvm.nvvm.e5m2x2.to.f16x2.rn" => "__nvvm_e5m2x2_to_f16x2_rn",
+    "llvm.nvvm.e5m2x2.to.f16x2.rn.relu" => "__nvvm_e5m2x2_to_f16x2_rn_relu",
     "llvm.nvvm.ex2.approx.d" => "__nvvm_ex2_approx_d",
     "llvm.nvvm.ex2.approx.f" => "__nvvm_ex2_approx_f",
     "llvm.nvvm.ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
     "llvm.nvvm.exit" => "__nvvm_exit",
+    "llvm.nvvm.f16x2.to.e4m3x2.rn" => "__nvvm_f16x2_to_e4m3x2_rn",
+    "llvm.nvvm.f16x2.to.e4m3x2.rn.relu" => "__nvvm_f16x2_to_e4m3x2_rn_relu",
+    "llvm.nvvm.f16x2.to.e5m2x2.rn" => "__nvvm_f16x2_to_e5m2x2_rn",
+    "llvm.nvvm.f16x2.to.e5m2x2.rn.relu" => "__nvvm_f16x2_to_e5m2x2_rn_relu",
     "llvm.nvvm.f2bf16.rn" => "__nvvm_f2bf16_rn",
     "llvm.nvvm.f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
     "llvm.nvvm.f2bf16.rz" => "__nvvm_f2bf16_rz",
@@ -4582,6 +4589,10 @@ match name {
     "llvm.nvvm.fabs.d" => "__nvvm_fabs_d",
     "llvm.nvvm.fabs.f" => "__nvvm_fabs_f",
     "llvm.nvvm.fabs.ftz.f" => "__nvvm_fabs_ftz_f",
+    "llvm.nvvm.ff.to.e4m3x2.rn" => "__nvvm_ff_to_e4m3x2_rn",
+    "llvm.nvvm.ff.to.e4m3x2.rn.relu" => "__nvvm_ff_to_e4m3x2_rn_relu",
+    "llvm.nvvm.ff.to.e5m2x2.rn" => "__nvvm_ff_to_e5m2x2_rn",
+    "llvm.nvvm.ff.to.e5m2x2.rn.relu" => "__nvvm_ff_to_e5m2x2_rn_relu",
     "llvm.nvvm.ff2bf16x2.rn" => "__nvvm_ff2bf16x2_rn",
     "llvm.nvvm.ff2bf16x2.rn.relu" => "__nvvm_ff2bf16x2_rn_relu",
     "llvm.nvvm.ff2bf16x2.rz" => "__nvvm_ff2bf16x2_rz",
@@ -4866,6 +4877,7 @@ match name {
     "llvm.nvvm.round.ftz.f" => "__nvvm_round_ftz_f",
     "llvm.nvvm.rsqrt.approx.d" => "__nvvm_rsqrt_approx_d",
     "llvm.nvvm.rsqrt.approx.f" => "__nvvm_rsqrt_approx_f",
+    "llvm.nvvm.rsqrt.approx.ftz.d" => "__nvvm_rsqrt_approx_ftz_d",
     "llvm.nvvm.rsqrt.approx.ftz.f" => "__nvvm_rsqrt_approx_ftz_f",
     "llvm.nvvm.sad.i" => "__nvvm_sad_i",
     "llvm.nvvm.sad.ll" => "__nvvm_sad_ll",
@@ -5164,6 +5176,8 @@ match name {
     // ppc
     "llvm.ppc.addex" => "__builtin_ppc_addex",
     "llvm.ppc.addf128.round.to.odd" => "__builtin_addf128_round_to_odd",
+    "llvm.ppc.addg6s" => "__builtin_addg6s",
+    "llvm.ppc.addg6sd" => "__builtin_ppc_addg6s",
     "llvm.ppc.altivec.crypto.vcipher" => "__builtin_altivec_crypto_vcipher",
     "llvm.ppc.altivec.crypto.vcipherlast" => "__builtin_altivec_crypto_vcipherlast",
     "llvm.ppc.altivec.crypto.vncipher" => "__builtin_altivec_crypto_vncipher",
@@ -5461,6 +5475,10 @@ match name {
     "llvm.ppc.bcdsub" => "__builtin_ppc_bcdsub",
     "llvm.ppc.bcdsub.p" => "__builtin_ppc_bcdsub_p",
     "llvm.ppc.bpermd" => "__builtin_bpermd",
+    "llvm.ppc.cbcdtd" => "__builtin_cbcdtd",
+    "llvm.ppc.cbcdtdd" => "__builtin_ppc_cbcdtd",
+    "llvm.ppc.cdtbcd" => "__builtin_cdtbcd",
+    "llvm.ppc.cdtbcdd" => "__builtin_ppc_cdtbcd",
     "llvm.ppc.cfuged" => "__builtin_cfuged",
     "llvm.ppc.cmpeqb" => "__builtin_ppc_cmpeqb",
     "llvm.ppc.cmprb" => "__builtin_ppc_cmprb",
@@ -5627,7 +5645,6 @@ match name {
     "llvm.ppc.qpx.qvstfs" => "__builtin_qpx_qvstfs",
     "llvm.ppc.qpx.qvstfsa" => "__builtin_qpx_qvstfsa",
     "llvm.ppc.readflm" => "__builtin_readflm",
-    "llvm.ppc.rldimi" => "__builtin_ppc_rldimi",
     "llvm.ppc.rlwimi" => "__builtin_ppc_rlwimi",
     "llvm.ppc.rlwnm" => "__builtin_ppc_rlwnm",
     "llvm.ppc.scalar.extract.expq" => "__builtin_vsx_scalar_extract_expq",
@@ -7210,29 +7227,6 @@ match name {
     "llvm.ve.vl.xorm.MMM" => "__builtin_ve_vl_xorm_MMM",
     "llvm.ve.vl.xorm.mmm" => "__builtin_ve_vl_xorm_mmm",
     // x86
-    "llvm.x86.3dnow.pavgusb" => "__builtin_ia32_pavgusb",
-    "llvm.x86.3dnow.pf2id" => "__builtin_ia32_pf2id",
-    "llvm.x86.3dnow.pfacc" => "__builtin_ia32_pfacc",
-    "llvm.x86.3dnow.pfadd" => "__builtin_ia32_pfadd",
-    "llvm.x86.3dnow.pfcmpeq" => "__builtin_ia32_pfcmpeq",
-    "llvm.x86.3dnow.pfcmpge" => "__builtin_ia32_pfcmpge",
-    "llvm.x86.3dnow.pfcmpgt" => "__builtin_ia32_pfcmpgt",
-    "llvm.x86.3dnow.pfmax" => "__builtin_ia32_pfmax",
-    "llvm.x86.3dnow.pfmin" => "__builtin_ia32_pfmin",
-    "llvm.x86.3dnow.pfmul" => "__builtin_ia32_pfmul",
-    "llvm.x86.3dnow.pfrcp" => "__builtin_ia32_pfrcp",
-    "llvm.x86.3dnow.pfrcpit1" => "__builtin_ia32_pfrcpit1",
-    "llvm.x86.3dnow.pfrcpit2" => "__builtin_ia32_pfrcpit2",
-    "llvm.x86.3dnow.pfrsqit1" => "__builtin_ia32_pfrsqit1",
-    "llvm.x86.3dnow.pfrsqrt" => "__builtin_ia32_pfrsqrt",
-    "llvm.x86.3dnow.pfsub" => "__builtin_ia32_pfsub",
-    "llvm.x86.3dnow.pfsubr" => "__builtin_ia32_pfsubr",
-    "llvm.x86.3dnow.pi2fd" => "__builtin_ia32_pi2fd",
-    "llvm.x86.3dnow.pmulhrw" => "__builtin_ia32_pmulhrw",
-    "llvm.x86.3dnowa.pf2iw" => "__builtin_ia32_pf2iw",
-    "llvm.x86.3dnowa.pfnacc" => "__builtin_ia32_pfnacc",
-    "llvm.x86.3dnowa.pfpnacc" => "__builtin_ia32_pfpnacc",
-    "llvm.x86.3dnowa.pi2fw" => "__builtin_ia32_pi2fw",
     "llvm.x86.aadd32" => "__builtin_ia32_aadd32",
     "llvm.x86.aadd64" => "__builtin_ia32_aadd64",
     "llvm.x86.aand32" => "__builtin_ia32_aand32",
@@ -7334,6 +7328,207 @@ match name {
     "llvm.x86.avx.vtestz.ps.256" => "__builtin_ia32_vtestzps256",
     "llvm.x86.avx.vzeroall" => "__builtin_ia32_vzeroall",
     "llvm.x86.avx.vzeroupper" => "__builtin_ia32_vzeroupper",
+    "llvm.x86.avx10.mask.vcvt2ps2phx.128" => "__builtin_ia32_vcvt2ps2phx128_mask",
+    "llvm.x86.avx10.mask.vcvt2ps2phx.256" => "__builtin_ia32_vcvt2ps2phx256_mask",
+    "llvm.x86.avx10.mask.vcvt2ps2phx.512" => "__builtin_ia32_vcvt2ps2phx512_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2bf8128" => "__builtin_ia32_vcvtbiasph2bf8_128_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2bf8256" => "__builtin_ia32_vcvtbiasph2bf8_256_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2bf8512" => "__builtin_ia32_vcvtbiasph2bf8_512_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2bf8s128" => "__builtin_ia32_vcvtbiasph2bf8s_128_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2bf8s256" => "__builtin_ia32_vcvtbiasph2bf8s_256_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2bf8s512" => "__builtin_ia32_vcvtbiasph2bf8s_512_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2hf8128" => "__builtin_ia32_vcvtbiasph2hf8_128_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2hf8256" => "__builtin_ia32_vcvtbiasph2hf8_256_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2hf8512" => "__builtin_ia32_vcvtbiasph2hf8_512_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2hf8s128" => "__builtin_ia32_vcvtbiasph2hf8s_128_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2hf8s256" => "__builtin_ia32_vcvtbiasph2hf8s_256_mask",
+    "llvm.x86.avx10.mask.vcvtbiasph2hf8s512" => "__builtin_ia32_vcvtbiasph2hf8s_512_mask",
+    "llvm.x86.avx10.mask.vcvthf82ph128" => "__builtin_ia32_vcvthf8_2ph128_mask",
+    "llvm.x86.avx10.mask.vcvthf82ph256" => "__builtin_ia32_vcvthf8_2ph256_mask",
+    "llvm.x86.avx10.mask.vcvthf82ph512" => "__builtin_ia32_vcvthf8_2ph512_mask",
+    "llvm.x86.avx10.mask.vcvtneph2bf8128" => "__builtin_ia32_vcvtneph2bf8_128_mask",
+    "llvm.x86.avx10.mask.vcvtneph2bf8256" => "__builtin_ia32_vcvtneph2bf8_256_mask",
+    "llvm.x86.avx10.mask.vcvtneph2bf8512" => "__builtin_ia32_vcvtneph2bf8_512_mask",
+    "llvm.x86.avx10.mask.vcvtneph2bf8s128" => "__builtin_ia32_vcvtneph2bf8s_128_mask",
+    "llvm.x86.avx10.mask.vcvtneph2bf8s256" => "__builtin_ia32_vcvtneph2bf8s_256_mask",
+    "llvm.x86.avx10.mask.vcvtneph2bf8s512" => "__builtin_ia32_vcvtneph2bf8s_512_mask",
+    "llvm.x86.avx10.mask.vcvtneph2hf8128" => "__builtin_ia32_vcvtneph2hf8_128_mask",
+    "llvm.x86.avx10.mask.vcvtneph2hf8256" => "__builtin_ia32_vcvtneph2hf8_256_mask",
+    "llvm.x86.avx10.mask.vcvtneph2hf8512" => "__builtin_ia32_vcvtneph2hf8_512_mask",
+    "llvm.x86.avx10.mask.vcvtneph2hf8s128" => "__builtin_ia32_vcvtneph2hf8s_128_mask",
+    "llvm.x86.avx10.mask.vcvtneph2hf8s256" => "__builtin_ia32_vcvtneph2hf8s_256_mask",
+    "llvm.x86.avx10.mask.vcvtneph2hf8s512" => "__builtin_ia32_vcvtneph2hf8s_512_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2dq256" => "__builtin_ia32_vcvtpd2dq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2ph256" => "__builtin_ia32_vcvtpd2ph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2ps256" => "__builtin_ia32_vcvtpd2ps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2qq256" => "__builtin_ia32_vcvtpd2qq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2udq256" => "__builtin_ia32_vcvtpd2udq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2uqq256" => "__builtin_ia32_vcvtpd2uqq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2dq256" => "__builtin_ia32_vcvtph2dq256_round_mask",
+    "llvm.x86.avx10.mask.vcvtph2ibs128" => "__builtin_ia32_vcvtph2ibs128_mask",
+    "llvm.x86.avx10.mask.vcvtph2ibs256" => "__builtin_ia32_vcvtph2ibs256_mask",
+    "llvm.x86.avx10.mask.vcvtph2ibs512" => "__builtin_ia32_vcvtph2ibs512_mask",
+    "llvm.x86.avx10.mask.vcvtph2iubs128" => "__builtin_ia32_vcvtph2iubs128_mask",
+    "llvm.x86.avx10.mask.vcvtph2iubs256" => "__builtin_ia32_vcvtph2iubs256_mask",
+    "llvm.x86.avx10.mask.vcvtph2iubs512" => "__builtin_ia32_vcvtph2iubs512_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2pd256" => "__builtin_ia32_vcvtph2pd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2psx256" => "__builtin_ia32_vcvtph2psx256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2qq256" => "__builtin_ia32_vcvtph2qq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2udq256" => "__builtin_ia32_vcvtph2udq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2uqq256" => "__builtin_ia32_vcvtph2uqq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2uw256" => "__builtin_ia32_vcvtph2uw256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2w256" => "__builtin_ia32_vcvtph2w256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2dq256" => "__builtin_ia32_vcvtps2dq256_round_mask",
+    "llvm.x86.avx10.mask.vcvtps2ibs128" => "__builtin_ia32_vcvtps2ibs128_mask",
+    "llvm.x86.avx10.mask.vcvtps2ibs256" => "__builtin_ia32_vcvtps2ibs256_mask",
+    "llvm.x86.avx10.mask.vcvtps2ibs512" => "__builtin_ia32_vcvtps2ibs512_mask",
+    "llvm.x86.avx10.mask.vcvtps2iubs128" => "__builtin_ia32_vcvtps2iubs128_mask",
+    "llvm.x86.avx10.mask.vcvtps2iubs256" => "__builtin_ia32_vcvtps2iubs256_mask",
+    "llvm.x86.avx10.mask.vcvtps2iubs512" => "__builtin_ia32_vcvtps2iubs512_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2pd256" => "__builtin_ia32_vcvtps2pd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2ph256" => "__builtin_ia32_vcvtps2ph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2phx256" => "__builtin_ia32_vcvtps2phx256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2qq256" => "__builtin_ia32_vcvtps2qq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2udq256" => "__builtin_ia32_vcvtps2udq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2uqq256" => "__builtin_ia32_vcvtps2uqq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2dq256" => "__builtin_ia32_vcvttpd2dq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2qq256" => "__builtin_ia32_vcvttpd2qq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2udq256" => "__builtin_ia32_vcvttpd2udq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2uqq256" => "__builtin_ia32_vcvttpd2uqq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2dq256" => "__builtin_ia32_vcvttph2dq256_round_mask",
+    "llvm.x86.avx10.mask.vcvttph2ibs128" => "__builtin_ia32_vcvttph2ibs128_mask",
+    "llvm.x86.avx10.mask.vcvttph2ibs256" => "__builtin_ia32_vcvttph2ibs256_mask",
+    "llvm.x86.avx10.mask.vcvttph2ibs512" => "__builtin_ia32_vcvttph2ibs512_mask",
+    "llvm.x86.avx10.mask.vcvttph2iubs128" => "__builtin_ia32_vcvttph2iubs128_mask",
+    "llvm.x86.avx10.mask.vcvttph2iubs256" => "__builtin_ia32_vcvttph2iubs256_mask",
+    "llvm.x86.avx10.mask.vcvttph2iubs512" => "__builtin_ia32_vcvttph2iubs512_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2qq256" => "__builtin_ia32_vcvttph2qq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2udq256" => "__builtin_ia32_vcvttph2udq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2uqq256" => "__builtin_ia32_vcvttph2uqq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2uw256" => "__builtin_ia32_vcvttph2uw256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2w256" => "__builtin_ia32_vcvttph2w256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2dq256" => "__builtin_ia32_vcvttps2dq256_round_mask",
+    "llvm.x86.avx10.mask.vcvttps2ibs128" => "__builtin_ia32_vcvttps2ibs128_mask",
+    "llvm.x86.avx10.mask.vcvttps2ibs256" => "__builtin_ia32_vcvttps2ibs256_mask",
+    "llvm.x86.avx10.mask.vcvttps2ibs512" => "__builtin_ia32_vcvttps2ibs512_mask",
+    "llvm.x86.avx10.mask.vcvttps2iubs128" => "__builtin_ia32_vcvttps2iubs128_mask",
+    "llvm.x86.avx10.mask.vcvttps2iubs256" => "__builtin_ia32_vcvttps2iubs256_mask",
+    "llvm.x86.avx10.mask.vcvttps2iubs512" => "__builtin_ia32_vcvttps2iubs512_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2qq256" => "__builtin_ia32_vcvttps2qq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2udq256" => "__builtin_ia32_vcvttps2udq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2uqq256" => "__builtin_ia32_vcvttps2uqq256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfcmaddcph256" => "__builtin_ia32_vfcmaddcph256_round_mask3",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfcmulcph256" => "__builtin_ia32_vfcmulcph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfixupimmpd256" => "__builtin_ia32_vfixupimmpd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfixupimmps256" => "__builtin_ia32_vfixupimmps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfmaddcph256" => "__builtin_ia32_vfmaddcph256_round_mask3",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfmulcph256" => "__builtin_ia32_vfmulcph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexppd256" => "__builtin_ia32_vgetexppd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexpph256" => "__builtin_ia32_vgetexpph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexpps256" => "__builtin_ia32_vgetexpps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantpd256" => "__builtin_ia32_vgetmantpd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantph256" => "__builtin_ia32_vgetmantph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantps256" => "__builtin_ia32_vgetmantps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxpd.round" => "__builtin_ia32_vminmaxpd512_round_mask",
+    "llvm.x86.avx10.mask.vminmaxpd128" => "__builtin_ia32_vminmaxpd128_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxpd256.round" => "__builtin_ia32_vminmaxpd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxph.round" => "__builtin_ia32_vminmaxph512_round_mask",
+    "llvm.x86.avx10.mask.vminmaxph128" => "__builtin_ia32_vminmaxph128_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxph256.round" => "__builtin_ia32_vminmaxph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxps.round" => "__builtin_ia32_vminmaxps512_round_mask",
+    "llvm.x86.avx10.mask.vminmaxps128" => "__builtin_ia32_vminmaxps128_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxps256.round" => "__builtin_ia32_vminmaxps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxsd.round" => "__builtin_ia32_vminmaxsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxsh.round" => "__builtin_ia32_vminmaxsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxss.round" => "__builtin_ia32_vminmaxss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrangepd256" => "__builtin_ia32_vrangepd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrangeps256" => "__builtin_ia32_vrangeps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreducepd256" => "__builtin_ia32_vreducepd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreduceph256" => "__builtin_ia32_vreduceph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreduceps256" => "__builtin_ia32_vreduceps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscalepd256" => "__builtin_ia32_vrndscalepd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscaleph256" => "__builtin_ia32_vrndscaleph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscaleps256" => "__builtin_ia32_vrndscaleps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefpd256" => "__builtin_ia32_vscalefpd256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefph256" => "__builtin_ia32_vscalefph256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefps256" => "__builtin_ia32_vscalefps256_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfcmaddcph256" => "__builtin_ia32_vfcmaddcph256_round_maskz",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfixupimmpd256" => "__builtin_ia32_vfixupimmpd256_round_maskz",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfixupimmps256" => "__builtin_ia32_vfixupimmps256_round_maskz",
+    // [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfmaddcph256" => "__builtin_ia32_vfmaddcph256_round_maskz",
+    "llvm.x86.avx10.vaddpd256" => "__builtin_ia32_vaddpd256_round",
+    "llvm.x86.avx10.vaddph256" => "__builtin_ia32_vaddph256_round",
+    "llvm.x86.avx10.vaddps256" => "__builtin_ia32_vaddps256_round",
+    "llvm.x86.avx10.vcvtne2ph2bf8128" => "__builtin_ia32_vcvtne2ph2bf8_128",
+    "llvm.x86.avx10.vcvtne2ph2bf8256" => "__builtin_ia32_vcvtne2ph2bf8_256",
+    "llvm.x86.avx10.vcvtne2ph2bf8512" => "__builtin_ia32_vcvtne2ph2bf8_512",
+    "llvm.x86.avx10.vcvtne2ph2bf8s128" => "__builtin_ia32_vcvtne2ph2bf8s_128",
+    "llvm.x86.avx10.vcvtne2ph2bf8s256" => "__builtin_ia32_vcvtne2ph2bf8s_256",
+    "llvm.x86.avx10.vcvtne2ph2bf8s512" => "__builtin_ia32_vcvtne2ph2bf8s_512",
+    "llvm.x86.avx10.vcvtne2ph2hf8128" => "__builtin_ia32_vcvtne2ph2hf8_128",
+    "llvm.x86.avx10.vcvtne2ph2hf8256" => "__builtin_ia32_vcvtne2ph2hf8_256",
+    "llvm.x86.avx10.vcvtne2ph2hf8512" => "__builtin_ia32_vcvtne2ph2hf8_512",
+    "llvm.x86.avx10.vcvtne2ph2hf8s128" => "__builtin_ia32_vcvtne2ph2hf8s_128",
+    "llvm.x86.avx10.vcvtne2ph2hf8s256" => "__builtin_ia32_vcvtne2ph2hf8s_256",
+    "llvm.x86.avx10.vcvtne2ph2hf8s512" => "__builtin_ia32_vcvtne2ph2hf8s_512",
+    "llvm.x86.avx10.vcvtnebf162ibs128" => "__builtin_ia32_vcvtnebf162ibs128",
+    "llvm.x86.avx10.vcvtnebf162ibs256" => "__builtin_ia32_vcvtnebf162ibs256",
+    "llvm.x86.avx10.vcvtnebf162ibs512" => "__builtin_ia32_vcvtnebf162ibs512",
+    "llvm.x86.avx10.vcvtnebf162iubs128" => "__builtin_ia32_vcvtnebf162iubs128",
+    "llvm.x86.avx10.vcvtnebf162iubs256" => "__builtin_ia32_vcvtnebf162iubs256",
+    "llvm.x86.avx10.vcvtnebf162iubs512" => "__builtin_ia32_vcvtnebf162iubs512",
+    "llvm.x86.avx10.vcvttnebf162ibs128" => "__builtin_ia32_vcvttnebf162ibs128",
+    "llvm.x86.avx10.vcvttnebf162ibs256" => "__builtin_ia32_vcvttnebf162ibs256",
+    "llvm.x86.avx10.vcvttnebf162ibs512" => "__builtin_ia32_vcvttnebf162ibs512",
+    "llvm.x86.avx10.vcvttnebf162iubs128" => "__builtin_ia32_vcvttnebf162iubs128",
+    "llvm.x86.avx10.vcvttnebf162iubs256" => "__builtin_ia32_vcvttnebf162iubs256",
+    "llvm.x86.avx10.vcvttnebf162iubs512" => "__builtin_ia32_vcvttnebf162iubs512",
+    "llvm.x86.avx10.vdivpd256" => "__builtin_ia32_vdivpd256_round",
+    "llvm.x86.avx10.vdivph256" => "__builtin_ia32_vdivph256_round",
+    "llvm.x86.avx10.vdivps256" => "__builtin_ia32_vdivps256_round",
+    "llvm.x86.avx10.vdpphps.128" => "__builtin_ia32_vdpphps128",
+    "llvm.x86.avx10.vdpphps.256" => "__builtin_ia32_vdpphps256",
+    "llvm.x86.avx10.vdpphps.512" => "__builtin_ia32_vdpphps512",
+    "llvm.x86.avx10.vfmaddsubpd256" => "__builtin_ia32_vfmaddsubpd256_round",
+    "llvm.x86.avx10.vfmaddsubph256" => "__builtin_ia32_vfmaddsubph256_round",
+    "llvm.x86.avx10.vfmaddsubps256" => "__builtin_ia32_vfmaddsubps256_round",
+    "llvm.x86.avx10.vmaxpd256" => "__builtin_ia32_vmaxpd256_round",
+    "llvm.x86.avx10.vmaxph256" => "__builtin_ia32_vmaxph256_round",
+    "llvm.x86.avx10.vmaxps256" => "__builtin_ia32_vmaxps256_round",
+    "llvm.x86.avx10.vminmaxnepbf16128" => "__builtin_ia32_vminmaxnepbf16128",
+    "llvm.x86.avx10.vminmaxnepbf16256" => "__builtin_ia32_vminmaxnepbf16256",
+    "llvm.x86.avx10.vminmaxnepbf16512" => "__builtin_ia32_vminmaxnepbf16512",
+    "llvm.x86.avx10.vminmaxpd128" => "__builtin_ia32_vminmaxpd128",
+    "llvm.x86.avx10.vminmaxpd256" => "__builtin_ia32_vminmaxpd256",
+    "llvm.x86.avx10.vminmaxph128" => "__builtin_ia32_vminmaxph128",
+    "llvm.x86.avx10.vminmaxph256" => "__builtin_ia32_vminmaxph256",
+    "llvm.x86.avx10.vminmaxps128" => "__builtin_ia32_vminmaxps128",
+    "llvm.x86.avx10.vminmaxps256" => "__builtin_ia32_vminmaxps256",
+    "llvm.x86.avx10.vminpd256" => "__builtin_ia32_vminpd256_round",
+    "llvm.x86.avx10.vminph256" => "__builtin_ia32_vminph256_round",
+    "llvm.x86.avx10.vminps256" => "__builtin_ia32_vminps256_round",
+    "llvm.x86.avx10.vmpsadbw.512" => "__builtin_ia32_mpsadbw512",
+    "llvm.x86.avx10.vmulpd256" => "__builtin_ia32_vmulpd256_round",
+    "llvm.x86.avx10.vmulph256" => "__builtin_ia32_vmulph256_round",
+    "llvm.x86.avx10.vmulps256" => "__builtin_ia32_vmulps256_round",
+    "llvm.x86.avx10.vpdpbssd.512" => "__builtin_ia32_vpdpbssd512",
+    "llvm.x86.avx10.vpdpbssds.512" => "__builtin_ia32_vpdpbssds512",
+    "llvm.x86.avx10.vpdpbsud.512" => "__builtin_ia32_vpdpbsud512",
+    "llvm.x86.avx10.vpdpbsuds.512" => "__builtin_ia32_vpdpbsuds512",
+    "llvm.x86.avx10.vpdpbuud.512" => "__builtin_ia32_vpdpbuud512",
+    "llvm.x86.avx10.vpdpbuuds.512" => "__builtin_ia32_vpdpbuuds512",
+    "llvm.x86.avx10.vpdpwsud.512" => "__builtin_ia32_vpdpwsud512",
+    "llvm.x86.avx10.vpdpwsuds.512" => "__builtin_ia32_vpdpwsuds512",
+    "llvm.x86.avx10.vpdpwusd.512" => "__builtin_ia32_vpdpwusd512",
+    "llvm.x86.avx10.vpdpwusds.512" => "__builtin_ia32_vpdpwusds512",
+    "llvm.x86.avx10.vpdpwuud.512" => "__builtin_ia32_vpdpwuud512",
+    "llvm.x86.avx10.vpdpwuuds.512" => "__builtin_ia32_vpdpwuuds512",
+    "llvm.x86.avx10.vsqrtpd256" => "__builtin_ia32_vsqrtpd256_round",
+    "llvm.x86.avx10.vsqrtph256" => "__builtin_ia32_vsqrtph256_round",
+    "llvm.x86.avx10.vsqrtps256" => "__builtin_ia32_vsqrtps256_round",
+    "llvm.x86.avx10.vsubpd256" => "__builtin_ia32_vsubpd256_round",
+    "llvm.x86.avx10.vsubph256" => "__builtin_ia32_vsubph256_round",
+    "llvm.x86.avx10.vsubps256" => "__builtin_ia32_vsubps256_round",
     "llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gatherd_d",
     "llvm.x86.avx2.gather.d.d.256" => "__builtin_ia32_gatherd_d256",
     "llvm.x86.avx2.gather.d.pd" => "__builtin_ia32_gatherd_pd",
@@ -8738,10 +8933,10 @@ match name {
     "llvm.x86.avx512.rcp14.ss" => "__builtin_ia32_rcp14ss_mask",
     "llvm.x86.avx512.rcp28.pd" => "__builtin_ia32_rcp28pd_mask",
     "llvm.x86.avx512.rcp28.ps" => "__builtin_ia32_rcp28ps_mask",
-    // [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
-    // [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
-    // [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
-    // [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
+    "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
+    // [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
+    "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
+    // [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
     "llvm.x86.avx512.rndscale.sd" => "__builtin_ia32_rndscalesd",
     "llvm.x86.avx512.rndscale.ss" => "__builtin_ia32_rndscaless",
     "llvm.x86.avx512.rsqrt14.pd.128" => "__builtin_ia32_rsqrt14pd128_mask",
@@ -8754,10 +8949,10 @@ match name {
     "llvm.x86.avx512.rsqrt14.ss" => "__builtin_ia32_rsqrt14ss_mask",
     "llvm.x86.avx512.rsqrt28.pd" => "__builtin_ia32_rsqrt28pd_mask",
     "llvm.x86.avx512.rsqrt28.ps" => "__builtin_ia32_rsqrt28ps_mask",
-    // [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
-    // [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
-    // [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
-    // [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
+    "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
+    // [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
+    "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
+    // [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
     "llvm.x86.avx512.scatter.dpd.512" => "__builtin_ia32_scattersiv8df",
     "llvm.x86.avx512.scatter.dpi.512" => "__builtin_ia32_scattersiv16si",
     "llvm.x86.avx512.scatter.dpq.512" => "__builtin_ia32_scattersiv8di",
@@ -9082,75 +9277,6 @@ match name {
     "llvm.x86.lwpval64" => "__builtin_ia32_lwpval64",
     "llvm.x86.mmx.emms" => "__builtin_ia32_emms",
     "llvm.x86.mmx.femms" => "__builtin_ia32_femms",
-    "llvm.x86.mmx.maskmovq" => "__builtin_ia32_maskmovq",
-    "llvm.x86.mmx.movnt.dq" => "__builtin_ia32_movntq",
-    "llvm.x86.mmx.packssdw" => "__builtin_ia32_packssdw",
-    "llvm.x86.mmx.packsswb" => "__builtin_ia32_packsswb",
-    "llvm.x86.mmx.packuswb" => "__builtin_ia32_packuswb",
-    "llvm.x86.mmx.padd.b" => "__builtin_ia32_paddb",
-    "llvm.x86.mmx.padd.d" => "__builtin_ia32_paddd",
-    "llvm.x86.mmx.padd.q" => "__builtin_ia32_paddq",
-    "llvm.x86.mmx.padd.w" => "__builtin_ia32_paddw",
-    "llvm.x86.mmx.padds.b" => "__builtin_ia32_paddsb",
-    "llvm.x86.mmx.padds.w" => "__builtin_ia32_paddsw",
-    "llvm.x86.mmx.paddus.b" => "__builtin_ia32_paddusb",
-    "llvm.x86.mmx.paddus.w" => "__builtin_ia32_paddusw",
-    "llvm.x86.mmx.palignr.b" => "__builtin_ia32_palignr",
-    "llvm.x86.mmx.pand" => "__builtin_ia32_pand",
-    "llvm.x86.mmx.pandn" => "__builtin_ia32_pandn",
-    "llvm.x86.mmx.pavg.b" => "__builtin_ia32_pavgb",
-    "llvm.x86.mmx.pavg.w" => "__builtin_ia32_pavgw",
-    "llvm.x86.mmx.pcmpeq.b" => "__builtin_ia32_pcmpeqb",
-    "llvm.x86.mmx.pcmpeq.d" => "__builtin_ia32_pcmpeqd",
-    "llvm.x86.mmx.pcmpeq.w" => "__builtin_ia32_pcmpeqw",
-    "llvm.x86.mmx.pcmpgt.b" => "__builtin_ia32_pcmpgtb",
-    "llvm.x86.mmx.pcmpgt.d" => "__builtin_ia32_pcmpgtd",
-    "llvm.x86.mmx.pcmpgt.w" => "__builtin_ia32_pcmpgtw",
-    "llvm.x86.mmx.pextr.w" => "__builtin_ia32_vec_ext_v4hi",
-    "llvm.x86.mmx.pinsr.w" => "__builtin_ia32_vec_set_v4hi",
-    "llvm.x86.mmx.pmadd.wd" => "__builtin_ia32_pmaddwd",
-    "llvm.x86.mmx.pmaxs.w" => "__builtin_ia32_pmaxsw",
-    "llvm.x86.mmx.pmaxu.b" => "__builtin_ia32_pmaxub",
-    "llvm.x86.mmx.pmins.w" => "__builtin_ia32_pminsw",
-    "llvm.x86.mmx.pminu.b" => "__builtin_ia32_pminub",
-    "llvm.x86.mmx.pmovmskb" => "__builtin_ia32_pmovmskb",
-    "llvm.x86.mmx.pmulh.w" => "__builtin_ia32_pmulhw",
-    "llvm.x86.mmx.pmulhu.w" => "__builtin_ia32_pmulhuw",
-    "llvm.x86.mmx.pmull.w" => "__builtin_ia32_pmullw",
-    "llvm.x86.mmx.pmulu.dq" => "__builtin_ia32_pmuludq",
-    "llvm.x86.mmx.por" => "__builtin_ia32_por",
-    "llvm.x86.mmx.psad.bw" => "__builtin_ia32_psadbw",
-    "llvm.x86.mmx.psll.d" => "__builtin_ia32_pslld",
-    "llvm.x86.mmx.psll.q" => "__builtin_ia32_psllq",
-    "llvm.x86.mmx.psll.w" => "__builtin_ia32_psllw",
-    "llvm.x86.mmx.pslli.d" => "__builtin_ia32_pslldi",
-    "llvm.x86.mmx.pslli.q" => "__builtin_ia32_psllqi",
-    "llvm.x86.mmx.pslli.w" => "__builtin_ia32_psllwi",
-    "llvm.x86.mmx.psra.d" => "__builtin_ia32_psrad",
-    "llvm.x86.mmx.psra.w" => "__builtin_ia32_psraw",
-    "llvm.x86.mmx.psrai.d" => "__builtin_ia32_psradi",
-    "llvm.x86.mmx.psrai.w" => "__builtin_ia32_psrawi",
-    "llvm.x86.mmx.psrl.d" => "__builtin_ia32_psrld",
-    "llvm.x86.mmx.psrl.q" => "__builtin_ia32_psrlq",
-    "llvm.x86.mmx.psrl.w" => "__builtin_ia32_psrlw",
-    "llvm.x86.mmx.psrli.d" => "__builtin_ia32_psrldi",
-    "llvm.x86.mmx.psrli.q" => "__builtin_ia32_psrlqi",
-    "llvm.x86.mmx.psrli.w" => "__builtin_ia32_psrlwi",
-    "llvm.x86.mmx.psub.b" => "__builtin_ia32_psubb",
-    "llvm.x86.mmx.psub.d" => "__builtin_ia32_psubd",
-    "llvm.x86.mmx.psub.q" => "__builtin_ia32_psubq",
-    "llvm.x86.mmx.psub.w" => "__builtin_ia32_psubw",
-    "llvm.x86.mmx.psubs.b" => "__builtin_ia32_psubsb",
-    "llvm.x86.mmx.psubs.w" => "__builtin_ia32_psubsw",
-    "llvm.x86.mmx.psubus.b" => "__builtin_ia32_psubusb",
-    "llvm.x86.mmx.psubus.w" => "__builtin_ia32_psubusw",
-    "llvm.x86.mmx.punpckhbw" => "__builtin_ia32_punpckhbw",
-    "llvm.x86.mmx.punpckhdq" => "__builtin_ia32_punpckhdq",
-    "llvm.x86.mmx.punpckhwd" => "__builtin_ia32_punpckhwd",
-    "llvm.x86.mmx.punpcklbw" => "__builtin_ia32_punpcklbw",
-    "llvm.x86.mmx.punpckldq" => "__builtin_ia32_punpckldq",
-    "llvm.x86.mmx.punpcklwd" => "__builtin_ia32_punpcklwd",
-    "llvm.x86.mmx.pxor" => "__builtin_ia32_pxor",
     "llvm.x86.monitorx" => "__builtin_ia32_monitorx",
     "llvm.x86.movdir64b" => "__builtin_ia32_movdir64b",
     "llvm.x86.mwaitx" => "__builtin_ia32_mwaitx",
@@ -9193,16 +9319,10 @@ match name {
     "llvm.x86.sse.comile.ss" => "__builtin_ia32_comile",
     "llvm.x86.sse.comilt.ss" => "__builtin_ia32_comilt",
     "llvm.x86.sse.comineq.ss" => "__builtin_ia32_comineq",
-    "llvm.x86.sse.cvtpd2pi" => "__builtin_ia32_cvtpd2pi",
-    "llvm.x86.sse.cvtpi2pd" => "__builtin_ia32_cvtpi2pd",
-    "llvm.x86.sse.cvtpi2ps" => "__builtin_ia32_cvtpi2ps",
-    "llvm.x86.sse.cvtps2pi" => "__builtin_ia32_cvtps2pi",
     "llvm.x86.sse.cvtsi2ss" => "__builtin_ia32_cvtsi2ss",
     "llvm.x86.sse.cvtsi642ss" => "__builtin_ia32_cvtsi642ss",
     "llvm.x86.sse.cvtss2si" => "__builtin_ia32_cvtss2si",
     "llvm.x86.sse.cvtss2si64" => "__builtin_ia32_cvtss2si64",
-    "llvm.x86.sse.cvttpd2pi" => "__builtin_ia32_cvttpd2pi",
-    "llvm.x86.sse.cvttps2pi" => "__builtin_ia32_cvttps2pi",
     "llvm.x86.sse.cvttss2si" => "__builtin_ia32_cvttss2si",
     "llvm.x86.sse.cvttss2si64" => "__builtin_ia32_cvttss2si64",
     "llvm.x86.sse.div.ss" => "__builtin_ia32_divss",
@@ -9212,7 +9332,6 @@ match name {
     "llvm.x86.sse.min.ss" => "__builtin_ia32_minss",
     "llvm.x86.sse.movmsk.ps" => "__builtin_ia32_movmskps",
     "llvm.x86.sse.mul.ss" => "__builtin_ia32_mulss",
-    "llvm.x86.sse.pshuf.w" => "__builtin_ia32_pshufw",
     "llvm.x86.sse.rcp.ps" => "__builtin_ia32_rcpps",
     "llvm.x86.sse.rcp.ss" => "__builtin_ia32_rcpss",
     "llvm.x86.sse.rsqrt.ps" => "__builtin_ia32_rsqrtps",
@@ -9398,35 +9517,20 @@ match name {
     "llvm.x86.sse4a.insertqi" => "__builtin_ia32_insertqi",
     "llvm.x86.sse4a.movnt.sd" => "__builtin_ia32_movntsd",
     "llvm.x86.sse4a.movnt.ss" => "__builtin_ia32_movntss",
-    "llvm.x86.ssse3.pabs.b" => "__builtin_ia32_pabsb",
     "llvm.x86.ssse3.pabs.b.128" => "__builtin_ia32_pabsb128",
-    "llvm.x86.ssse3.pabs.d" => "__builtin_ia32_pabsd",
     "llvm.x86.ssse3.pabs.d.128" => "__builtin_ia32_pabsd128",
-    "llvm.x86.ssse3.pabs.w" => "__builtin_ia32_pabsw",
     "llvm.x86.ssse3.pabs.w.128" => "__builtin_ia32_pabsw128",
-    "llvm.x86.ssse3.phadd.d" => "__builtin_ia32_phaddd",
     "llvm.x86.ssse3.phadd.d.128" => "__builtin_ia32_phaddd128",
-    "llvm.x86.ssse3.phadd.sw" => "__builtin_ia32_phaddsw",
     "llvm.x86.ssse3.phadd.sw.128" => "__builtin_ia32_phaddsw128",
-    "llvm.x86.ssse3.phadd.w" => "__builtin_ia32_phaddw",
     "llvm.x86.ssse3.phadd.w.128" => "__builtin_ia32_phaddw128",
-    "llvm.x86.ssse3.phsub.d" => "__builtin_ia32_phsubd",
     "llvm.x86.ssse3.phsub.d.128" => "__builtin_ia32_phsubd128",
-    "llvm.x86.ssse3.phsub.sw" => "__builtin_ia32_phsubsw",
     "llvm.x86.ssse3.phsub.sw.128" => "__builtin_ia32_phsubsw128",
-    "llvm.x86.ssse3.phsub.w" => "__builtin_ia32_phsubw",
     "llvm.x86.ssse3.phsub.w.128" => "__builtin_ia32_phsubw128",
-    "llvm.x86.ssse3.pmadd.ub.sw" => "__builtin_ia32_pmaddubsw",
     "llvm.x86.ssse3.pmadd.ub.sw.128" => "__builtin_ia32_pmaddubsw128",
-    "llvm.x86.ssse3.pmul.hr.sw" => "__builtin_ia32_pmulhrsw",
     "llvm.x86.ssse3.pmul.hr.sw.128" => "__builtin_ia32_pmulhrsw128",
-    "llvm.x86.ssse3.pshuf.b" => "__builtin_ia32_pshufb",
     "llvm.x86.ssse3.pshuf.b.128" => "__builtin_ia32_pshufb128",
-    "llvm.x86.ssse3.psign.b" => "__builtin_ia32_psignb",
     "llvm.x86.ssse3.psign.b.128" => "__builtin_ia32_psignb128",
-    "llvm.x86.ssse3.psign.d" => "__builtin_ia32_psignd",
     "llvm.x86.ssse3.psign.d.128" => "__builtin_ia32_psignd128",
-    "llvm.x86.ssse3.psign.w" => "__builtin_ia32_psignw",
     "llvm.x86.ssse3.psign.w.128" => "__builtin_ia32_psignw128",
     "llvm.x86.sttilecfg" => "__builtin_ia32_tile_storeconfig",
     "llvm.x86.stui" => "__builtin_ia32_stui",

compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs

@@ -1,11 +1,43 @@
 use std::borrow::Cow;
 
-use gccjit::{Function, FunctionPtrType, RValue, ToRValue, UnaryOp};
+use gccjit::{CType, Context, Function, FunctionPtrType, RValue, ToRValue, UnaryOp};
 use rustc_codegen_ssa::traits::BuilderMethods;
 
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 
+#[cfg_attr(not(feature = "master"), allow(unused_variables))]
+pub fn adjust_function<'gcc>(
+    context: &'gcc Context<'gcc>,
+    func_name: &str,
+    func_ptr: RValue<'gcc>,
+    args: &[RValue<'gcc>],
+) -> RValue<'gcc> {
+    // FIXME: we should not need this hack: this is required because both _mm_fcmadd_sch
+    // and _mm_mask3_fcmadd_round_sch calls llvm.x86.avx512fp16.mask.vfcmadd.csh and we
+    // seem to need to map this one LLVM intrinsic to 2 different GCC builtins.
+    #[cfg(feature = "master")]
+    match func_name {
+        "__builtin_ia32_vfcmaddcsh_mask3_round" => {
+            if format!("{:?}", args[3]).ends_with("255") {
+                return context
+                    .get_target_builtin_function("__builtin_ia32_vfcmaddcsh_mask_round")
+                    .get_address(None);
+            }
+        }
+        "__builtin_ia32_vfmaddcsh_mask3_round" => {
+            if format!("{:?}", args[3]).ends_with("255") {
+                return context
+                    .get_target_builtin_function("__builtin_ia32_vfmaddcsh_mask_round")
+                    .get_address(None);
+            }
+        }
+        _ => (),
+    }
+
+    func_ptr
+}
+
 pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
     builder: &Builder<'a, 'gcc, 'tcx>,
     gcc_func: FunctionPtrType<'gcc>,
@@ -13,6 +45,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
     func_name: &str,
     original_function_name: Option<&String>,
 ) -> Cow<'b, [RValue<'gcc>]> {
+    // TODO: this might not be a good way to workaround the missing tile builtins.
+    if func_name == "__builtin_trap" {
+        return vec![].into();
+    }
+
     // Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
     // arguments here.
     if gcc_func.get_param_count() != args.len() {
@@ -147,7 +184,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
             | "__builtin_ia32_psrav16hi_mask"
             | "__builtin_ia32_psrav8hi_mask"
             | "__builtin_ia32_permvarhi256_mask"
-            | "__builtin_ia32_permvarhi128_mask" => {
+            | "__builtin_ia32_permvarhi128_mask"
+            | "__builtin_ia32_maxph128_mask"
+            | "__builtin_ia32_maxph256_mask"
+            | "__builtin_ia32_minph128_mask"
+            | "__builtin_ia32_minph256_mask" => {
                 let mut new_args = args.to_vec();
                 let arg3_type = gcc_func.get_param_type(2);
                 let vector_type = arg3_type.dyncast_vector().expect("vector type");
@@ -182,7 +223,19 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
             | "__builtin_ia32_vplzcntd_128_mask"
             | "__builtin_ia32_vplzcntq_512_mask"
             | "__builtin_ia32_vplzcntq_256_mask"
-            | "__builtin_ia32_vplzcntq_128_mask" => {
+            | "__builtin_ia32_vplzcntq_128_mask"
+            | "__builtin_ia32_cvtqq2pd128_mask"
+            | "__builtin_ia32_cvtqq2pd256_mask"
+            | "__builtin_ia32_cvtqq2ps256_mask"
+            | "__builtin_ia32_cvtuqq2pd128_mask"
+            | "__builtin_ia32_cvtuqq2pd256_mask"
+            | "__builtin_ia32_cvtuqq2ps256_mask"
+            | "__builtin_ia32_vcvtw2ph128_mask"
+            | "__builtin_ia32_vcvtw2ph256_mask"
+            | "__builtin_ia32_vcvtuw2ph128_mask"
+            | "__builtin_ia32_vcvtuw2ph256_mask"
+            | "__builtin_ia32_vcvtdq2ph256_mask"
+            | "__builtin_ia32_vcvtudq2ph256_mask" => {
                 let mut new_args = args.to_vec();
                 // Remove last arg as it doesn't seem to be used in GCC and is always false.
                 new_args.pop();
@@ -281,7 +334,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
                 new_args.push(last_arg);
                 args = new_args.into();
             }
-            "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
+            "__builtin_ia32_vfmaddsubps512_mask"
+            | "__builtin_ia32_vfmaddsubpd512_mask"
+            | "__builtin_ia32_cmpsh_mask_round"
+            | "__builtin_ia32_vfmaddph512_mask"
+            | "__builtin_ia32_vfmaddsubph512_mask" => {
                 let mut new_args = args.to_vec();
                 let last_arg = new_args.pop().expect("last arg");
                 let arg4_type = gcc_func.get_param_type(3);
@@ -304,9 +361,8 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
             | "__builtin_ia32_vpermi2varpd128_mask"
             | "__builtin_ia32_vpmadd52huq512_mask"
             | "__builtin_ia32_vpmadd52luq512_mask"
-            | "__builtin_ia32_vpmadd52huq256_mask"
-            | "__builtin_ia32_vpmadd52luq256_mask"
-            | "__builtin_ia32_vpmadd52huq128_mask" => {
+            | "__builtin_ia32_vfmaddsubph128_mask"
+            | "__builtin_ia32_vfmaddsubph256_mask" => {
                 let mut new_args = args.to_vec();
                 let arg4_type = gcc_func.get_param_type(3);
                 let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
@@ -355,7 +411,14 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
                 let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
                 args = vec![new_args[1], new_args[0], new_args[2], minus_one].into();
             }
-            "__builtin_ia32_xrstor" | "__builtin_ia32_xsavec" => {
+            "__builtin_ia32_xrstor"
+            | "__builtin_ia32_xrstor64"
+            | "__builtin_ia32_xsavec"
+            | "__builtin_ia32_xsavec64"
+            | "__builtin_ia32_xsave"
+            | "__builtin_ia32_xsave64"
+            | "__builtin_ia32_xsaveopt"
+            | "__builtin_ia32_xsaveopt64" => {
                 let new_args = args.to_vec();
                 let thirty_two = builder.context.new_rvalue_from_int(new_args[1].get_type(), 32);
                 let arg2 = new_args[1] << thirty_two | new_args[2];
@@ -378,11 +441,76 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
                 );
                 args = vec![arg.get_address(None)].into();
             }
+            "__builtin_ia32_cvtqq2pd512_mask"
+            | "__builtin_ia32_cvtqq2ps512_mask"
+            | "__builtin_ia32_cvtuqq2pd512_mask"
+            | "__builtin_ia32_cvtuqq2ps512_mask"
+            | "__builtin_ia32_sqrtph512_mask_round"
+            | "__builtin_ia32_vcvtw2ph512_mask_round"
+            | "__builtin_ia32_vcvtuw2ph512_mask_round"
+            | "__builtin_ia32_vcvtdq2ph512_mask_round"
+            | "__builtin_ia32_vcvtudq2ph512_mask_round"
+            | "__builtin_ia32_vcvtqq2ph512_mask_round"
+            | "__builtin_ia32_vcvtuqq2ph512_mask_round" => {
+                let mut old_args = args.to_vec();
+                let mut new_args = vec![];
+                new_args.push(old_args.swap_remove(0));
+                let arg2_type = gcc_func.get_param_type(1);
+                let vector_type = arg2_type.dyncast_vector().expect("vector type");
+                let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
+                let num_units = vector_type.get_num_units();
+                let first_arg =
+                    builder.context.new_rvalue_from_vector(None, arg2_type, &vec![zero; num_units]);
+                new_args.push(first_arg);
+                let arg3_type = gcc_func.get_param_type(2);
+                let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1);
+                new_args.push(minus_one);
+                new_args.push(old_args.swap_remove(0));
+                args = new_args.into();
+            }
+            "__builtin_ia32_addph512_mask_round"
+            | "__builtin_ia32_subph512_mask_round"
+            | "__builtin_ia32_mulph512_mask_round"
+            | "__builtin_ia32_divph512_mask_round"
+            | "__builtin_ia32_maxph512_mask_round"
+            | "__builtin_ia32_minph512_mask_round" => {
+                let mut new_args = args.to_vec();
+                let last_arg = new_args.pop().expect("last arg");
+
+                let arg3_type = gcc_func.get_param_type(2);
+                let vector_type = arg3_type.dyncast_vector().expect("vector type");
+                let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
+                let num_units = vector_type.get_num_units();
+                let first_arg =
+                    builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units]);
+                new_args.push(first_arg);
+
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+                new_args.push(last_arg);
+                args = new_args.into();
+            }
+            // NOTE: the LLVM intrinsics receive 3 floats, but the GCC builtin requires 3 vectors.
+            "__builtin_ia32_vfmaddsh3_mask" => {
+                let new_args = args.to_vec();
+                let arg1_type = gcc_func.get_param_type(0);
+                let arg2_type = gcc_func.get_param_type(1);
+                let arg3_type = gcc_func.get_param_type(2);
+                let arg4_type = gcc_func.get_param_type(3);
+                let a = builder.context.new_rvalue_from_vector(None, arg1_type, &[new_args[0]; 8]);
+                let b = builder.context.new_rvalue_from_vector(None, arg2_type, &[new_args[1]; 8]);
+                let c = builder.context.new_rvalue_from_vector(None, arg3_type, &[new_args[2]; 8]);
+                let arg4 = builder.context.new_rvalue_from_int(arg4_type, -1);
+                args = vec![a, b, c, arg4, new_args[3]].into();
+            }
             _ => (),
         }
     } else {
         match func_name {
-            "__builtin_ia32_rndscaless_mask_round" | "__builtin_ia32_rndscalesd_mask_round" => {
+            "__builtin_ia32_rndscaless_mask_round"
+            | "__builtin_ia32_rndscalesd_mask_round"
+            | "__builtin_ia32_reducesh_mask_round" => {
                 let new_args = args.to_vec();
                 let arg3_type = gcc_func.get_param_type(2);
                 let arg3 = builder.context.new_cast(None, new_args[4], arg3_type);
@@ -390,7 +518,7 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
                 let arg4 = builder.context.new_bitcast(None, new_args[2], arg4_type);
                 args = vec![new_args[0], new_args[1], arg3, arg4, new_args[3], new_args[5]].into();
             }
-            // NOTE: the LLVM intrinsic receives 3 floats, but the GCC builtin requires 3 vectors.
+            // NOTE: the LLVM intrinsics receive 3 floats, but the GCC builtin requires 3 vectors.
             // FIXME: the intrinsics like _mm_mask_fmadd_sd should probably directly call the GCC
             // intrinsic to avoid this.
             "__builtin_ia32_vfmaddss3_round" => {
@@ -473,6 +601,52 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
                 let new_args = args.to_vec();
                 args = vec![new_args[1], new_args[0], new_args[2]].into();
             }
+            "__builtin_ia32_rangesd128_mask_round"
+            | "__builtin_ia32_rangess128_mask_round"
+            | "__builtin_ia32_reducesd_mask_round"
+            | "__builtin_ia32_reducess_mask_round" => {
+                let new_args = args.to_vec();
+                args = vec![
+                    new_args[0],
+                    new_args[1],
+                    new_args[4],
+                    new_args[2],
+                    new_args[3],
+                    new_args[5],
+                ]
+                .into();
+            }
+            "__builtin_ia32_rndscalesh_mask_round" => {
+                let new_args = args.to_vec();
+                args = vec![
+                    new_args[0],
+                    new_args[1],
+                    new_args[4],
+                    new_args[2],
+                    new_args[3],
+                    new_args[5],
+                ]
+                .into();
+            }
+            "fma" => {
+                let mut new_args = args.to_vec();
+                new_args[0] = builder.context.new_cast(None, new_args[0], builder.double_type);
+                new_args[1] = builder.context.new_cast(None, new_args[1], builder.double_type);
+                new_args[2] = builder.context.new_cast(None, new_args[2], builder.double_type);
+                args = new_args.into();
+            }
+            "__builtin_ia32_sqrtsh_mask_round"
+            | "__builtin_ia32_vcvtss2sh_mask_round"
+            | "__builtin_ia32_vcvtsd2sh_mask_round"
+            | "__builtin_ia32_vcvtsh2ss_mask_round"
+            | "__builtin_ia32_vcvtsh2sd_mask_round"
+            | "__builtin_ia32_rcpsh_mask"
+            | "__builtin_ia32_rsqrtsh_mask" => {
+                // The first two arguments are inverted, so swap them.
+                let mut new_args = args.to_vec();
+                new_args.swap(0, 1);
+                args = new_args.into();
+            }
             _ => (),
         }
     }
@@ -489,7 +663,9 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
     orig_args: &[RValue<'gcc>],
 ) -> RValue<'gcc> {
     match func_name {
-        "__builtin_ia32_vfmaddss3_round" | "__builtin_ia32_vfmaddsd3_round" => {
+        "__builtin_ia32_vfmaddss3_round"
+        | "__builtin_ia32_vfmaddsd3_round"
+        | "__builtin_ia32_vfmaddsh3_mask" => {
             #[cfg(feature = "master")]
             {
                 let zero = builder.context.new_rvalue_zero(builder.int_type);
@@ -546,6 +722,10 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
                     success_variable.to_rvalue(),
                 ]);
         }
+        "fma" => {
+            let f16_type = builder.context.new_c_type(CType::Float16);
+            return_value = builder.context.new_cast(None, return_value, f16_type);
+        }
         _ => (),
     }
 
@@ -779,7 +959,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
         "llvm.x86.avx512.mask.cmp.b.256" => "__builtin_ia32_cmpb256_mask",
         "llvm.x86.avx512.mask.cmp.b.128" => "__builtin_ia32_cmpb128_mask",
         "llvm.x86.xrstor" => "__builtin_ia32_xrstor",
+        "llvm.x86.xrstor64" => "__builtin_ia32_xrstor64",
         "llvm.x86.xsavec" => "__builtin_ia32_xsavec",
+        "llvm.x86.xsavec64" => "__builtin_ia32_xsavec64",
         "llvm.x86.addcarry.32" => "__builtin_ia32_addcarryx_u32",
         "llvm.x86.subborrow.32" => "__builtin_ia32_sbb_u32",
         "llvm.x86.avx512.mask.compress.store.w.512" => "__builtin_ia32_compressstoreuhi512_mask",
@@ -968,9 +1150,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
         "llvm.x86.avx512.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128_mask",
         "llvm.x86.avx512.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_mask",
         "llvm.x86.avx512.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_mask",
-        "llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_mask",
-        "llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_mask",
-        "llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_mask",
+        "llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256",
+        "llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256",
+        "llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128",
         "llvm.x86.avx512.vpdpwssd.512" => "__builtin_ia32_vpdpwssd_v16si",
         "llvm.x86.avx512.vpdpwssd.256" => "__builtin_ia32_vpdpwssd_v8si",
         "llvm.x86.avx512.vpdpwssd.128" => "__builtin_ia32_vpdpwssd_v4si",
@@ -983,6 +1165,180 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
         "llvm.x86.avx512.vpdpbusds.512" => "__builtin_ia32_vpdpbusds_v16si",
         "llvm.x86.avx512.vpdpbusds.256" => "__builtin_ia32_vpdpbusds_v8si",
         "llvm.x86.avx512.vpdpbusds.128" => "__builtin_ia32_vpdpbusds_v4si",
+        "llvm.x86.xsave" => "__builtin_ia32_xsave",
+        "llvm.x86.xsave64" => "__builtin_ia32_xsave64",
+        "llvm.x86.xsaveopt" => "__builtin_ia32_xsaveopt",
+        "llvm.x86.xsaveopt64" => "__builtin_ia32_xsaveopt64",
+        "llvm.x86.avx512.mask.loadu.w.512" => "__builtin_ia32_loaddquhi512_mask",
+        "llvm.x86.avx512.mask.loadu.b.512" => "__builtin_ia32_loaddquqi512_mask",
+        "llvm.x86.avx512.mask.loadu.w.256" => "__builtin_ia32_loaddquhi256_mask",
+        "llvm.x86.avx512.mask.loadu.b.256" => "__builtin_ia32_loaddquqi256_mask",
+        "llvm.x86.avx512.mask.loadu.w.128" => "__builtin_ia32_loaddquhi128_mask",
+        "llvm.x86.avx512.mask.loadu.b.128" => "__builtin_ia32_loaddquqi128_mask",
+        "llvm.x86.avx512.mask.storeu.w.512" => "__builtin_ia32_storedquhi512_mask",
+        "llvm.x86.avx512.mask.storeu.b.512" => "__builtin_ia32_storedquqi512_mask",
+        "llvm.x86.avx512.mask.storeu.w.256" => "__builtin_ia32_storedquhi256_mask",
+        "llvm.x86.avx512.mask.storeu.b.256" => "__builtin_ia32_storedquqi256_mask",
+        "llvm.x86.avx512.mask.storeu.w.128" => "__builtin_ia32_storedquhi128_mask",
+        "llvm.x86.avx512.mask.storeu.b.128" => "__builtin_ia32_storedquqi128_mask",
+        "llvm.x86.avx512.mask.expand.load.w.512" => "__builtin_ia32_expandloadhi512_mask",
+        "llvm.x86.avx512.mask.expand.load.w.256" => "__builtin_ia32_expandloadhi256_mask",
+        "llvm.x86.avx512.mask.expand.load.w.128" => "__builtin_ia32_expandloadhi128_mask",
+        "llvm.x86.avx512.mask.expand.load.b.512" => "__builtin_ia32_expandloadqi512_mask",
+        "llvm.x86.avx512.mask.expand.load.b.256" => "__builtin_ia32_expandloadqi256_mask",
+        "llvm.x86.avx512.mask.expand.load.b.128" => "__builtin_ia32_expandloadqi128_mask",
+        "llvm.x86.avx512.sitofp.round.v8f64.v8i64" => "__builtin_ia32_cvtqq2pd512_mask",
+        "llvm.x86.avx512.sitofp.round.v2f64.v2i64" => "__builtin_ia32_cvtqq2pd128_mask",
+        "llvm.x86.avx512.sitofp.round.v4f64.v4i64" => "__builtin_ia32_cvtqq2pd256_mask",
+        "llvm.x86.avx512.sitofp.round.v8f32.v8i64" => "__builtin_ia32_cvtqq2ps512_mask",
+        "llvm.x86.avx512.sitofp.round.v4f32.v4i64" => "__builtin_ia32_cvtqq2ps256_mask",
+        "llvm.x86.avx512.uitofp.round.v8f64.v8u64" => "__builtin_ia32_cvtuqq2pd512_mask",
+        "llvm.x86.avx512.uitofp.round.v2f64.v2u64" => "__builtin_ia32_cvtuqq2pd128_mask",
+        "llvm.x86.avx512.uitofp.round.v4f64.v4u64" => "__builtin_ia32_cvtuqq2pd256_mask",
+        "llvm.x86.avx512.uitofp.round.v8f32.v8u64" => "__builtin_ia32_cvtuqq2ps512_mask",
+        "llvm.x86.avx512.uitofp.round.v4f32.v4u64" => "__builtin_ia32_cvtuqq2ps256_mask",
+        "llvm.x86.avx512.mask.reduce.pd.512" => "__builtin_ia32_reducepd512_mask_round",
+        "llvm.x86.avx512.mask.reduce.ps.512" => "__builtin_ia32_reduceps512_mask_round",
+        "llvm.x86.avx512.mask.reduce.sd" => "__builtin_ia32_reducesd_mask_round",
+        "llvm.x86.avx512.mask.reduce.ss" => "__builtin_ia32_reducess_mask_round",
+        "llvm.x86.avx512.mask.loadu.d.256" => "__builtin_ia32_loaddqusi256_mask",
+        "llvm.x86.avx512.mask.loadu.q.256" => "__builtin_ia32_loaddqudi256_mask",
+        "llvm.x86.avx512.mask.loadu.ps.256" => "__builtin_ia32_loadups256_mask",
+        "llvm.x86.avx512.mask.loadu.pd.256" => "__builtin_ia32_loadupd256_mask",
+        "llvm.x86.avx512.mask.loadu.d.128" => "__builtin_ia32_loaddqusi128_mask",
+        "llvm.x86.avx512.mask.loadu.q.128" => "__builtin_ia32_loaddqudi128_mask",
+        "llvm.x86.avx512.mask.loadu.ps.128" => "__builtin_ia32_loadups128_mask",
+        "llvm.x86.avx512.mask.loadu.pd.128" => "__builtin_ia32_loadupd128_mask",
+        "llvm.x86.avx512.mask.load.d.512" => "__builtin_ia32_movdqa32load512_mask",
+        "llvm.x86.avx512.mask.load.q.512" => "__builtin_ia32_movdqa64load512_mask",
+        "llvm.x86.avx512.mask.load.ps.512" => "__builtin_ia32_loadaps512_mask",
+        "llvm.x86.avx512.mask.load.pd.512" => "__builtin_ia32_loadapd512_mask",
+        "llvm.x86.avx512.mask.load.d.256" => "__builtin_ia32_movdqa32load256_mask",
+        "llvm.x86.avx512.mask.load.q.256" => "__builtin_ia32_movdqa64load256_mask",
+        "llvm.x86.avx512fp16.mask.cmp.sh" => "__builtin_ia32_cmpsh_mask_round",
+        "llvm.x86.avx512fp16.vcomi.sh" => "__builtin_ia32_cmpsh_mask_round",
+        "llvm.x86.avx512fp16.add.ph.512" => "__builtin_ia32_addph512_mask_round",
+        "llvm.x86.avx512fp16.sub.ph.512" => "__builtin_ia32_subph512_mask_round",
+        "llvm.x86.avx512fp16.mul.ph.512" => "__builtin_ia32_mulph512_mask_round",
+        "llvm.x86.avx512fp16.div.ph.512" => "__builtin_ia32_divph512_mask_round",
+        "llvm.x86.avx512fp16.mask.vfmul.cph.512" => "__builtin_ia32_vfmulcph512_mask_round",
+        "llvm.x86.avx512fp16.mask.vfmul.csh" => "__builtin_ia32_vfmulcsh_mask_round",
+        "llvm.x86.avx512fp16.mask.vfcmul.cph.512" => "__builtin_ia32_vfcmulcph512_mask_round",
+        "llvm.x86.avx512fp16.mask.vfcmul.csh" => "__builtin_ia32_vfcmulcsh_mask_round",
+        "llvm.x86.avx512fp16.mask.vfmadd.cph.512" => "__builtin_ia32_vfmaddcph512_mask3_round",
+        "llvm.x86.avx512fp16.maskz.vfmadd.cph.512" => "__builtin_ia32_vfmaddcph512_maskz_round",
+        "llvm.x86.avx512fp16.mask.vfmadd.csh" => "__builtin_ia32_vfmaddcsh_mask3_round",
+        "llvm.x86.avx512fp16.maskz.vfmadd.csh" => "__builtin_ia32_vfmaddcsh_maskz_round",
+        "llvm.x86.avx512fp16.mask.vfcmadd.cph.512" => "__builtin_ia32_vfcmaddcph512_mask3_round",
+        "llvm.x86.avx512fp16.maskz.vfcmadd.cph.512" => "__builtin_ia32_vfcmaddcph512_maskz_round",
+        "llvm.x86.avx512fp16.mask.vfcmadd.csh" => "__builtin_ia32_vfcmaddcsh_mask3_round",
+        "llvm.x86.avx512fp16.maskz.vfcmadd.csh" => "__builtin_ia32_vfcmaddcsh_maskz_round",
+        "llvm.x86.avx512fp16.vfmadd.ph.512" => "__builtin_ia32_vfmaddph512_mask",
+        "llvm.x86.avx512fp16.vcvtsi642sh" => "__builtin_ia32_vcvtsi2sh64_round",
+        "llvm.x86.avx512fp16.vcvtusi642sh" => "__builtin_ia32_vcvtusi2sh64_round",
+        "llvm.x86.avx512fp16.vcvtsh2si64" => "__builtin_ia32_vcvtsh2si64_round",
+        "llvm.x86.avx512fp16.vcvtsh2usi64" => "__builtin_ia32_vcvtsh2usi64_round",
+        "llvm.x86.avx512fp16.vcvttsh2si64" => "__builtin_ia32_vcvttsh2si64_round",
+        "llvm.x86.avx512fp16.vcvttsh2usi64" => "__builtin_ia32_vcvttsh2usi64_round",
+        "llvm.x86.avx512.mask.load.ps.256" => "__builtin_ia32_loadaps256_mask",
+        "llvm.x86.avx512.mask.load.pd.256" => "__builtin_ia32_loadapd256_mask",
+        "llvm.x86.avx512.mask.load.d.128" => "__builtin_ia32_movdqa32load128_mask",
+        "llvm.x86.avx512.mask.load.q.128" => "__builtin_ia32_movdqa64load128_mask",
+        "llvm.x86.avx512.mask.load.ps.128" => "__builtin_ia32_loadaps128_mask",
+        "llvm.x86.avx512.mask.load.pd.128" => "__builtin_ia32_loadapd128_mask",
+        "llvm.x86.avx512.mask.storeu.d.256" => "__builtin_ia32_storedqusi256_mask",
+        "llvm.x86.avx512.mask.storeu.q.256" => "__builtin_ia32_storedqudi256_mask",
+        "llvm.x86.avx512.mask.storeu.ps.256" => "__builtin_ia32_storeups256_mask",
+        "llvm.x86.avx512.mask.storeu.pd.256" => "__builtin_ia32_storeupd256_mask",
+        "llvm.x86.avx512.mask.storeu.d.128" => "__builtin_ia32_storedqusi128_mask",
+        "llvm.x86.avx512.mask.storeu.q.128" => "__builtin_ia32_storedqudi128_mask",
+        "llvm.x86.avx512.mask.storeu.ps.128" => "__builtin_ia32_storeups128_mask",
+        "llvm.x86.avx512.mask.storeu.pd.128" => "__builtin_ia32_storeupd128_mask",
+        "llvm.x86.avx512.mask.store.d.512" => "__builtin_ia32_movdqa32store512_mask",
+        "llvm.x86.avx512.mask.store.q.512" => "__builtin_ia32_movdqa64store512_mask",
+        "llvm.x86.avx512.mask.store.ps.512" => "__builtin_ia32_storeaps512_mask",
+        "llvm.x86.avx512.mask.store.pd.512" => "__builtin_ia32_storeapd512_mask",
+        "llvm.x86.avx512.mask.store.d.256" => "__builtin_ia32_movdqa32store256_mask",
+        "llvm.x86.avx512.mask.store.q.256" => "__builtin_ia32_movdqa64store256_mask",
+        "llvm.x86.avx512.mask.store.ps.256" => "__builtin_ia32_storeaps256_mask",
+        "llvm.x86.avx512.mask.store.pd.256" => "__builtin_ia32_storeapd256_mask",
+        "llvm.x86.avx512.mask.store.d.128" => "__builtin_ia32_movdqa32store128_mask",
+        "llvm.x86.avx512.mask.store.q.128" => "__builtin_ia32_movdqa64store128_mask",
+        "llvm.x86.avx512.mask.store.ps.128" => "__builtin_ia32_storeaps128_mask",
+        "llvm.x86.avx512.mask.store.pd.128" => "__builtin_ia32_storeapd128_mask",
+        "llvm.x86.avx512fp16.vfmadd.f16" => "__builtin_ia32_vfmaddsh3_mask",
+        "llvm.x86.avx512fp16.vfmaddsub.ph.128" => "__builtin_ia32_vfmaddsubph128_mask",
+        "llvm.x86.avx512fp16.vfmaddsub.ph.256" => "__builtin_ia32_vfmaddsubph256_mask",
+        "llvm.x86.avx512fp16.vfmaddsub.ph.512" => "__builtin_ia32_vfmaddsubph512_mask",
+        "llvm.x86.avx512fp16.sqrt.ph.512" => "__builtin_ia32_sqrtph512_mask_round",
+        "llvm.x86.avx512fp16.mask.sqrt.sh" => "__builtin_ia32_sqrtsh_mask_round",
+        "llvm.x86.avx512fp16.max.ph.128" => "__builtin_ia32_maxph128_mask",
+        "llvm.x86.avx512fp16.max.ph.256" => "__builtin_ia32_maxph256_mask",
+        "llvm.x86.avx512fp16.max.ph.512" => "__builtin_ia32_maxph512_mask_round",
+        "llvm.x86.avx512fp16.min.ph.128" => "__builtin_ia32_minph128_mask",
+        "llvm.x86.avx512fp16.min.ph.256" => "__builtin_ia32_minph256_mask",
+        "llvm.x86.avx512fp16.min.ph.512" => "__builtin_ia32_minph512_mask_round",
+        "llvm.x86.avx512fp16.mask.getexp.sh" => "__builtin_ia32_getexpsh_mask_round",
+        "llvm.x86.avx512fp16.mask.rndscale.ph.128" => "__builtin_ia32_rndscaleph128_mask",
+        "llvm.x86.avx512fp16.mask.rndscale.ph.256" => "__builtin_ia32_rndscaleph256_mask",
+        "llvm.x86.avx512fp16.mask.rndscale.ph.512" => "__builtin_ia32_rndscaleph512_mask_round",
+        "llvm.x86.avx512fp16.mask.scalef.ph.512" => "__builtin_ia32_scalefph512_mask_round",
+        "llvm.x86.avx512fp16.mask.reduce.ph.512" => "__builtin_ia32_reduceph512_mask_round",
+        "llvm.x86.avx512fp16.mask.reduce.sh" => "__builtin_ia32_reducesh_mask_round",
+        "llvm.x86.avx512.sitofp.round.v8f16.v8i16" => "__builtin_ia32_vcvtw2ph128_mask",
+        "llvm.x86.avx512.sitofp.round.v16f16.v16i16" => "__builtin_ia32_vcvtw2ph256_mask",
+        "llvm.x86.avx512.sitofp.round.v32f16.v32i16" => "__builtin_ia32_vcvtw2ph512_mask_round",
+        "llvm.x86.avx512.uitofp.round.v8f16.v8u16" => "__builtin_ia32_vcvtuw2ph128_mask",
+        "llvm.x86.avx512.uitofp.round.v16f16.v16u16" => "__builtin_ia32_vcvtuw2ph256_mask",
+        "llvm.x86.avx512.uitofp.round.v32f16.v32u16" => "__builtin_ia32_vcvtuw2ph512_mask_round",
+        "llvm.x86.avx512.sitofp.round.v8f16.v8i32" => "__builtin_ia32_vcvtdq2ph256_mask",
+        "llvm.x86.avx512.sitofp.round.v16f16.v16i32" => "__builtin_ia32_vcvtdq2ph512_mask_round",
+        "llvm.x86.avx512fp16.vcvtsi2sh" => "__builtin_ia32_vcvtsi2sh32_round",
+        "llvm.x86.avx512.uitofp.round.v8f16.v8u32" => "__builtin_ia32_vcvtudq2ph256_mask",
+        "llvm.x86.avx512.uitofp.round.v16f16.v16u32" => "__builtin_ia32_vcvtudq2ph512_mask_round",
+        "llvm.x86.avx512fp16.vcvtusi2sh" => "__builtin_ia32_vcvtusi2sh32_round",
+        "llvm.x86.avx512.sitofp.round.v8f16.v8i64" => "__builtin_ia32_vcvtqq2ph512_mask_round",
+        "llvm.x86.avx512.uitofp.round.v8f16.v8u64" => "__builtin_ia32_vcvtuqq2ph512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtps2phx.512" => "__builtin_ia32_vcvtps2phx512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtpd2ph.512" => "__builtin_ia32_vcvtpd2ph512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2uw.512" => "__builtin_ia32_vcvtph2uw512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvttph2w.512" => "__builtin_ia32_vcvttph2w512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvttph2uw.512" => "__builtin_ia32_vcvttph2uw512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2dq.512" => "__builtin_ia32_vcvtph2dq512_mask_round",
+        "llvm.x86.avx512fp16.vcvtsh2si32" => "__builtin_ia32_vcvtsh2si32_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2udq.512" => "__builtin_ia32_vcvtph2udq512_mask_round",
+        "llvm.x86.avx512fp16.vcvtsh2usi32" => "__builtin_ia32_vcvtsh2usi32_round",
+        "llvm.x86.avx512fp16.mask.vcvttph2dq.512" => "__builtin_ia32_vcvttph2dq512_mask_round",
+        "llvm.x86.avx512fp16.vcvttsh2si32" => "__builtin_ia32_vcvttsh2si32_round",
+        "llvm.x86.avx512fp16.mask.vcvttph2udq.512" => "__builtin_ia32_vcvttph2udq512_mask_round",
+        "llvm.x86.avx512fp16.vcvttsh2usi32" => "__builtin_ia32_vcvttsh2usi32_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2qq.512" => "__builtin_ia32_vcvtph2qq512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2uqq.512" => "__builtin_ia32_vcvtph2uqq512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvttph2qq.512" => "__builtin_ia32_vcvttph2qq512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvttph2uqq.512" => "__builtin_ia32_vcvttph2uqq512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2psx.512" => "__builtin_ia32_vcvtph2psx512_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtph2pd.512" => "__builtin_ia32_vcvtph2pd512_mask_round",
+        "llvm.x86.avx512fp16.mask.vfcmadd.cph.256" => "__builtin_ia32_vfcmaddcph256_mask3",
+        "llvm.x86.avx512fp16.mask.vfmadd.cph.256" => "__builtin_ia32_vfmaddcph256_mask3",
+        "llvm.x86.avx512fp16.mask.vfcmadd.cph.128" => "__builtin_ia32_vfcmaddcph128_mask3",
+        "llvm.x86.avx512fp16.mask.vfmadd.cph.128" => "__builtin_ia32_vfmaddcph128_mask3",
+
+        // TODO: support the tile builtins:
+        "llvm.x86.ldtilecfg" => "__builtin_trap",
+        "llvm.x86.sttilecfg" => "__builtin_trap",
+        "llvm.x86.tileloadd64" => "__builtin_trap",
+        "llvm.x86.tilerelease" => "__builtin_trap",
+        "llvm.x86.tilestored64" => "__builtin_trap",
+        "llvm.x86.tileloaddt164" => "__builtin_trap",
+        "llvm.x86.tilezero" => "__builtin_trap",
+        "llvm.x86.tdpbf16ps" => "__builtin_trap",
+        "llvm.x86.tdpbssd" => "__builtin_trap",
+        "llvm.x86.tdpbsud" => "__builtin_trap",
+        "llvm.x86.tdpbusd" => "__builtin_trap",
+        "llvm.x86.tdpbuud" => "__builtin_trap",
+        "llvm.x86.tdpfp16ps" => "__builtin_trap",
+        "llvm.x86.tcmmimfp16ps" => "__builtin_trap",
+        "llvm.x86.tcmmrlfp16ps" => "__builtin_trap",
 
         // NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py
         _ => include!("archs.rs"),

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -127,20 +127,13 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
         // https://github.com/rust-lang/rust-clippy/issues/12497
         // and leave `else if use_integer_compare` to be placed "as is".
         #[allow(clippy::suspicious_else_formatting)]
-        let llval = match name {
+        let value = match name {
             _ if simple.is_some() => {
-                // FIXME(antoyo): remove this cast when the API supports function.
-                let func = unsafe {
-                    std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(simple.expect("simple"))
-                };
-                self.call(
-                    self.type_void(),
-                    None,
-                    None,
+                let func = simple.expect("simple function");
+                self.cx.context.new_call(
+                    self.location,
                     func,
                     &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
-                    None,
-                    None,
                 )
             }
             sym::likely => self.expect(args[0].immediate(), true),
@@ -383,7 +376,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
 
             _ if name_str.starts_with("simd_") => {
                 match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
-                    Ok(llval) => llval,
+                    Ok(value) => value,
                     Err(()) => return Ok(()),
                 }
             }
@@ -396,9 +389,9 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode {
                 let ptr_llty = self.type_ptr_to(ty.gcc_type(self));
                 let ptr = self.pointercast(result.val.llval, ptr_llty);
-                self.store(llval, ptr, result.val.align);
+                self.store(value, ptr, result.val.align);
             } else {
-                OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
+                OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
                     .val
                     .store(self, result);
             }

compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

@@ -198,7 +198,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         bx.context.new_bitcast(None, shuffled, v_type)
     };
 
-    if name == sym::simd_bswap || name == sym::simd_bitreverse {
+    if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop) {
         require!(
             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
             InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
@@ -209,6 +209,22 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         return Ok(simd_bswap(bx, args[0].immediate()));
     }
 
+    let simd_ctpop = |bx: &mut Builder<'a, 'gcc, 'tcx>, vector: RValue<'gcc>| -> RValue<'gcc> {
+        let mut vector_elements = vec![];
+        let elem_ty = bx.element_type(llret_ty);
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
+            let element = bx.extract_element(vector, index).to_rvalue();
+            let result = bx.context.new_cast(None, bx.pop_count(element), elem_ty);
+            vector_elements.push(result);
+        }
+        bx.context.new_rvalue_from_vector(None, llret_ty, &vector_elements)
+    };
+
+    if name == sym::simd_ctpop {
+        return Ok(simd_ctpop(bx, args[0].immediate()));
+    }
+
     // We use a different algorithm from non-vector bitreverse to take advantage of most
     // processors' vector shuffle units.  It works like this:
     // 1. Generate pre-reversed low and high nibbles as a vector.
@@ -462,7 +478,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
                 });
-                require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer {
+                require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer {
                     span,
                     name,
                     ty: in_elem
@@ -477,7 +493,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
                 });
-                require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer {
+                require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer {
                     span,
                     name,
                     ty: out_elem
@@ -718,11 +734,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 return Err(());
             }};
         }
-        let (elem_ty_str, elem_ty) = if let ty::Float(ref f) = *in_elem.kind() {
+        let (elem_ty_str, elem_ty, cast_type) = if let ty::Float(ref f) = *in_elem.kind() {
             let elem_ty = bx.cx.type_float_from_ty(*f);
             match f.bit_width() {
-                32 => ("f", elem_ty),
-                64 => ("", elem_ty),
+                16 => ("", elem_ty, Some(bx.cx.double_type)),
+                32 => ("f", elem_ty, None),
+                64 => ("", elem_ty, None),
                 _ => {
                     return_error!(InvalidMonomorphization::FloatingPointVector {
                         span,
@@ -758,10 +775,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
             _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
         };
         let builtin_name = format!("{}{}", intr_name, elem_ty_str);
-        let funcs = bx.cx.functions.borrow();
-        let function = funcs
-            .get(&builtin_name)
-            .unwrap_or_else(|| panic!("unable to find builtin function {}", builtin_name));
+        let function = bx.context.get_builtin_function(builtin_name);
 
         // TODO(antoyo): add platform-specific behavior here for architectures that have these
         // intrinsics as instructions (for instance, gpus)
@@ -769,17 +783,28 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         for i in 0..in_len {
             let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
             // we have to treat fpowi specially, since fpowi's second argument is always an i32
-            let arguments = if name == sym::simd_fpowi {
-                vec![
+            let mut arguments = vec![];
+            if name == sym::simd_fpowi {
+                arguments = vec![
                     bx.extract_element(args[0].immediate(), index).to_rvalue(),
                     args[1].immediate(),
-                ]
+                ];
             } else {
-                args.iter()
-                    .map(|arg| bx.extract_element(arg.immediate(), index).to_rvalue())
-                    .collect()
+                for arg in args {
+                    let mut element = bx.extract_element(arg.immediate(), index).to_rvalue();
+                    // FIXME: it would probably be better to not have casts here and use the proper
+                    // instructions.
+                    if let Some(typ) = cast_type {
+                        element = bx.context.new_cast(None, element, typ);
+                    }
+                    arguments.push(element);
+                }
             };
-            vector_elements.push(bx.context.new_call(None, *function, &arguments));
+            let mut result = bx.context.new_call(None, function, &arguments);
+            if cast_type.is_some() {
+                result = bx.context.new_cast(None, result, elem_ty);
+            }
+            vector_elements.push(result);
         }
         let c = bx.context.new_rvalue_from_vector(None, vec_ty, &vector_elements);
         Ok(c)

compiler/rustc_codegen_gcc/src/lib.rs

@@ -363,7 +363,7 @@ impl Deref for SyncContext {
 
 unsafe impl Send for SyncContext {}
 // FIXME(antoyo): that shouldn't be Sync. Parallel compilation is currently disabled with "-Zno-parallel-llvm".
-// TODO: disable it here by returing false in CodegenBackend::supports_parallel().
+// TODO: disable it here by returning false in CodegenBackend::supports_parallel().
 unsafe impl Sync for SyncContext {}
 
 impl WriteBackendMethods for GccCodegenBackend {

compiler/rustc_codegen_gcc/src/mono_item.rs

@@ -37,7 +37,7 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
         let global = self.define_global(symbol_name, gcc_type, is_tls, attrs.link_section);
         #[cfg(feature = "master")]
-        global.add_string_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
+        global.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
 
         // TODO(antoyo): set linkage.
         self.instances.borrow_mut().insert(instance, global);

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -207,7 +207,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
         // layout.
         if let Abi::Scalar(ref scalar) = self.abi {
             // Use a different cache for scalars because pointers to DSTs
-            // can be either fat or thin (data pointers of fat pointers).
+            // can be either wide or thin (data pointers of wide pointers).
             if let Some(&ty) = cx.scalar_types.borrow().get(&self.ty) {
                 return ty;
             }

compiler/rustc_codegen_gcc/tests/failing-ice-tests.txt

@@ -34,3 +34,7 @@ tests/ui/sepcomp/sepcomp-unwind.rs
 tests/ui/extern/issue-64655-extern-rust-must-allow-unwind.rs
 tests/ui/extern/issue-64655-allow-unwind-when-calling-panic-directly.rs
 tests/ui/unwind-no-uwtable.rs
+tests/ui/delegation/fn-header.rs
+tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
+tests/ui/simd/masked-load-store.rs
+tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs

compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt

@@ -95,3 +95,29 @@ tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
 tests/ui/backtrace/backtrace.rs
 tests/ui/lifetimes/tail-expr-lock-poisoning.rs
 tests/ui/runtime/rt-explody-panic-payloads.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/function.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/basic.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/inline1.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/print.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/inline2.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/print3.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/segfault.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/function.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/basic.rs
+tests/ui/codegen/equal-pointers-unequal/as-cast/zero.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline1.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/print.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/print3.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline2.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/segfault.rs
+tests/ui/codegen/equal-pointers-unequal/exposed-provenance/zero.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/basic.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/function.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/print.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline1.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/print3.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline2.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/segfault.rs
+tests/ui/codegen/equal-pointers-unequal/strict-provenance/zero.rs
+tests/ui/sanitizer/cfi-sized-associated-ty.rs
+tests/ui/sanitizer/cfi-can-reveal-opaques.rs

compiler/rustc_codegen_gcc/tests/run/array.rs

@@ -31,6 +31,7 @@ impl Copy for i32 {}
 impl Copy for u8 {}
 impl Copy for i8 {}
 impl Copy for i16 {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tests/run/closure.rs

@@ -33,6 +33,7 @@ impl Copy for i32 {}
 impl Copy for u32 {}
 impl Copy for u8 {}
 impl Copy for i8 {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tests/run/condition.rs

@@ -34,6 +34,7 @@ impl Copy for i16 {}
 impl Copy for char {}
 impl Copy for i8 {}
 impl Copy for u8 {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tests/run/fun_ptr.rs

@@ -28,6 +28,7 @@ impl Copy for i32 {}
 impl Copy for u8 {}
 impl Copy for i8 {}
 impl Copy for i16 {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tests/run/ptr_cast.rs

@@ -28,6 +28,7 @@ impl Copy for i32 {}
 impl Copy for u8 {}
 impl Copy for i8 {}
 impl Copy for i16 {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tests/run/slice.rs

@@ -26,6 +26,7 @@ impl Copy for isize {}
 impl Copy for usize {}
 impl Copy for i32 {}
 impl Copy for u32 {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tests/run/static.rs

@@ -34,6 +34,7 @@ trait Copy {
 }
 
 impl Copy for isize {}
+impl<T: ?Sized> Copy for *mut T {}
 
 #[lang = "receiver"]
 trait Receiver {

compiler/rustc_codegen_gcc/tools/generate_intrinsics.py

@@ -45,7 +45,7 @@ def convert_to_string(content):
     return content
 
 
-def extract_instrinsics_from_llvm(llvm_path, intrinsics):
+def extract_intrinsics_from_llvm(llvm_path, intrinsics):
     command = ["llvm-tblgen", "llvm/IR/Intrinsics.td"]
     cwd = os.path.join(llvm_path, "llvm/include")
     print("=> Running command `{}` from `{}`".format(command, cwd))
@@ -88,7 +88,7 @@ def append_translation(json_data, p, array):
     append_intrinsic(array, content[1], content[3])
 
 
-def extract_instrinsics_from_llvmint(llvmint, intrinsics):
+def extract_intrinsics_from_llvmint(llvmint, intrinsics):
     archs = [
         "AMDGPU",
         "aarch64",
@@ -152,9 +152,9 @@ def update_intrinsics(llvm_path, llvmint, llvmint2):
     intrinsics_llvmint = {}
     all_intrinsics = {}
 
-    extract_instrinsics_from_llvm(llvm_path, intrinsics_llvm)
-    extract_instrinsics_from_llvmint(llvmint, intrinsics_llvmint)
-    extract_instrinsics_from_llvmint(llvmint2, intrinsics_llvmint)
+    extract_intrinsics_from_llvm(llvm_path, intrinsics_llvm)
+    extract_intrinsics_from_llvmint(llvmint, intrinsics_llvmint)
+    extract_intrinsics_from_llvmint(llvmint2, intrinsics_llvmint)
 
     intrinsics = {}
     # We give priority to translations from LLVM over the ones from llvmint.

compiler/rustc_codegen_llvm/src/abi.rs

@@ -7,7 +7,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
 use rustc_codegen_ssa::traits::*;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::LayoutOf;
-pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
+pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA};
 use rustc_middle::{bug, ty};
 use rustc_session::config;
 pub(crate) use rustc_target::abi::call::*;

compiler/rustc_codegen_llvm/src/allocator.rs

@@ -77,18 +77,20 @@ pub(crate) unsafe fn codegen(
         // __rust_alloc_error_handler_should_panic
         let name = OomStrategy::SYMBOL;
         let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8);
-        if tcx.sess.default_hidden_visibility() {
-            llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden);
-        }
+        llvm::LLVMRustSetVisibility(
+            ll_g,
+            llvm::Visibility::from_generic(tcx.sess.default_visibility()),
+        );
         let val = tcx.sess.opts.unstable_opts.oom.should_panic();
         let llval = llvm::LLVMConstInt(i8, val as u64, False);
         llvm::LLVMSetInitializer(ll_g, llval);
 
         let name = NO_ALLOC_SHIM_IS_UNSTABLE;
         let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8);
-        if tcx.sess.default_hidden_visibility() {
-            llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden);
-        }
+        llvm::LLVMRustSetVisibility(
+            ll_g,
+            llvm::Visibility::from_generic(tcx.sess.default_visibility()),
+        );
         let llval = llvm::LLVMConstInt(i8, 0, False);
         llvm::LLVMSetInitializer(ll_g, llval);
     }
@@ -132,9 +134,11 @@ fn create_wrapper_function(
             None
         };
 
-        if tcx.sess.default_hidden_visibility() {
-            llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
-        }
+        llvm::LLVMRustSetVisibility(
+            llfn,
+            llvm::Visibility::from_generic(tcx.sess.default_visibility()),
+        );
+
         if tcx.sess.must_emit_unwind_tables() {
             let uwtable =
                 attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind);

compiler/rustc_codegen_llvm/src/asm.rs

@@ -708,6 +708,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
             S390x(S390xInlineAsmRegClass::reg) => "r",
             S390x(S390xInlineAsmRegClass::reg_addr) => "a",
             S390x(S390xInlineAsmRegClass::freg) => "f",
+            S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
+                unreachable!("clobber-only")
+            }
             Msp430(Msp430InlineAsmRegClass::reg) => "r",
             M68k(M68kInlineAsmRegClass::reg) => "r",
             M68k(M68kInlineAsmRegClass::reg_addr) => "a",
@@ -866,6 +869,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
         Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
         S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
         S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
+        S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
+            unreachable!("clobber-only")
+        }
         Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
         M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
         M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),