Auto merge of #102503 - cuviper:x86-stack-probes, r=nagisa

Enable inline stack probes on X86 with LLVM 16

The known problems with x86 inline-asm stack probes have been solved on LLVM main (16), so this flips the switch. Anyone using bleeding-edge LLVM with rustc can start testing this, as I have done locally. We'll get more direct rust-ci when LLVM 16 branches and we start our upgrade, and we can always patch or disable it then if we find new problems.

The previous attempt was #77885, reverted in #84708.

compiler/rustc_target/src/spec/i386_apple_ios.rs

@@ -14,8 +14,7 @@ pub fn target() -> Target {
         arch: "x86".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+            stack_probes: StackProbeType::X86,
-            stack_probes: StackProbeType::Call,
             ..base
         },
     }

compiler/rustc_target/src/spec/i686_apple_darwin.rs

@@ -7,8 +7,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.link_env_remove.to_mut().extend(super::apple_base::macos_link_env_remove());
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.frame_pointer = FramePointer::Always;
 
     // Clang automatically chooses a more specific target based on

compiler/rustc_target/src/spec/i686_linux_android.rs

@@ -11,8 +11,7 @@ pub fn target() -> Target {
     // https://developer.android.com/ndk/guides/abis.html#x86
     base.cpu = "pentiumpro".into();
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3".into();
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-linux-android".into(),

compiler/rustc_target/src/spec/i686_unknown_freebsd.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "-Wl,-znotext"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-unknown-freebsd".into(),

compiler/rustc_target/src/spec/i686_unknown_haiku.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-unknown-haiku".into(),

compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs

@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.supported_sanitizers = SanitizerSet::ADDRESS;
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "-Wl,-melf_i386"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     // The unwinder used by i686-unknown-linux-musl, the LLVM libunwind
     // implementation, apparently relies on frame pointers existing... somehow.

compiler/rustc_target/src/spec/i686_unknown_netbsd.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-unknown-netbsdelf".into(),

compiler/rustc_target/src/spec/i686_unknown_openbsd.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "-fuse-ld=lld"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-unknown-openbsd".into(),

compiler/rustc_target/src/spec/i686_wrs_vxworks.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "i686-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/linux_kernel_base.rs

@@ -6,8 +6,7 @@ pub fn opts() -> TargetOptions {
         env: "gnu".into(),
         disable_redzone: true,
         panic_strategy: PanicStrategy::Abort,
-        // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+        stack_probes: StackProbeType::X86,
-        stack_probes: StackProbeType::Call,
         frame_pointer: FramePointer::Always,
         position_independent_executables: true,
         needs_plt: true,

compiler/rustc_target/src/spec/mod.rs

@@ -635,6 +635,10 @@ pub enum StackProbeType {
 }
 
 impl StackProbeType {
+    // LLVM X86 targets (ix86 and x86_64) can use inline-asm stack probes starting with LLVM 16.
+    // Notable past issues were rust#83139 (fixed in 14) and rust#84667 (fixed in 16).
+    const X86: Self = Self::InlineOrCall { min_llvm_version_for_inline: (16, 0, 0) };
+
     fn from_json(json: &Json) -> Result<Self, String> {
         let object = json.as_object().ok_or_else(|| "expected a JSON object")?;
         let kind = object

compiler/rustc_target/src/spec/x86_64_apple_darwin.rs

@@ -9,8 +9,7 @@ pub fn target() -> Target {
     base.frame_pointer = FramePointer::Always;
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.link_env_remove.to_mut().extend(super::apple_base::macos_link_env_remove());
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
 

compiler/rustc_target/src/spec/x86_64_apple_ios.rs

@@ -13,8 +13,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+            stack_probes: StackProbeType::X86,
-            stack_probes: StackProbeType::Call,
             ..base
         },
     }

compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs

@@ -15,8 +15,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+            stack_probes: StackProbeType::X86,
-            stack_probes: StackProbeType::Call,
             ..base
         },
     }

compiler/rustc_target/src/spec/x86_64_apple_tvos.rs

@@ -10,8 +10,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+            stack_probes: StackProbeType::X86,
-            stack_probes: StackProbeType::Call,
             ..base
         },
     }

compiler/rustc_target/src/spec/x86_64_apple_watchos_sim.rs

@@ -15,8 +15,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+            stack_probes: StackProbeType::X86,
-            stack_probes: StackProbeType::Call,
             forces_embed_bitcode: true,
             // Taken from a clang build on Xcode 11.4.1.
             // These arguments are not actually invoked - they just have

compiler/rustc_target/src/spec/x86_64_fuchsia.rs

@@ -4,8 +4,7 @@ pub fn target() -> Target {
     let mut base = super::fuchsia_base::opts();
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI;
 
     Target {

compiler/rustc_target/src/spec/x86_64_linux_android.rs

@@ -7,8 +7,7 @@ pub fn target() -> Target {
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "x86_64-linux-android".into(),

compiler/rustc_target/src/spec/x86_64_pc_solaris.rs

@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.vendor = "pc".into();
     base.max_atomic_width = Some(64);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI;
 
     Target {

compiler/rustc_target/src/spec/x86_64_sun_solaris.rs

@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.vendor = "sun".into();
     base.max_atomic_width = Some(64);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "x86_64-pc-solaris".into(),

compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "x86_64-unknown-dragonfly".into(),

compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 

compiler/rustc_target/src/spec/x86_64_unknown_haiku.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     // This option is required to build executables on Haiku x86_64
     base.position_independent_executables = true;
 

compiler/rustc_target/src/spec/x86_64_unknown_hermit.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.features = "+rdrnd,+rdseed".into();
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "x86_64-unknown-hermit".into(),

compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI

compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs

@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.abi = "x32".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-mx32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.has_thread_local = false;
     // BUG(GabrielMajeri): disabling the PLT on x86_64 Linux with x32 ABI
     // breaks code gen. See LLVM bug 36743

compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI

compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
         | SanitizerSet::LEAK

compiler/rustc_target/src/spec/x86_64_unknown_none.rs

@@ -11,8 +11,7 @@ pub fn target() -> Target {
     let opts = TargetOptions {
         cpu: "x86-64".into(),
         max_atomic_width: Some(64),
-        // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+        stack_probes: StackProbeType::X86,
-        stack_probes: StackProbeType::Call,
         position_independent_executables: true,
         static_position_independent_executables: true,
         relro_level: RelroLevel::Full,

compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "x86_64-unknown-openbsd".into(),

compiler/rustc_target/src/spec/x86_64_unknown_redox.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
 
     Target {
         llvm_target: "x86_64-unknown-redox".into(),

compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs

@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
+    base.stack_probes = StackProbeType::X86;
-    base.stack_probes = StackProbeType::Call;
     base.disable_redzone = true;
 
     Target {

src/test/assembly/x86-stack-probes.rs

@@ -0,0 +1,42 @@
+// min-llvm-version: 16
+// revisions: x86_64 i686
+// assembly-output: emit-asm
+//[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//[x86_64] needs-llvm-components: x86
+//[i686] compile-flags: --target i686-unknown-linux-gnu
+//[i686] needs-llvm-components: x86
+// compile-flags: -C llvm-args=-x86-asm-syntax=intel
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+#[lang = "copy"]
+trait Copy {}
+
+impl Copy for u8 {}
+
+// Check that inline-asm stack probes are generated correctly.
+// To avoid making this test fragile to slight asm changes,
+// we only check that the stack pointer is decremented by a page at a time,
+// instead of matching the whole probe sequence.
+
+// CHECK-LABEL: small_stack_probe:
+#[no_mangle]
+pub fn small_stack_probe(x: u8, f: fn(&mut [u8; 8192])) {
+    // CHECK-NOT: __rust_probestack
+    // x86_64: sub rsp, 4096
+    // i686: sub esp, 4096
+    f(&mut [x; 8192]);
+}
+
+// CHECK-LABEL: big_stack_probe:
+#[no_mangle]
+pub fn big_stack_probe(x: u8, f: fn(&[u8; 65536])) {
+    // CHECK-NOT: __rust_probestack
+    // x86_64: sub rsp, 4096
+    // i686: sub esp, 4096
+    f(&mut [x; 65536]);
+}

src/test/codegen/stack-probes-call.rs

@@ -5,8 +5,10 @@
 // revisions: i686 x86_64
 //[i686] compile-flags: --target i686-unknown-linux-gnu
 //[i686] needs-llvm-components: x86
+//[i686] ignore-llvm-version: 16 - 99
 //[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
 //[x86_64] needs-llvm-components: x86
+//[x86_64] ignore-llvm-version: 16 - 99
 
 #![crate_type = "rlib"]
 #![feature(no_core, lang_items)]

src/test/codegen/stack-probes-inline.rs

@@ -2,7 +2,7 @@
 // or `StackProbeType::InlineOrCall` when running on newer LLVM.
 
 // compile-flags: -C no-prepopulate-passes
-// revisions: powerpc powerpc64 powerpc64le s390x
+// revisions: powerpc powerpc64 powerpc64le s390x i686 x86_64
 //[powerpc] compile-flags: --target powerpc-unknown-linux-gnu
 //[powerpc] needs-llvm-components: powerpc
 //[powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu
@@ -11,6 +11,12 @@
 //[powerpc64le] needs-llvm-components: powerpc
 //[s390x] compile-flags: --target s390x-unknown-linux-gnu
 //[s390x] needs-llvm-components: systemz
+//[i686] compile-flags: --target i686-unknown-linux-gnu
+//[i686] needs-llvm-components: x86
+//[i686] min-llvm-version: 16
+//[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//[x86_64] needs-llvm-components: x86
+//[x86_64] min-llvm-version: 16
 
 #![crate_type = "rlib"]
 #![feature(no_core, lang_items)]