allow system abi to be variadic

compiler/rustc_hir_analysis/src/lib.rs

@@ -116,7 +116,8 @@ use rustc_hir::def::DefKind;
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi, span: Span) {
-    const CONVENTIONS_UNSTABLE: &str = "`C`, `cdecl`, `aapcs`, `win64`, `sysv64` or `efiapi`";
+    const CONVENTIONS_UNSTABLE: &str =
+        "`C`, `cdecl`, `system`, `aapcs`, `win64`, `sysv64` or `efiapi`";
     const CONVENTIONS_STABLE: &str = "`C` or `cdecl`";
     const UNSTABLE_EXPLAIN: &str =
         "using calling conventions other than `C` or `cdecl` for varargs functions is unstable";

compiler/rustc_metadata/src/native_libs.rs

@@ -520,11 +520,23 @@ impl<'tcx> Collector<'tcx> {
     ) -> DllImport {
         let span = self.tcx.def_span(item);
 
+        // this logic is similar to `Target::adjust_abi` (in rustc_target/src/spec/mod.rs) but errors on unsupported inputs
         let calling_convention = if self.tcx.sess.target.arch == "x86" {
             match abi {
                 Abi::C { .. } | Abi::Cdecl { .. } => DllCallingConvention::C,
-                Abi::Stdcall { .. } | Abi::System { .. } => {
-                    DllCallingConvention::Stdcall(self.i686_arg_list_size(item))
+                Abi::Stdcall { .. } => DllCallingConvention::Stdcall(self.i686_arg_list_size(item)),
+                // On Windows, `extern "system"` behaves like msvc's `__stdcall`.
+                // `__stdcall` only applies on x86 and on non-variadic functions:
+                // https://learn.microsoft.com/en-us/cpp/cpp/stdcall?view=msvc-170
+                Abi::System { .. } => {
+                    let c_variadic =
+                        self.tcx.type_of(item).instantiate_identity().fn_sig(self.tcx).c_variadic();
+
+                    if c_variadic {
+                        DllCallingConvention::C
+                    } else {
+                        DllCallingConvention::Stdcall(self.i686_arg_list_size(item))
+                    }
                 }
                 Abi::Fastcall { .. } => {
                     DllCallingConvention::Fastcall(self.i686_arg_list_size(item))

compiler/rustc_target/src/abi/call/mod.rs

@@ -840,7 +840,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
             "sparc" => sparc::compute_abi_info(cx, self),
             "sparc64" => sparc64::compute_abi_info(cx, self),
             "nvptx64" => {
-                if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::PtxKernel {
+                if cx.target_spec().adjust_abi(abi, self.c_variadic) == spec::abi::Abi::PtxKernel {
                     nvptx64::compute_ptx_kernel_abi_info(cx, self)
                 } else {
                     nvptx64::compute_abi_info(self)
@@ -849,7 +849,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
             "hexagon" => hexagon::compute_abi_info(self),
             "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self),
             "wasm32" | "wasm64" => {
-                if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::Wasm {
+                if cx.target_spec().adjust_abi(abi, self.c_variadic) == spec::abi::Abi::Wasm {
                     wasm::compute_wasm_abi_info(self)
                 } else {
                     wasm::compute_c_abi_info(cx, self)

compiler/rustc_target/src/spec/abi/mod.rs

@@ -70,15 +70,16 @@ impl Abi {
         // * C and Cdecl obviously support varargs.
         // * C can be based on Aapcs, SysV64 or Win64, so they must support varargs.
         // * EfiApi is based on Win64 or C, so it also supports it.
+        // * System falls back to C for functions with varargs.
         //
         // * Stdcall does not, because it would be impossible for the callee to clean
         //   up the arguments. (callee doesn't know how many arguments are there)
         // * Same for Fastcall, Vectorcall and Thiscall.
-        // * System can become Stdcall, so is also a no-no.
         // * Other calling conventions are related to hardware or the compiler itself.
         match self {
             Self::C { .. }
             | Self::Cdecl { .. }
+            | Self::System { .. }
             | Self::Aapcs { .. }
             | Self::Win64 { .. }
             | Self::SysV64 { .. }

compiler/rustc_target/src/spec/mod.rs

@@ -2399,10 +2399,14 @@ impl DerefMut for Target {
 
 impl Target {
     /// Given a function ABI, turn it into the correct ABI for this target.
-    pub fn adjust_abi(&self, abi: Abi) -> Abi {
+    pub fn adjust_abi(&self, abi: Abi, c_variadic: bool) -> Abi {
         match abi {
             Abi::C { .. } => self.default_adjusted_cabi.unwrap_or(abi),
-            Abi::System { unwind } if self.is_like_windows && self.arch == "x86" => {
+
+            // On Windows, `extern "system"` behaves like msvc's `__stdcall`.
+            // `__stdcall` only applies on x86 and on non-variadic functions:
+            // https://learn.microsoft.com/en-us/cpp/cpp/stdcall?view=msvc-170
+            Abi::System { unwind } if self.is_like_windows && self.arch == "x86" && !c_variadic => {
                 Abi::Stdcall { unwind }
             }
             Abi::System { unwind } => Abi::C { unwind },

compiler/rustc_ty_utils/src/abi.rs

@@ -228,9 +228,9 @@ fn fn_sig_for_fn_abi<'tcx>(
 }
 
 #[inline]
-fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv {
+fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi, c_variadic: bool) -> Conv {
     use rustc_target::spec::abi::Abi::*;
-    match tcx.sess.target.adjust_abi(abi) {
+    match tcx.sess.target.adjust_abi(abi, c_variadic) {
         RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
 
         // This is intentionally not using `Conv::Cold`, as that has to preserve
@@ -488,7 +488,7 @@ fn fn_abi_new_uncached<'tcx>(
 ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, &'tcx FnAbiError<'tcx>> {
     let sig = cx.tcx.normalize_erasing_late_bound_regions(cx.param_env, sig);
 
-    let conv = conv_from_spec_abi(cx.tcx(), sig.abi);
+    let conv = conv_from_spec_abi(cx.tcx(), sig.abi, sig.c_variadic);
 
     let mut inputs = sig.inputs();
     let extra_args = if sig.abi == RustCall {

tests/ui/c-variadic/variadic-ffi-2.rs

@@ -3,10 +3,13 @@
 
 fn baz(f: extern "stdcall" fn(usize, ...)) {
     //~^ ERROR: C-variadic function must have a compatible calling convention,
-    // like C, cdecl, aapcs, win64, sysv64 or efiapi
+    // like C, cdecl, system, aapcs, win64, sysv64 or efiapi
     f(22, 44);
 }
 
+fn system(f: extern "system" fn(usize, ...)) {
+    f(22, 44);
+}
 fn aapcs(f: extern "aapcs" fn(usize, ...)) {
     f(22, 44);
 }

tests/ui/c-variadic/variadic-ffi-2.stderr

@@ -1,4 +1,4 @@
-error[E0045]: C-variadic function must have a compatible calling convention, like `C`, `cdecl`, `aapcs`, `win64`, `sysv64` or `efiapi`
+error[E0045]: C-variadic function must have a compatible calling convention, like `C`, `cdecl`, `system`, `aapcs`, `win64`, `sysv64` or `efiapi`
   --> $DIR/variadic-ffi-2.rs:4:11
    |
 LL | fn baz(f: extern "stdcall" fn(usize, ...)) {