Windows x86: Change i128 to return via the vector ABI

Clang and GCC both return `i128` in xmm0 on Windows, both for MSVC and
MinGW targets. Currently, Rust returns `i128` indirectly. Add a fixup
for return ABI so we also return scalar `i128`s, which should make our
`i128` compatible with C.

In the future, Clang may change to return `i128` on the stack for its
`-msvc` targets (more at [1]). If this happens, the change here will
need to be adjusted to only affect MinGW.

Link: rust-lang#134288 (does not fix)

Author: tgross35

compiler/rustc_target/src/callconv/x86_win64.rs

@@ -1,12 +1,12 @@
-use rustc_abi::{BackendRepr, Float, Primitive};
+use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind, Size};
 
 use crate::abi::call::{ArgAbi, FnAbi, Reg};
 use crate::spec::HasTargetSpec;
 
 // Win64 ABI: https://docs.microsoft.com/en-us/cpp/build/parameter-passing
 
 pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) {
-    let fixup = |a: &mut ArgAbi<'_, Ty>| {
+    let fixup = |a: &mut ArgAbi<'_, Ty>, is_ret: bool| {
         match a.layout.backend_repr {
             BackendRepr::Uninhabited | BackendRepr::Memory { sized: false } => {}
             BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
@@ -23,11 +23,16 @@ pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'
                 // (probably what clang calls "illegal vectors").
             }
             BackendRepr::Scalar(scalar) => {
-                // Match what LLVM does for `f128` so that `compiler-builtins` builtins match up
-                // with what LLVM expects.
-                if a.layout.size.bytes() > 8
+                if is_ret && matches!(scalar.primitive(), Primitive::Int(Integer::I128, _)) {
+                    // `i128` is returned in xmm0 by Clang and GCC, no
+                    // FIXME(#134288): This may change for the `-msvc` targets in the future.
+                    let reg = Reg { kind: RegKind::Vector, size: Size::from_bits(128) };
+                    a.cast_to(reg);
+                } else if a.layout.size.bytes() > 8
                     && !matches!(scalar.primitive(), Primitive::Float(Float::F128))
                 {
+                    // Match what LLVM does for `f128` so that `compiler-builtins` builtins match up
+                    // with what LLVM expects.
                     a.make_indirect();
                 } else {
                     a.extend_integer_width_to(32);
@@ -37,8 +42,9 @@ pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'
     };
 
     if !fn_abi.ret.is_ignore() {
-        fixup(&mut fn_abi.ret);
+        fixup(&mut fn_abi.ret, true);
     }
+
     for arg in fn_abi.args.iter_mut() {
         if arg.is_ignore() {
             // x86_64-pc-windows-gnu doesn't ignore ZSTs.
@@ -50,6 +56,6 @@ pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'
             }
             continue;
         }
-        fixup(arg);
+        fixup(arg, false);
     }
 }

tests/codegen/i128-x86-callconv.rs

@@ -40,10 +40,9 @@ pub extern "C" fn ret(_arg0: u32, arg1: i128) -> i128 {
     // CHECK-LABEL: @ret(
     // i128 is returned on the stack on Windows.
     // FIXME: this ABI does not agree with Clang or MinGW GCC
-    // WIN-SAME: ptr{{.*}} sret([16 x i8]){{.*}} [[RET:%_[0-9]+]], i32{{.*}} %_arg0, ptr{{.*}} %arg1)
-    // WIN: [[LOADED:%[0-9]+]] = load i128, ptr %arg1
-    // WIN: store i128 [[LOADED]], ptr [[RET]]
-    // WIN: ret void
+    // WIN-SAME: i32{{.*}} %_arg0, ptr{{.*}} %arg1)
+    // WIN: [[LOADED:%[_0-9]+]] = load <16 x i8>, ptr %arg1
+    // WIN-NEXT: ret <16 x i8> [[LOADED]]
     arg1
 }