force_array -> is_consecutive

The actual ABI implication here is that in some cases the values
are required to be "consecutive", i.e. must either all be passed
in registers or all on stack (without padding).

Adjust the code to either use Uniform::new() or Uniform::consecutive()
depending on which behavior is needed.

Then, when lowering this in LLVM, skip the [1 x i128] to i128
simplification if is_consecutive is set. i128 is the only case
I'm aware of where this is problematic right now. If we find
other cases, we can extend this (either based on target information
or possibly just by not simplifying for is_consecutive entirely).

Author: nikic

compiler/rustc_codegen_llvm/src/abi.rs

@@ -140,8 +140,12 @@ impl LlvmType for CastTarget {
         };
 
         if self.prefix.iter().all(|x| x.is_none()) {
-            // Simplify to a single unit when there is no prefix and size <= unit size
-            if self.rest.total <= self.rest.unit.size && !self.rest.force_array {
+            // Try to simplify to a single unit when there is no prefix and size <= unit size.
+            // We can't do this if is_consecutive is set and the unit would get split on the
+            // target. Currently, this is only relevant for i128 registers.
+            if self.rest.total <= self.rest.unit.size
+                && (!self.rest.is_consecutive || self.rest.unit != Reg::i128())
+            {
                 return rest_ll_unit;
             }
 

compiler/rustc_target/src/abi/call/aarch64.rs

@@ -31,7 +31,7 @@ where
             RegKind::Vector => size.bits() == 64 || size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: size, force_array: false })
+        valid_unit.then_some(Uniform::consecutive(unit, size))
     })
 }
 
@@ -60,7 +60,7 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 128 {
-        ret.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
+        ret.cast_to(Uniform::new(Reg::i64(), size));
         return;
     }
     ret.make_indirect();
@@ -100,9 +100,9 @@ where
     };
     if size.bits() <= 128 {
         if align.bits() == 128 {
-            arg.cast_to(Uniform { unit: Reg::i128(), total: size, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i128(), size));
         } else {
-            arg.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i64(), size));
         }
         return;
     }

compiler/rustc_target/src/abi/call/arm.rs

@@ -21,7 +21,7 @@ where
             RegKind::Vector => size.bits() == 64 || size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: size, force_array: false })
+        valid_unit.then_some(Uniform::consecutive(unit, size))
     })
 }
 
@@ -49,7 +49,7 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 32 {
-        ret.cast_to(Uniform { unit: Reg::i32(), total: size, force_array: false });
+        ret.cast_to(Uniform::new(Reg::i32(), size));
         return;
     }
     ret.make_indirect();
@@ -78,11 +78,7 @@ where
 
     let align = arg.layout.align.abi.bytes();
     let total = arg.layout.size;
-    arg.cast_to(Uniform {
-        unit: if align <= 4 { Reg::i32() } else { Reg::i64() },
-        total,
-        force_array: false,
-    });
+    arg.cast_to(Uniform::consecutive(if align <= 4 { Reg::i32() } else { Reg::i64() }, total));
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)

compiler/rustc_target/src/abi/call/csky.rs

@@ -18,7 +18,7 @@ fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) {
         if total.bits() > 64 {
             arg.make_indirect();
         } else if total.bits() > 32 {
-            arg.cast_to(Uniform { unit: Reg::i32(), total, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i32(), total));
         } else {
             arg.cast_to(Reg::i32());
         }
@@ -38,7 +38,7 @@ fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
     if arg.layout.is_aggregate() {
         let total = arg.layout.size;
         if total.bits() > 32 {
-            arg.cast_to(Uniform { unit: Reg::i32(), total, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i32(), total));
         } else {
             arg.cast_to(Reg::i32());
         }

compiler/rustc_target/src/abi/call/loongarch.rs

@@ -195,11 +195,7 @@ where
         if total.bits() <= xlen {
             arg.cast_to(xlen_reg);
         } else {
-            arg.cast_to(Uniform {
-                unit: xlen_reg,
-                total: Size::from_bits(xlen * 2),
-                force_array: false,
-            });
+            arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2)));
         }
         return false;
     }
@@ -282,11 +278,10 @@ fn classify_arg<'a, Ty, C>(
     if total.bits() > xlen {
         let align_regs = align > xlen;
         if is_loongarch_aggregate(arg) {
-            arg.cast_to(Uniform {
-                unit: if align_regs { double_xlen_reg } else { xlen_reg },
-                total: Size::from_bits(xlen * 2),
-                force_array: false,
-            });
+            arg.cast_to(Uniform::new(
+                if align_regs { double_xlen_reg } else { xlen_reg },
+                Size::from_bits(xlen * 2),
+            ));
         }
         if align_regs && is_vararg {
             *avail_gprs -= *avail_gprs % 2;

compiler/rustc_target/src/abi/call/mips.rs

@@ -27,10 +27,7 @@ where
 
     if arg.layout.is_aggregate() {
         let pad_i32 = !offset.is_aligned(align);
-        arg.cast_to_and_pad_i32(
-            Uniform { unit: Reg::i32(), total: size, force_array: false },
-            pad_i32,
-        );
+        arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32);
     } else {
         arg.extend_integer_width_to(32);
     }

compiler/rustc_target/src/abi/call/mips64.rs

@@ -68,7 +68,7 @@ where
         }
 
         // Cast to a uniform int structure
-        ret.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
+        ret.cast_to(Uniform::new(Reg::i64(), size));
     } else {
         ret.make_indirect();
     }
@@ -139,7 +139,7 @@ where
     let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
     arg.cast_to(CastTarget {
         prefix,
-        rest: Uniform { unit: Reg::i64(), total: rest_size, force_array: false },
+        rest: Uniform::new(Reg::i64(), rest_size),
         attrs: ArgAttributes {
             regular: ArgAttribute::default(),
             arg_ext: ArgExtension::None,

compiler/rustc_target/src/abi/call/mod.rs

@@ -256,20 +256,34 @@ pub struct Uniform {
     ///   64-bit integers with a total size of 20 bytes.
     pub total: Size,
 
-    /// Force the use of an array, even if there is only a single element.
-    pub force_array: bool,
+    /// Indicate that the argument is consecutive, in the sense that either all values need to be
+    /// passed in register, or all on the stack. If they are passed on the stack, there should be
+    /// no additional padding between elements.
+    pub is_consecutive: bool,
 }
 
 impl From<Reg> for Uniform {
     fn from(unit: Reg) -> Uniform {
-        Uniform { unit, total: unit.size, force_array: false }
+        Uniform { unit, total: unit.size, is_consecutive: false }
     }
 }
 
 impl Uniform {
     pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
         self.unit.align(cx)
     }
+
+    /// Pass using one or more values of the given type, without requiring them to be consecutive.
+    /// That is, some values may be passed in register and some on the stack.
+    pub fn new(unit: Reg, total: Size) -> Self {
+        Uniform { unit, total, is_consecutive: false }
+    }
+
+    /// Pass using one or more consecutive values of the given type. Either all values will be
+    /// passed in registers, or all on the stack.
+    pub fn consecutive(unit: Reg, total: Size) -> Self {
+        Uniform { unit, total, is_consecutive: true }
+    }
 }
 
 /// Describes the type used for `PassMode::Cast`.

compiler/rustc_target/src/abi/call/nvptx64.rs

@@ -35,7 +35,11 @@ where
             16 => Reg::i128(),
             _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"),
         };
-        arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes), force_array: false });
+        arg.cast_to(Uniform {
+            unit,
+            total: Size::from_bytes(2 * align_bytes),
+            is_consecutive: false,
+        });
     } else {
         // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271.
         arg.make_direct_deprecated();

compiler/rustc_target/src/abi/call/powerpc64.rs

@@ -37,7 +37,7 @@ where
             RegKind::Vector => arg.layout.size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: arg.layout.size, force_array: false })
+        valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size))
     })
 }
 
@@ -81,7 +81,7 @@ where
             Reg::i64()
         };
 
-        ret.cast_to(Uniform { unit, total: size, force_array: false });
+        ret.cast_to(Uniform::new(unit, size));
         return;
     }
 
@@ -117,11 +117,10 @@ where
         // of i64s or i128s, depending on the aggregate alignment. Always use an array for
         // this, even if there is only a single element.
         let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() };
-        arg.cast_to(Uniform {
-            unit: reg,
-            total: size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()),
-            force_array: true,
-        })
+        arg.cast_to(Uniform::consecutive(
+            reg,
+            size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()),
+        ))
     };
 }
 

compiler/rustc_target/src/abi/call/riscv.rs

@@ -204,7 +204,7 @@ where
             arg.cast_to(Uniform {
                 unit: xlen_reg,
                 total: Size::from_bits(xlen * 2),
-                force_array: false,
+                is_consecutive: false,
             });
         }
         return false;
@@ -291,7 +291,7 @@ fn classify_arg<'a, Ty, C>(
             arg.cast_to(Uniform {
                 unit: if align_regs { double_xlen_reg } else { xlen_reg },
                 total: Size::from_bits(xlen * 2),
-                force_array: false,
+                is_consecutive: false,
             });
         }
         if align_regs && is_vararg {

compiler/rustc_target/src/abi/call/sparc.rs

@@ -28,7 +28,7 @@ where
     if arg.layout.is_aggregate() {
         let pad_i32 = !offset.is_aligned(align);
         arg.cast_to_and_pad_i32(
-            Uniform { unit: Reg::i32(), total: size, force_array: false },
+            Uniform { unit: Reg::i32(), total: size, is_consecutive: false },
             pad_i32,
         );
     } else {

compiler/rustc_target/src/abi/call/sparc64.rs

@@ -192,7 +192,7 @@ where
 
                 arg.cast_to(CastTarget {
                     prefix: data.prefix,
-                    rest: Uniform { unit: Reg::i64(), total: rest_size, force_array: false },
+                    rest: Uniform { unit: Reg::i64(), total: rest_size, is_consecutive: false },
                     attrs: ArgAttributes {
                         regular: data.arg_attribute,
                         arg_ext: ArgExtension::None,
@@ -205,7 +205,7 @@ where
         }
     }
 
-    arg.cast_to(Uniform { unit: Reg::i64(), total, force_array: false });
+    arg.cast_to(Uniform { unit: Reg::i64(), total, is_consecutive: false });
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)