diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index d2828669d438f..f918facc86db1 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -150,7 +150,10 @@ impl LlvmType for CastTarget {
// Simplify to a single unit or an array if there's no prefix.
// This produces the same layout, but using a simpler type.
if self.prefix.iter().all(|x| x.is_none()) {
- if rest_count == 1 {
+ // We can't do this if is_consecutive is set and the unit would get
+ // split on the target. Currently, this is only relevant for i128
+ // registers.
+ if rest_count == 1 && (!self.rest.is_consecutive || self.rest.unit != Reg::i128()) {
return rest_ll_unit;
}
diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs
index f99f6a3b72164..04020d13f22f9 100644
--- a/compiler/rustc_target/src/abi/call/aarch64.rs
+++ b/compiler/rustc_target/src/abi/call/aarch64.rs
@@ -31,7 +31,7 @@ where
RegKind::Vector => size.bits() == 64 || size.bits() == 128,
};
- valid_unit.then_some(Uniform { unit, total: size })
+ valid_unit.then_some(Uniform::consecutive(unit, size))
})
}
@@ -60,7 +60,7 @@ where
let size = ret.layout.size;
let bits = size.bits();
if bits <= 128 {
- ret.cast_to(Uniform { unit: Reg::i64(), total: size });
+ ret.cast_to(Uniform::new(Reg::i64(), size));
return;
}
ret.make_indirect();
@@ -100,9 +100,9 @@ where
};
if size.bits() <= 128 {
if align.bits() == 128 {
- arg.cast_to(Uniform { unit: Reg::i128(), total: size });
+ arg.cast_to(Uniform::new(Reg::i128(), size));
} else {
- arg.cast_to(Uniform { unit: Reg::i64(), total: size });
+ arg.cast_to(Uniform::new(Reg::i64(), size));
}
return;
}
diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs
index 95f6691d42aeb..9371e1b395865 100644
--- a/compiler/rustc_target/src/abi/call/arm.rs
+++ b/compiler/rustc_target/src/abi/call/arm.rs
@@ -21,7 +21,7 @@ where
RegKind::Vector => size.bits() == 64 || size.bits() == 128,
};
- valid_unit.then_some(Uniform { unit, total: size })
+ valid_unit.then_some(Uniform::consecutive(unit, size))
})
}
@@ -49,7 +49,7 @@ where
let size = ret.layout.size;
let bits = size.bits();
if bits <= 32 {
- ret.cast_to(Uniform { unit: Reg::i32(), total: size });
+ ret.cast_to(Uniform::new(Reg::i32(), size));
return;
}
ret.make_indirect();
@@ -78,7 +78,7 @@ where
let align = arg.layout.align.abi.bytes();
let total = arg.layout.size;
- arg.cast_to(Uniform { unit: if align <= 4 { Reg::i32() } else { Reg::i64() }, total });
+ arg.cast_to(Uniform::consecutive(if align <= 4 { Reg::i32() } else { Reg::i64() }, total));
}
pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/csky.rs b/compiler/rustc_target/src/abi/call/csky.rs
index 8b4328db52ebb..7951f28beeaff 100644
--- a/compiler/rustc_target/src/abi/call/csky.rs
+++ b/compiler/rustc_target/src/abi/call/csky.rs
@@ -18,7 +18,7 @@ fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) {
if total.bits() > 64 {
arg.make_indirect();
} else if total.bits() > 32 {
- arg.cast_to(Uniform { unit: Reg::i32(), total });
+ arg.cast_to(Uniform::new(Reg::i32(), total));
} else {
arg.cast_to(Reg::i32());
}
@@ -38,7 +38,7 @@ fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
if arg.layout.is_aggregate() {
let total = arg.layout.size;
if total.bits() > 32 {
- arg.cast_to(Uniform { unit: Reg::i32(), total });
+ arg.cast_to(Uniform::new(Reg::i32(), total));
} else {
arg.cast_to(Reg::i32());
}
diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs
index 35d4b331cb4d3..943b12a9fbfcc 100644
--- a/compiler/rustc_target/src/abi/call/loongarch.rs
+++ b/compiler/rustc_target/src/abi/call/loongarch.rs
@@ -195,7 +195,7 @@ where
if total.bits() <= xlen {
arg.cast_to(xlen_reg);
} else {
- arg.cast_to(Uniform { unit: xlen_reg, total: Size::from_bits(xlen * 2) });
+ arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2)));
}
return false;
}
@@ -278,10 +278,10 @@ fn classify_arg<'a, Ty, C>(
if total.bits() > xlen {
let align_regs = align > xlen;
if is_loongarch_aggregate(arg) {
- arg.cast_to(Uniform {
- unit: if align_regs { double_xlen_reg } else { xlen_reg },
- total: Size::from_bits(xlen * 2),
- });
+ arg.cast_to(Uniform::new(
+ if align_regs { double_xlen_reg } else { xlen_reg },
+ Size::from_bits(xlen * 2),
+ ));
}
if align_regs && is_vararg {
*avail_gprs -= *avail_gprs % 2;
diff --git a/compiler/rustc_target/src/abi/call/mips.rs b/compiler/rustc_target/src/abi/call/mips.rs
index 57ccfe2152bdf..0e5a7f37a094c 100644
--- a/compiler/rustc_target/src/abi/call/mips.rs
+++ b/compiler/rustc_target/src/abi/call/mips.rs
@@ -27,7 +27,7 @@ where
if arg.layout.is_aggregate() {
let pad_i32 = !offset.is_aligned(align);
- arg.cast_to_and_pad_i32(Uniform { unit: Reg::i32(), total: size }, pad_i32);
+ arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32);
} else {
arg.extend_integer_width_to(32);
}
diff --git a/compiler/rustc_target/src/abi/call/mips64.rs b/compiler/rustc_target/src/abi/call/mips64.rs
index 2700f67b2096b..b2a2c34b980f6 100644
--- a/compiler/rustc_target/src/abi/call/mips64.rs
+++ b/compiler/rustc_target/src/abi/call/mips64.rs
@@ -68,7 +68,7 @@ where
}
// Cast to a uniform int structure
- ret.cast_to(Uniform { unit: Reg::i64(), total: size });
+ ret.cast_to(Uniform::new(Reg::i64(), size));
} else {
ret.make_indirect();
}
@@ -139,7 +139,7 @@ where
let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
arg.cast_to(CastTarget {
prefix,
- rest: Uniform { unit: Reg::i64(), total: rest_size },
+ rest: Uniform::new(Reg::i64(), rest_size),
attrs: ArgAttributes {
regular: ArgAttribute::default(),
arg_ext: ArgExtension::None,
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index cdd3f0afd79a6..4502df339d1d8 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -255,11 +255,16 @@ pub struct Uniform {
/// for 64-bit integers with a total size of 20 bytes. When the argument is actually passed,
/// this size will be rounded up to the nearest multiple of `unit.size`.
pub total: Size,
+
+ /// Indicate that the argument is consecutive, in the sense that either all values need to be
+ /// passed in register, or all on the stack. If they are passed on the stack, there should be
+ /// no additional padding between elements.
+ pub is_consecutive: bool,
}
impl From<Reg> for Uniform {
fn from(unit: Reg) -> Uniform {
- Uniform { unit, total: unit.size }
+ Uniform { unit, total: unit.size, is_consecutive: false }
}
}
@@ -267,6 +272,18 @@ impl Uniform {
pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
self.unit.align(cx)
}
+
+ /// Pass using one or more values of the given type, without requiring them to be consecutive.
+ /// That is, some values may be passed in register and some on the stack.
+ pub fn new(unit: Reg, total: Size) -> Self {
+ Uniform { unit, total, is_consecutive: false }
+ }
+
+ /// Pass using one or more consecutive values of the given type. Either all values will be
+ /// passed in registers, or all on the stack.
+ pub fn consecutive(unit: Reg, total: Size) -> Self {
+ Uniform { unit, total, is_consecutive: true }
+ }
}
/// Describes the type used for `PassMode::Cast`.
diff --git a/compiler/rustc_target/src/abi/call/nvptx64.rs b/compiler/rustc_target/src/abi/call/nvptx64.rs
index 5c040ce9c3b10..f85fa2419f0f6 100644
--- a/compiler/rustc_target/src/abi/call/nvptx64.rs
+++ b/compiler/rustc_target/src/abi/call/nvptx64.rs
@@ -35,7 +35,7 @@ where
16 => Reg::i128(),
_ => unreachable!("Align is given as power of 2 no larger than 16 bytes"),
};
- arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes) });
+ arg.cast_to(Uniform::new(unit, Size::from_bytes(2 * align_bytes)));
} else {
// FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271.
arg.make_direct_deprecated();
diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs
index 2d41f77e50e1e..11a6cb52babc9 100644
--- a/compiler/rustc_target/src/abi/call/powerpc64.rs
+++ b/compiler/rustc_target/src/abi/call/powerpc64.rs
@@ -2,7 +2,7 @@
// Alignment of 128 bit types is not currently handled, this will
// need to be fixed when PowerPC vector support is added.
-use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind, Uniform};
+use crate::abi::call::{Align, ArgAbi, FnAbi, Reg, RegKind, Uniform};
use crate::abi::{Endian, HasDataLayout, TyAbiInterface};
use crate::spec::HasTargetSpec;
@@ -37,7 +37,7 @@ where
RegKind::Vector => arg.layout.size.bits() == 128,
};
- valid_unit.then_some(Uniform { unit, total: arg.layout.size })
+ valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size))
})
}
@@ -81,7 +81,7 @@ where
Reg::i64()
};
- ret.cast_to(Uniform { unit, total: size });
+ ret.cast_to(Uniform::new(unit, size));
return;
}
@@ -108,18 +108,20 @@ where
}
let size = arg.layout.size;
- let (unit, total) = if size.bits() <= 64 {
+ if size.bits() <= 64 {
// Aggregates smaller than a doubleword should appear in
// the least-significant bits of the parameter doubleword.
- (Reg { kind: RegKind::Integer, size }, size)
+ arg.cast_to(Reg { kind: RegKind::Integer, size })
} else {
- // Aggregates larger than a doubleword should be padded
- // at the tail to fill out a whole number of doublewords.
- let reg_i64 = Reg::i64();
- (reg_i64, size.align_to(reg_i64.align(cx)))
+ // Aggregates larger than i64 should be padded at the tail to fill out a whole number
+ // of i64s or i128s, depending on the aggregate alignment. Always use an array for
+ // this, even if there is only a single element.
+ let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() };
+ arg.cast_to(Uniform::consecutive(
+ reg,
+ size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()),
+ ))
};
-
- arg.cast_to(Uniform { unit, total });
}
pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs
index 6a38496dc5761..5d4b3a9d245a4 100644
--- a/compiler/rustc_target/src/abi/call/riscv.rs
+++ b/compiler/rustc_target/src/abi/call/riscv.rs
@@ -201,7 +201,7 @@ where
if total.bits() <= xlen {
arg.cast_to(xlen_reg);
} else {
- arg.cast_to(Uniform { unit: xlen_reg, total: Size::from_bits(xlen * 2) });
+ arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2)));
}
return false;
}
@@ -284,10 +284,10 @@ fn classify_arg<'a, Ty, C>(
if total.bits() > xlen {
let align_regs = align > xlen;
if is_riscv_aggregate(arg) {
- arg.cast_to(Uniform {
- unit: if align_regs { double_xlen_reg } else { xlen_reg },
- total: Size::from_bits(xlen * 2),
- });
+ arg.cast_to(Uniform::new(
+ if align_regs { double_xlen_reg } else { xlen_reg },
+ Size::from_bits(xlen * 2),
+ ));
}
if align_regs && is_vararg {
*avail_gprs -= *avail_gprs % 2;
diff --git a/compiler/rustc_target/src/abi/call/sparc.rs b/compiler/rustc_target/src/abi/call/sparc.rs
index 57ccfe2152bdf..0e5a7f37a094c 100644
--- a/compiler/rustc_target/src/abi/call/sparc.rs
+++ b/compiler/rustc_target/src/abi/call/sparc.rs
@@ -27,7 +27,7 @@ where
if arg.layout.is_aggregate() {
let pad_i32 = !offset.is_aligned(align);
- arg.cast_to_and_pad_i32(Uniform { unit: Reg::i32(), total: size }, pad_i32);
+ arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32);
} else {
arg.extend_integer_width_to(32);
}
diff --git a/compiler/rustc_target/src/abi/call/sparc64.rs b/compiler/rustc_target/src/abi/call/sparc64.rs
index cbed5b4afc134..acdcd5cc0d482 100644
--- a/compiler/rustc_target/src/abi/call/sparc64.rs
+++ b/compiler/rustc_target/src/abi/call/sparc64.rs
@@ -192,7 +192,7 @@ where
arg.cast_to(CastTarget {
prefix: data.prefix,
- rest: Uniform { unit: Reg::i64(), total: rest_size },
+ rest: Uniform::new(Reg::i64(), rest_size),
attrs: ArgAttributes {
regular: data.arg_attribute,
arg_ext: ArgExtension::None,
@@ -205,7 +205,7 @@ where
}
}
- arg.cast_to(Uniform { unit: Reg::i64(), total });
+ arg.cast_to(Uniform::new(Reg::i64(), total));
}
pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/wasm.rs b/compiler/rustc_target/src/abi/call/wasm.rs
index a7a2b314a94f0..a773fb1e814e0 100644
--- a/compiler/rustc_target/src/abi/call/wasm.rs
+++ b/compiler/rustc_target/src/abi/call/wasm.rs
@@ -1,4 +1,4 @@
-use crate::abi::call::{ArgAbi, FnAbi, Uniform};
+use crate::abi::call::{ArgAbi, FnAbi};
use crate::abi::{HasDataLayout, TyAbiInterface};
fn unwrap_trivial_aggregate<'a, Ty, C>(cx: &C, val: &mut ArgAbi<'a, Ty>) -> bool
@@ -10,7 +10,7 @@ where
if let Some(unit) = val.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()) {
let size = val.layout.size;
if unit.size == size {
- val.cast_to(Uniform { unit, total: size });
+ val.cast_to(unit);
return true;
}
}
diff --git a/tests/codegen/powerpc64le-struct-align-128.rs b/tests/codegen/powerpc64le-struct-align-128.rs
new file mode 100644
index 0000000000000..0096c6d3138b8
--- /dev/null
+++ b/tests/codegen/powerpc64le-struct-align-128.rs
@@ -0,0 +1,93 @@
+// Test that structs aligned to 128 bits are passed with the correct ABI on powerpc64le.
+// This is similar to aarch64-struct-align-128.rs, but for ppc.
+
+//@ compile-flags: --target powerpc64le-unknown-linux-gnu
+//@ needs-llvm-components: powerpc
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+#[lang="sized"]
+trait Sized { }
+#[lang="freeze"]
+trait Freeze { }
+#[lang="copy"]
+trait Copy { }
+
+#[repr(C)]
+pub struct Align8 {
+ pub a: u64,
+ pub b: u64,
+}
+
+#[repr(transparent)]
+pub struct Transparent8 {
+ a: Align8
+}
+
+#[repr(C)]
+pub struct Wrapped8 {
+ a: Align8,
+}
+
+extern "C" {
+ // CHECK: declare void @test_8([2 x i64], [2 x i64], [2 x i64])
+ fn test_8(a: Align8, b: Transparent8, c: Wrapped8);
+}
+
+#[repr(C)]
+#[repr(align(16))]
+pub struct Align16 {
+ pub a: u64,
+ pub b: u64,
+}
+
+#[repr(transparent)]
+pub struct Transparent16 {
+ a: Align16
+}
+
+#[repr(C)]
+pub struct Wrapped16 {
+ pub a: Align16,
+}
+
+extern "C" {
+ // It's important that this produces [1 x i128] rather than just i128!
+ // CHECK: declare void @test_16([1 x i128], [1 x i128], [1 x i128])
+ fn test_16(a: Align16, b: Transparent16, c: Wrapped16);
+}
+
+#[repr(C)]
+#[repr(align(32))]
+pub struct Align32 {
+ pub a: u64,
+ pub b: u64,
+ pub c: u64,
+}
+
+#[repr(transparent)]
+pub struct Transparent32 {
+ a: Align32
+}
+
+#[repr(C)]
+pub struct Wrapped32 {
+ pub a: Align32,
+}
+
+extern "C" {
+ // CHECK: declare void @test_32([2 x i128], [2 x i128], [2 x i128])
+ fn test_32(a: Align32, b: Transparent32, c: Wrapped32);
+}
+
+pub unsafe fn main(
+ a1: Align8, a2: Transparent8, a3: Wrapped8,
+ b1: Align16, b2: Transparent16, b3: Wrapped16,
+ c1: Align32, c2: Transparent32, c3: Wrapped32,
+) {
+ test_8(a1, a2, a3);
+ test_16(b1, b2, b3);
+ test_32(c1, c2, c3);
+}