diff --git a/rust-version b/rust-version
index 5396b338ce..362bcc35ea 100644
--- a/rust-version
+++ b/rust-version
@@ -1 +1 @@
-db41351753df840773ca628d8daa040e95d00eef
+880416180b0a9ee1141c07d4d17667edb77daebd
diff --git a/src/shims/foreign_items.rs b/src/shims/foreign_items.rs
index cd4fedad0f..8d82c912f3 100644
--- a/src/shims/foreign_items.rs
+++ b/src/shims/foreign_items.rs
@@ -297,8 +297,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
Some(p) => p,
};
- // Second: functions that return.
- match this.emulate_foreign_item_by_name(link_name, abi, args, dest, ret)? {
+ // Second: functions that return immediately.
+ match this.emulate_foreign_item_by_name(link_name, abi, args, dest)? {
EmulateByNameResult::NeedsJumping => {
trace!("{:?}", this.dump_place(**dest));
this.go_to_block(ret);
@@ -355,7 +355,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
- ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
@@ -702,8 +701,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
// Platform-specific shims
_ => match this.tcx.sess.target.os.as_ref() {
- target if target_os_is_unix(target) => return shims::unix::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
- "windows" => return shims::windows::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
+ target if target_os_is_unix(target) => return shims::unix::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
+ "windows" => return shims::windows::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
target => throw_unsup_format!("the target `{}` is not supported", target),
}
};
diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs
deleted file mode 100644
index ab79438c73..0000000000
--- a/src/shims/intrinsics.rs
+++ /dev/null
@@ -1,1423 +0,0 @@
-use std::iter;
-
-use log::trace;
-
-use rustc_apfloat::{Float, Round};
-use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
-use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
-use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size};
-
-use crate::*;
-use helpers::check_arg_count;
-
-pub enum AtomicOp {
- MirOp(mir::BinOp, bool),
- Max,
- Min,
-}
-
-impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
-pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
- fn call_intrinsic(
- &mut self,
- instance: ty::Instance<'tcx>,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- ret: Option<mir::BasicBlock>,
- _unwind: StackPopUnwind,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
-
- if this.emulate_intrinsic(instance, args, dest, ret)? {
- return Ok(());
- }
-
- // All supported intrinsics have a return place.
- let intrinsic_name = this.tcx.item_name(instance.def_id());
- let intrinsic_name = intrinsic_name.as_str();
- let ret = match ret {
- None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"),
- Some(p) => p,
- };
-
- // Then handle terminating intrinsics.
- match intrinsic_name {
- // Miri overwriting CTFE intrinsics.
- "ptr_guaranteed_eq" => {
- let [left, right] = check_arg_count(args)?;
- let left = this.read_immediate(left)?;
- let right = this.read_immediate(right)?;
- this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
- }
- "ptr_guaranteed_ne" => {
- let [left, right] = check_arg_count(args)?;
- let left = this.read_immediate(left)?;
- let right = this.read_immediate(right)?;
- this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
- }
- "const_allocate" => {
- // For now, for compatibility with the run-time implementation of this, we just return null.
- // See <https://github.com/rust-lang/rust/issues/93935>.
- this.write_null(dest)?;
- }
- "const_deallocate" => {
- // complete NOP
- }
-
- // Raw memory accesses
- "volatile_load" => {
- let [place] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
- this.copy_op(&place.into(), dest, /*allow_transmute*/ false)?;
- }
- "volatile_store" => {
- let [place, dest] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
- this.copy_op(dest, &place.into(), /*allow_transmute*/ false)?;
- }
-
- "write_bytes" | "volatile_set_memory" => {
- let [ptr, val_byte, count] = check_arg_count(args)?;
- let ty = instance.substs.type_at(0);
- let ty_layout = this.layout_of(ty)?;
- let val_byte = this.read_scalar(val_byte)?.to_u8()?;
- let ptr = this.read_pointer(ptr)?;
- let count = this.read_scalar(count)?.to_machine_usize(this)?;
- // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
- // but no actual allocation can be big enough for the difference to be noticeable.
- let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
- err_ub_format!("overflow computing total size of `{intrinsic_name}`")
- })?;
- this.write_bytes_ptr(
- ptr,
- iter::repeat(val_byte).take(byte_count.bytes() as usize),
- )?;
- }
-
- // Floating-point operations
- "fabsf32" => {
- let [f] = check_arg_count(args)?;
- let f = this.read_scalar(f)?.to_f32()?;
- // Can be implemented in soft-floats.
- this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
- }
- "fabsf64" => {
- let [f] = check_arg_count(args)?;
- let f = this.read_scalar(f)?.to_f64()?;
- // Can be implemented in soft-floats.
- this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
- }
- #[rustfmt::skip]
- | "sinf32"
- | "cosf32"
- | "sqrtf32"
- | "expf32"
- | "exp2f32"
- | "logf32"
- | "log10f32"
- | "log2f32"
- | "floorf32"
- | "ceilf32"
- | "truncf32"
- | "roundf32"
- => {
- let [f] = check_arg_count(args)?;
- // FIXME: Using host floats.
- let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
- let f = match intrinsic_name {
- "sinf32" => f.sin(),
- "cosf32" => f.cos(),
- "sqrtf32" => f.sqrt(),
- "expf32" => f.exp(),
- "exp2f32" => f.exp2(),
- "logf32" => f.ln(),
- "log10f32" => f.log10(),
- "log2f32" => f.log2(),
- "floorf32" => f.floor(),
- "ceilf32" => f.ceil(),
- "truncf32" => f.trunc(),
- "roundf32" => f.round(),
- _ => bug!(),
- };
- this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
- }
-
- #[rustfmt::skip]
- | "sinf64"
- | "cosf64"
- | "sqrtf64"
- | "expf64"
- | "exp2f64"
- | "logf64"
- | "log10f64"
- | "log2f64"
- | "floorf64"
- | "ceilf64"
- | "truncf64"
- | "roundf64"
- => {
- let [f] = check_arg_count(args)?;
- // FIXME: Using host floats.
- let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
- let f = match intrinsic_name {
- "sinf64" => f.sin(),
- "cosf64" => f.cos(),
- "sqrtf64" => f.sqrt(),
- "expf64" => f.exp(),
- "exp2f64" => f.exp2(),
- "logf64" => f.ln(),
- "log10f64" => f.log10(),
- "log2f64" => f.log2(),
- "floorf64" => f.floor(),
- "ceilf64" => f.ceil(),
- "truncf64" => f.trunc(),
- "roundf64" => f.round(),
- _ => bug!(),
- };
- this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
- }
-
- #[rustfmt::skip]
- | "fadd_fast"
- | "fsub_fast"
- | "fmul_fast"
- | "fdiv_fast"
- | "frem_fast"
- => {
- let [a, b] = check_arg_count(args)?;
- let a = this.read_immediate(a)?;
- let b = this.read_immediate(b)?;
- let op = match intrinsic_name {
- "fadd_fast" => mir::BinOp::Add,
- "fsub_fast" => mir::BinOp::Sub,
- "fmul_fast" => mir::BinOp::Mul,
- "fdiv_fast" => mir::BinOp::Div,
- "frem_fast" => mir::BinOp::Rem,
- _ => bug!(),
- };
- let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
- Ok(match x.layout.ty.kind() {
- ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
- ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
- _ => bug!(
- "`{intrinsic_name}` called with non-float input type {ty:?}",
- ty = x.layout.ty,
- ),
- })
- };
- match (float_finite(a)?, float_finite(b)?) {
- (false, false) => throw_ub_format!(
- "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
- ),
- (false, _) => throw_ub_format!(
- "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
- ),
- (_, false) => throw_ub_format!(
- "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
- ),
- _ => {}
- }
- this.binop_ignore_overflow(op, &a, &b, dest)?;
- }
-
- #[rustfmt::skip]
- | "minnumf32"
- | "maxnumf32"
- | "copysignf32"
- => {
- let [a, b] = check_arg_count(args)?;
- let a = this.read_scalar(a)?.to_f32()?;
- let b = this.read_scalar(b)?.to_f32()?;
- let res = match intrinsic_name {
- "minnumf32" => a.min(b),
- "maxnumf32" => a.max(b),
- "copysignf32" => a.copy_sign(b),
- _ => bug!(),
- };
- this.write_scalar(Scalar::from_f32(res), dest)?;
- }
-
- #[rustfmt::skip]
- | "minnumf64"
- | "maxnumf64"
- | "copysignf64"
- => {
- let [a, b] = check_arg_count(args)?;
- let a = this.read_scalar(a)?.to_f64()?;
- let b = this.read_scalar(b)?.to_f64()?;
- let res = match intrinsic_name {
- "minnumf64" => a.min(b),
- "maxnumf64" => a.max(b),
- "copysignf64" => a.copy_sign(b),
- _ => bug!(),
- };
- this.write_scalar(Scalar::from_f64(res), dest)?;
- }
-
- "powf32" => {
- let [f, f2] = check_arg_count(args)?;
- // FIXME: Using host floats.
- let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
- let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
- this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
- }
-
- "powf64" => {
- let [f, f2] = check_arg_count(args)?;
- // FIXME: Using host floats.
- let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
- let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
- this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
- }
-
- "fmaf32" => {
- let [a, b, c] = check_arg_count(args)?;
- let a = this.read_scalar(a)?.to_f32()?;
- let b = this.read_scalar(b)?.to_f32()?;
- let c = this.read_scalar(c)?.to_f32()?;
- let res = a.mul_add(b, c).value;
- this.write_scalar(Scalar::from_f32(res), dest)?;
- }
-
- "fmaf64" => {
- let [a, b, c] = check_arg_count(args)?;
- let a = this.read_scalar(a)?.to_f64()?;
- let b = this.read_scalar(b)?.to_f64()?;
- let c = this.read_scalar(c)?.to_f64()?;
- let res = a.mul_add(b, c).value;
- this.write_scalar(Scalar::from_f64(res), dest)?;
- }
-
- "powif32" => {
- let [f, i] = check_arg_count(args)?;
- // FIXME: Using host floats.
- let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
- let i = this.read_scalar(i)?.to_i32()?;
- this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
- }
-
- "powif64" => {
- let [f, i] = check_arg_count(args)?;
- // FIXME: Using host floats.
- let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
- let i = this.read_scalar(i)?.to_i32()?;
- this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
- }
-
- "float_to_int_unchecked" => {
- let [val] = check_arg_count(args)?;
- let val = this.read_immediate(val)?;
-
- let res = match val.layout.ty.kind() {
- ty::Float(FloatTy::F32) =>
- this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
- ty::Float(FloatTy::F64) =>
- this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
- _ =>
- span_bug!(
- this.cur_span(),
- "`float_to_int_unchecked` called with non-float input type {:?}",
- val.layout.ty
- ),
- };
-
- this.write_scalar(res, dest)?;
- }
-
- // SIMD operations
- #[rustfmt::skip]
- | "simd_neg"
- | "simd_fabs"
- | "simd_ceil"
- | "simd_floor"
- | "simd_round"
- | "simd_trunc"
- | "simd_fsqrt" => {
- let [op] = check_arg_count(args)?;
- let (op, op_len) = this.operand_to_simd(op)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- assert_eq!(dest_len, op_len);
-
- #[derive(Copy, Clone)]
- enum HostFloatOp {
- Ceil,
- Floor,
- Round,
- Trunc,
- Sqrt,
- }
- #[derive(Copy, Clone)]
- enum Op {
- MirOp(mir::UnOp),
- Abs,
- HostOp(HostFloatOp),
- }
- let which = match intrinsic_name {
- "simd_neg" => Op::MirOp(mir::UnOp::Neg),
- "simd_fabs" => Op::Abs,
- "simd_ceil" => Op::HostOp(HostFloatOp::Ceil),
- "simd_floor" => Op::HostOp(HostFloatOp::Floor),
- "simd_round" => Op::HostOp(HostFloatOp::Round),
- "simd_trunc" => Op::HostOp(HostFloatOp::Trunc),
- "simd_fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
- _ => unreachable!(),
- };
-
- for i in 0..dest_len {
- let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
- let dest = this.mplace_index(&dest, i)?;
- let val = match which {
- Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
- Op::Abs => {
- // Works for f32 and f64.
- let ty::Float(float_ty) = op.layout.ty.kind() else {
- span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
- };
- let op = op.to_scalar()?;
- match float_ty {
- FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
- FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
- }
- }
- Op::HostOp(host_op) => {
- let ty::Float(float_ty) = op.layout.ty.kind() else {
- span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
- };
- // FIXME using host floats
- match float_ty {
- FloatTy::F32 => {
- let f = f32::from_bits(op.to_scalar()?.to_u32()?);
- let res = match host_op {
- HostFloatOp::Ceil => f.ceil(),
- HostFloatOp::Floor => f.floor(),
- HostFloatOp::Round => f.round(),
- HostFloatOp::Trunc => f.trunc(),
- HostFloatOp::Sqrt => f.sqrt(),
- };
- Scalar::from_u32(res.to_bits())
- }
- FloatTy::F64 => {
- let f = f64::from_bits(op.to_scalar()?.to_u64()?);
- let res = match host_op {
- HostFloatOp::Ceil => f.ceil(),
- HostFloatOp::Floor => f.floor(),
- HostFloatOp::Round => f.round(),
- HostFloatOp::Trunc => f.trunc(),
- HostFloatOp::Sqrt => f.sqrt(),
- };
- Scalar::from_u64(res.to_bits())
- }
- }
-
- }
- };
- this.write_scalar(val, &dest.into())?;
- }
- }
- #[rustfmt::skip]
- | "simd_add"
- | "simd_sub"
- | "simd_mul"
- | "simd_div"
- | "simd_rem"
- | "simd_shl"
- | "simd_shr"
- | "simd_and"
- | "simd_or"
- | "simd_xor"
- | "simd_eq"
- | "simd_ne"
- | "simd_lt"
- | "simd_le"
- | "simd_gt"
- | "simd_ge"
- | "simd_fmax"
- | "simd_fmin"
- | "simd_saturating_add"
- | "simd_saturating_sub"
- | "simd_arith_offset" => {
- use mir::BinOp;
-
- let [left, right] = check_arg_count(args)?;
- let (left, left_len) = this.operand_to_simd(left)?;
- let (right, right_len) = this.operand_to_simd(right)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- assert_eq!(dest_len, left_len);
- assert_eq!(dest_len, right_len);
-
- enum Op {
- MirOp(BinOp),
- SaturatingOp(BinOp),
- FMax,
- FMin,
- WrappingOffset,
- }
- let which = match intrinsic_name {
- "simd_add" => Op::MirOp(BinOp::Add),
- "simd_sub" => Op::MirOp(BinOp::Sub),
- "simd_mul" => Op::MirOp(BinOp::Mul),
- "simd_div" => Op::MirOp(BinOp::Div),
- "simd_rem" => Op::MirOp(BinOp::Rem),
- "simd_shl" => Op::MirOp(BinOp::Shl),
- "simd_shr" => Op::MirOp(BinOp::Shr),
- "simd_and" => Op::MirOp(BinOp::BitAnd),
- "simd_or" => Op::MirOp(BinOp::BitOr),
- "simd_xor" => Op::MirOp(BinOp::BitXor),
- "simd_eq" => Op::MirOp(BinOp::Eq),
- "simd_ne" => Op::MirOp(BinOp::Ne),
- "simd_lt" => Op::MirOp(BinOp::Lt),
- "simd_le" => Op::MirOp(BinOp::Le),
- "simd_gt" => Op::MirOp(BinOp::Gt),
- "simd_ge" => Op::MirOp(BinOp::Ge),
- "simd_fmax" => Op::FMax,
- "simd_fmin" => Op::FMin,
- "simd_saturating_add" => Op::SaturatingOp(BinOp::Add),
- "simd_saturating_sub" => Op::SaturatingOp(BinOp::Sub),
- "simd_arith_offset" => Op::WrappingOffset,
- _ => unreachable!(),
- };
-
- for i in 0..dest_len {
- let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
- let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
- let dest = this.mplace_index(&dest, i)?;
- let val = match which {
- Op::MirOp(mir_op) => {
- let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
- if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
- // Shifts have extra UB as SIMD operations that the MIR binop does not have.
- // See <https://github.com/rust-lang/rust/issues/91237>.
- if overflowed {
- let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
- throw_ub_format!("overflowing shift by {r_val} in `{intrinsic_name}` in SIMD lane {i}");
- }
- }
- if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
- // Special handling for boolean-returning operations
- assert_eq!(ty, this.tcx.types.bool);
- let val = val.to_bool().unwrap();
- bool_to_simd_element(val, dest.layout.size)
- } else {
- assert_ne!(ty, this.tcx.types.bool);
- assert_eq!(ty, dest.layout.ty);
- val
- }
- }
- Op::SaturatingOp(mir_op) => {
- this.saturating_arith(mir_op, &left, &right)?
- }
- Op::WrappingOffset => {
- let ptr = this.scalar_to_ptr(left.to_scalar()?)?;
- let offset_count = right.to_scalar()?.to_machine_isize(this)?;
- let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty;
-
- let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
- let offset_bytes = offset_count.wrapping_mul(pointee_size);
- let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this);
- Scalar::from_maybe_pointer(offset_ptr, this)
- }
- Op::FMax => {
- fmax_op(&left, &right)?
- }
- Op::FMin => {
- fmin_op(&left, &right)?
- }
- };
- this.write_scalar(val, &dest.into())?;
- }
- }
- "simd_fma" => {
- let [a, b, c] = check_arg_count(args)?;
- let (a, a_len) = this.operand_to_simd(a)?;
- let (b, b_len) = this.operand_to_simd(b)?;
- let (c, c_len) = this.operand_to_simd(c)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- assert_eq!(dest_len, a_len);
- assert_eq!(dest_len, b_len);
- assert_eq!(dest_len, c_len);
-
- for i in 0..dest_len {
- let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
- let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
- let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
- let dest = this.mplace_index(&dest, i)?;
-
- // Works for f32 and f64.
- let ty::Float(float_ty) = dest.layout.ty.kind() else {
- span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
- };
- let val = match float_ty {
- FloatTy::F32 =>
- Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
- FloatTy::F64 =>
- Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
- };
- this.write_scalar(val, &dest.into())?;
- }
- }
- #[rustfmt::skip]
- | "simd_reduce_and"
- | "simd_reduce_or"
- | "simd_reduce_xor"
- | "simd_reduce_any"
- | "simd_reduce_all"
- | "simd_reduce_max"
- | "simd_reduce_min" => {
- use mir::BinOp;
-
- let [op] = check_arg_count(args)?;
- let (op, op_len) = this.operand_to_simd(op)?;
-
- let imm_from_bool =
- |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
-
- enum Op {
- MirOp(BinOp),
- MirOpBool(BinOp),
- Max,
- Min,
- }
- let which = match intrinsic_name {
- "simd_reduce_and" => Op::MirOp(BinOp::BitAnd),
- "simd_reduce_or" => Op::MirOp(BinOp::BitOr),
- "simd_reduce_xor" => Op::MirOp(BinOp::BitXor),
- "simd_reduce_any" => Op::MirOpBool(BinOp::BitOr),
- "simd_reduce_all" => Op::MirOpBool(BinOp::BitAnd),
- "simd_reduce_max" => Op::Max,
- "simd_reduce_min" => Op::Min,
- _ => unreachable!(),
- };
-
- // Initialize with first lane, then proceed with the rest.
- let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
- if matches!(which, Op::MirOpBool(_)) {
- // Convert to `bool` scalar.
- res = imm_from_bool(simd_element_to_bool(res)?);
- }
- for i in 1..op_len {
- let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
- res = match which {
- Op::MirOp(mir_op) => {
- this.binary_op(mir_op, &res, &op)?
- }
- Op::MirOpBool(mir_op) => {
- let op = imm_from_bool(simd_element_to_bool(op)?);
- this.binary_op(mir_op, &res, &op)?
- }
- Op::Max => {
- if matches!(res.layout.ty.kind(), ty::Float(_)) {
- ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
- } else {
- // Just boring integers, so NaNs to worry about
- if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
- res
- } else {
- op
- }
- }
- }
- Op::Min => {
- if matches!(res.layout.ty.kind(), ty::Float(_)) {
- ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
- } else {
- // Just boring integers, so NaNs to worry about
- if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
- res
- } else {
- op
- }
- }
- }
- };
- }
- this.write_immediate(*res, dest)?;
- }
- #[rustfmt::skip]
- | "simd_reduce_add_ordered"
- | "simd_reduce_mul_ordered" => {
- use mir::BinOp;
-
- let [op, init] = check_arg_count(args)?;
- let (op, op_len) = this.operand_to_simd(op)?;
- let init = this.read_immediate(init)?;
-
- let mir_op = match intrinsic_name {
- "simd_reduce_add_ordered" => BinOp::Add,
- "simd_reduce_mul_ordered" => BinOp::Mul,
- _ => unreachable!(),
- };
-
- let mut res = init;
- for i in 0..op_len {
- let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
- res = this.binary_op(mir_op, &res, &op)?;
- }
- this.write_immediate(*res, dest)?;
- }
- "simd_select" => {
- let [mask, yes, no] = check_arg_count(args)?;
- let (mask, mask_len) = this.operand_to_simd(mask)?;
- let (yes, yes_len) = this.operand_to_simd(yes)?;
- let (no, no_len) = this.operand_to_simd(no)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- assert_eq!(dest_len, mask_len);
- assert_eq!(dest_len, yes_len);
- assert_eq!(dest_len, no_len);
-
- for i in 0..dest_len {
- let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
- let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
- let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
- let dest = this.mplace_index(&dest, i)?;
-
- let val = if simd_element_to_bool(mask)? { yes } else { no };
- this.write_immediate(*val, &dest.into())?;
- }
- }
- "simd_select_bitmask" => {
- let [mask, yes, no] = check_arg_count(args)?;
- let (yes, yes_len) = this.operand_to_simd(yes)?;
- let (no, no_len) = this.operand_to_simd(no)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
- let bitmask_len = dest_len.max(8);
-
- assert!(mask.layout.ty.is_integral());
- assert!(bitmask_len <= 64);
- assert_eq!(bitmask_len, mask.layout.size.bits());
- assert_eq!(dest_len, yes_len);
- assert_eq!(dest_len, no_len);
-
- let mask: u64 = this
- .read_scalar(mask)?
- .check_init()?
- .to_bits(mask.layout.size)?
- .try_into()
- .unwrap();
- for i in 0..dest_len {
- let mask =
- mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
- let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
- let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
- let dest = this.mplace_index(&dest, i)?;
-
- let val = if mask != 0 { yes } else { no };
- this.write_immediate(*val, &dest.into())?;
- }
- for i in dest_len..bitmask_len {
- // If the mask is "padded", ensure that padding is all-zero.
- let mask = mask & (1 << i);
- if mask != 0 {
- throw_ub_format!(
- "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
- );
- }
- }
- }
- #[rustfmt::skip]
- "simd_cast" | "simd_as" => {
- let [op] = check_arg_count(args)?;
- let (op, op_len) = this.operand_to_simd(op)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- assert_eq!(dest_len, op_len);
-
- let safe_cast = intrinsic_name == "simd_as";
-
- for i in 0..dest_len {
- let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
- let dest = this.mplace_index(&dest, i)?;
-
- let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
- // Int-to-(int|float): always safe
- (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
- this.misc_cast(&op, dest.layout.ty)?,
- // Float-to-float: always safe
- (ty::Float(_), ty::Float(_)) =>
- this.misc_cast(&op, dest.layout.ty)?,
- // Float-to-int in safe mode
- (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
- this.misc_cast(&op, dest.layout.ty)?,
- // Float-to-int in unchecked mode
- (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
- this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
- (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
- this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
- _ =>
- throw_unsup_format!(
- "Unsupported SIMD cast from element type {from_ty} to {to_ty}",
- from_ty = op.layout.ty,
- to_ty = dest.layout.ty,
- ),
- };
- this.write_immediate(val, &dest.into())?;
- }
- }
- "simd_shuffle" => {
- let [left, right, index] = check_arg_count(args)?;
- let (left, left_len) = this.operand_to_simd(left)?;
- let (right, right_len) = this.operand_to_simd(right)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- // `index` is an array, not a SIMD type
- let ty::Array(_, index_len) = index.layout.ty.kind() else {
- span_bug!(this.cur_span(), "simd_shuffle index argument has non-array type {}", index.layout.ty)
- };
- let index_len = index_len.eval_usize(*this.tcx, this.param_env());
-
- assert_eq!(left_len, right_len);
- assert_eq!(index_len, dest_len);
-
- for i in 0..dest_len {
- let src_index: u64 = this
- .read_immediate(&this.operand_index(index, i)?)?
- .to_scalar()?
- .to_u32()?
- .into();
- let dest = this.mplace_index(&dest, i)?;
-
- let val = if src_index < left_len {
- this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
- } else if src_index < left_len.checked_add(right_len).unwrap() {
- this.read_immediate(
- &this.mplace_index(&right, src_index - left_len)?.into(),
- )?
- } else {
- span_bug!(
- this.cur_span(),
- "simd_shuffle index {src_index} is out of bounds for 2 vectors of size {left_len}",
- );
- };
- this.write_immediate(*val, &dest.into())?;
- }
- }
- "simd_gather" => {
- let [passthru, ptrs, mask] = check_arg_count(args)?;
- let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
- let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
- let (mask, mask_len) = this.operand_to_simd(mask)?;
- let (dest, dest_len) = this.place_to_simd(dest)?;
-
- assert_eq!(dest_len, passthru_len);
- assert_eq!(dest_len, ptrs_len);
- assert_eq!(dest_len, mask_len);
-
- for i in 0..dest_len {
- let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
- let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
- let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
- let dest = this.mplace_index(&dest, i)?;
-
- let val = if simd_element_to_bool(mask)? {
- let place = this.deref_operand(&ptr.into())?;
- this.read_immediate(&place.into())?
- } else {
- passthru
- };
- this.write_immediate(*val, &dest.into())?;
- }
- }
- "simd_scatter" => {
- let [value, ptrs, mask] = check_arg_count(args)?;
- let (value, value_len) = this.operand_to_simd(value)?;
- let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
- let (mask, mask_len) = this.operand_to_simd(mask)?;
-
- assert_eq!(ptrs_len, value_len);
- assert_eq!(ptrs_len, mask_len);
-
- for i in 0..ptrs_len {
- let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
- let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
- let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
-
- if simd_element_to_bool(mask)? {
- let place = this.deref_operand(&ptr.into())?;
- this.write_immediate(*value, &place.into())?;
- }
- }
- }
- "simd_bitmask" => {
- let [op] = check_arg_count(args)?;
- let (op, op_len) = this.operand_to_simd(op)?;
- let bitmask_len = op_len.max(8);
-
- assert!(dest.layout.ty.is_integral());
- assert!(bitmask_len <= 64);
- assert_eq!(bitmask_len, dest.layout.size.bits());
-
- let mut res = 0u64;
- for i in 0..op_len {
- let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
- if simd_element_to_bool(op)? {
- res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
- }
- }
- this.write_int(res, dest)?;
- }
-
- // Atomic operations
- "atomic_load_seqcst" => this.atomic_load(args, dest, AtomicReadOrd::SeqCst)?,
- "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOrd::Relaxed)?,
- "atomic_load_acquire" => this.atomic_load(args, dest, AtomicReadOrd::Acquire)?,
-
- "atomic_store_seqcst" => this.atomic_store(args, AtomicWriteOrd::SeqCst)?,
- "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOrd::Relaxed)?,
- "atomic_store_release" => this.atomic_store(args, AtomicWriteOrd::Release)?,
-
- "atomic_fence_acquire" => this.atomic_fence(args, AtomicFenceOrd::Acquire)?,
- "atomic_fence_release" => this.atomic_fence(args, AtomicFenceOrd::Release)?,
- "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOrd::AcqRel)?,
- "atomic_fence_seqcst" => this.atomic_fence(args, AtomicFenceOrd::SeqCst)?,
-
- "atomic_singlethreadfence_acquire" =>
- this.compiler_fence(args, AtomicFenceOrd::Acquire)?,
- "atomic_singlethreadfence_release" =>
- this.compiler_fence(args, AtomicFenceOrd::Release)?,
- "atomic_singlethreadfence_acqrel" =>
- this.compiler_fence(args, AtomicFenceOrd::AcqRel)?,
- "atomic_singlethreadfence_seqcst" =>
- this.compiler_fence(args, AtomicFenceOrd::SeqCst)?,
-
- "atomic_xchg_seqcst" => this.atomic_exchange(args, dest, AtomicRwOrd::SeqCst)?,
- "atomic_xchg_acquire" => this.atomic_exchange(args, dest, AtomicRwOrd::Acquire)?,
- "atomic_xchg_release" => this.atomic_exchange(args, dest, AtomicRwOrd::Release)?,
- "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOrd::AcqRel)?,
- "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOrd::Relaxed)?,
-
- #[rustfmt::skip]
- "atomic_cxchg_seqcst_seqcst" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::SeqCst, AtomicReadOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_cxchg_acquire_acquire" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::Acquire, AtomicReadOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_cxchg_release_relaxed" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::Release, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchg_acqrel_acquire" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::AcqRel, AtomicReadOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_cxchg_relaxed_relaxed" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::Relaxed, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchg_acquire_relaxed" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::Acquire, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchg_acqrel_relaxed" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::AcqRel, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchg_seqcst_relaxed" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::SeqCst, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchg_seqcst_acquire" =>
- this.atomic_compare_exchange(args, dest, AtomicRwOrd::SeqCst, AtomicReadOrd::Acquire)?,
-
- #[rustfmt::skip]
- "atomic_cxchgweak_seqcst_seqcst" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::SeqCst, AtomicReadOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_acquire_acquire" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::Acquire, AtomicReadOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_release_relaxed" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::Release, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_acqrel_acquire" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::AcqRel, AtomicReadOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_relaxed_relaxed" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::Relaxed, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_acquire_relaxed" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::Acquire, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_acqrel_relaxed" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::AcqRel, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_seqcst_relaxed" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::SeqCst, AtomicReadOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_cxchgweak_seqcst_acquire" =>
- this.atomic_compare_exchange_weak(args, dest, AtomicRwOrd::SeqCst, AtomicReadOrd::Acquire)?,
-
- #[rustfmt::skip]
- "atomic_or_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_or_acquire" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_or_release" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOrd::Release)?,
- #[rustfmt::skip]
- "atomic_or_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOrd::AcqRel)?,
- #[rustfmt::skip]
- "atomic_or_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_xor_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_xor_acquire" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_xor_release" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOrd::Release)?,
- #[rustfmt::skip]
- "atomic_xor_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOrd::AcqRel)?,
- #[rustfmt::skip]
- "atomic_xor_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_and_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_and_acquire" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_and_release" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOrd::Release)?,
- #[rustfmt::skip]
- "atomic_and_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOrd::AcqRel)?,
- #[rustfmt::skip]
- "atomic_and_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_nand_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_nand_acquire" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_nand_release" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOrd::Release)?,
- #[rustfmt::skip]
- "atomic_nand_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOrd::AcqRel)?,
- #[rustfmt::skip]
- "atomic_nand_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_xadd_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_xadd_acquire" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_xadd_release" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOrd::Release)?,
- #[rustfmt::skip]
- "atomic_xadd_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOrd::AcqRel)?,
- #[rustfmt::skip]
- "atomic_xadd_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOrd::Relaxed)?,
- #[rustfmt::skip]
- "atomic_xsub_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOrd::SeqCst)?,
- #[rustfmt::skip]
- "atomic_xsub_acquire" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOrd::Acquire)?,
- #[rustfmt::skip]
- "atomic_xsub_release" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOrd::Release)?,
- #[rustfmt::skip]
- "atomic_xsub_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOrd::AcqRel)?,
- #[rustfmt::skip]
- "atomic_xsub_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOrd::Relaxed)?,
- "atomic_min_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::SeqCst)?,
- "atomic_min_acquire" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::Acquire)?,
- "atomic_min_release" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::Release)?,
- "atomic_min_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::AcqRel)?,
- "atomic_min_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::Relaxed)?,
- "atomic_max_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::SeqCst)?,
- "atomic_max_acquire" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::Acquire)?,
- "atomic_max_release" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::Release)?,
- "atomic_max_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::AcqRel)?,
- "atomic_max_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::Relaxed)?,
- "atomic_umin_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::SeqCst)?,
- "atomic_umin_acquire" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::Acquire)?,
- "atomic_umin_release" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::Release)?,
- "atomic_umin_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::AcqRel)?,
- "atomic_umin_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOrd::Relaxed)?,
- "atomic_umax_seqcst" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::SeqCst)?,
- "atomic_umax_acquire" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::Acquire)?,
- "atomic_umax_release" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::Release)?,
- "atomic_umax_acqrel" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::AcqRel)?,
- "atomic_umax_relaxed" =>
- this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOrd::Relaxed)?,
-
- // Other
- "exact_div" => {
- let [num, denom] = check_arg_count(args)?;
- this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
- }
-
- "try" => return this.handle_try(args, dest, ret),
-
- "breakpoint" => {
- let [] = check_arg_count(args)?;
- // normally this would raise a SIGTRAP, which aborts if no debugger is connected
- throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
- }
-
- name => throw_unsup_format!("unimplemented intrinsic: `{name}`"),
- }
-
- trace!("{:?}", this.dump_place(**dest));
- this.go_to_block(ret);
- Ok(())
- }
-
- fn atomic_load(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- atomic: AtomicReadOrd,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
-
- let [place] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
-
- // make sure it fits into a scalar; otherwise it cannot be atomic
- let val = this.read_scalar_atomic(&place, atomic)?;
-
- // Check alignment requirements. Atomics must always be aligned to their size,
- // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
- // be 8-aligned).
- let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
- this.check_ptr_access_align(
- place.ptr,
- place.layout.size,
- align,
- CheckInAllocMsg::MemoryAccessTest,
- )?;
- // Perform regular access.
- this.write_scalar(val, dest)?;
- Ok(())
- }
-
- fn atomic_store(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- atomic: AtomicWriteOrd,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
-
- let [place, val] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
- let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
-
- // Check alignment requirements. Atomics must always be aligned to their size,
- // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
- // be 8-aligned).
- let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
- this.check_ptr_access_align(
- place.ptr,
- place.layout.size,
- align,
- CheckInAllocMsg::MemoryAccessTest,
- )?;
-
- // Perform atomic store
- this.write_scalar_atomic(val, &place, atomic)?;
- Ok(())
- }
-
- fn compiler_fence(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- atomic: AtomicFenceOrd,
- ) -> InterpResult<'tcx> {
- let [] = check_arg_count(args)?;
- let _ = atomic;
- //FIXME: compiler fences are currently ignored
- Ok(())
- }
-
- fn atomic_fence(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- atomic: AtomicFenceOrd,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
- let [] = check_arg_count(args)?;
- this.validate_atomic_fence(atomic)?;
- Ok(())
- }
-
- fn atomic_op(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- atomic_op: AtomicOp,
- atomic: AtomicRwOrd,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
-
- let [place, rhs] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
-
- if !place.layout.ty.is_integral() && !place.layout.ty.is_unsafe_ptr() {
- span_bug!(
- this.cur_span(),
- "atomic arithmetic operations only work on integer and raw pointer types",
- );
- }
- let rhs = this.read_immediate(rhs)?;
-
- // Check alignment requirements. Atomics must always be aligned to their size,
- // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
- // be 8-aligned).
- let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
- this.check_ptr_access_align(
- place.ptr,
- place.layout.size,
- align,
- CheckInAllocMsg::MemoryAccessTest,
- )?;
-
- match atomic_op {
- AtomicOp::Min => {
- let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
- this.write_immediate(*old, dest)?; // old value is returned
- Ok(())
- }
- AtomicOp::Max => {
- let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
- this.write_immediate(*old, dest)?; // old value is returned
- Ok(())
- }
- AtomicOp::MirOp(op, neg) => {
- let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
- this.write_immediate(*old, dest)?; // old value is returned
- Ok(())
- }
- }
- }
-
- fn atomic_exchange(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- atomic: AtomicRwOrd,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
-
- let [place, new] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
- let new = this.read_scalar(new)?;
-
- // Check alignment requirements. Atomics must always be aligned to their size,
- // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
- // be 8-aligned).
- let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
- this.check_ptr_access_align(
- place.ptr,
- place.layout.size,
- align,
- CheckInAllocMsg::MemoryAccessTest,
- )?;
-
- let old = this.atomic_exchange_scalar(&place, new, atomic)?;
- this.write_scalar(old, dest)?; // old value is returned
- Ok(())
- }
-
- fn atomic_compare_exchange_impl(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- success: AtomicRwOrd,
- fail: AtomicReadOrd,
- can_fail_spuriously: bool,
- ) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
-
- let [place, expect_old, new] = check_arg_count(args)?;
- let place = this.deref_operand(place)?;
- let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
- let new = this.read_scalar(new)?;
-
- // Check alignment requirements. Atomics must always be aligned to their size,
- // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
- // be 8-aligned).
- let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
- this.check_ptr_access_align(
- place.ptr,
- place.layout.size,
- align,
- CheckInAllocMsg::MemoryAccessTest,
- )?;
-
- let old = this.atomic_compare_exchange_scalar(
- &place,
- &expect_old,
- new,
- success,
- fail,
- can_fail_spuriously,
- )?;
-
- // Return old value.
- this.write_immediate(old, dest)?;
- Ok(())
- }
-
- fn atomic_compare_exchange(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- success: AtomicRwOrd,
- fail: AtomicReadOrd,
- ) -> InterpResult<'tcx> {
- self.atomic_compare_exchange_impl(args, dest, success, fail, false)
- }
-
- fn atomic_compare_exchange_weak(
- &mut self,
- args: &[OpTy<'tcx, Tag>],
- dest: &PlaceTy<'tcx, Tag>,
- success: AtomicRwOrd,
- fail: AtomicReadOrd,
- ) -> InterpResult<'tcx> {
- self.atomic_compare_exchange_impl(args, dest, success, fail, true)
- }
-
- fn float_to_int_unchecked<F>(
- &self,
- f: F,
- dest_ty: ty::Ty<'tcx>,
- ) -> InterpResult<'tcx, Scalar<Tag>>
- where
- F: Float + Into<Scalar<Tag>>,
- {
- let this = self.eval_context_ref();
-
- // Step 1: cut off the fractional part of `f`. The result of this is
- // guaranteed to be precisely representable in IEEE floats.
- let f = f.round_to_integral(Round::TowardZero).value;
-
- // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
- Ok(match dest_ty.kind() {
- // Unsigned
- ty::Uint(t) => {
- let size = Integer::from_uint_ty(this, *t).size();
- let res = f.to_u128(size.bits_usize());
- if res.status.is_empty() {
- // No status flags means there was no further rounding or other loss of precision.
- Scalar::from_uint(res.value, size)
- } else {
- // `f` was not representable in this integer type.
- throw_ub_format!(
- "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
- );
- }
- }
- // Signed
- ty::Int(t) => {
- let size = Integer::from_int_ty(this, *t).size();
- let res = f.to_i128(size.bits_usize());
- if res.status.is_empty() {
- // No status flags means there was no further rounding or other loss of precision.
- Scalar::from_int(res.value, size)
- } else {
- // `f` was not representable in this integer type.
- throw_ub_format!(
- "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
- );
- }
- }
- // Nothing else
- _ =>
- span_bug!(
- this.cur_span(),
- "`float_to_int_unchecked` called with non-int output type {dest_ty:?}"
- ),
- })
- }
-}
-
-fn fmax_op<'tcx>(
- left: &ImmTy<'tcx, Tag>,
- right: &ImmTy<'tcx, Tag>,
-) -> InterpResult<'tcx, Scalar<Tag>> {
- assert_eq!(left.layout.ty, right.layout.ty);
- let ty::Float(float_ty) = left.layout.ty.kind() else {
- bug!("fmax operand is not a float")
- };
- let left = left.to_scalar()?;
- let right = right.to_scalar()?;
- Ok(match float_ty {
- FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
- FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
- })
-}
-
-fn fmin_op<'tcx>(
- left: &ImmTy<'tcx, Tag>,
- right: &ImmTy<'tcx, Tag>,
-) -> InterpResult<'tcx, Scalar<Tag>> {
- assert_eq!(left.layout.ty, right.layout.ty);
- let ty::Float(float_ty) = left.layout.ty.kind() else {
- bug!("fmin operand is not a float")
- };
- let left = left.to_scalar()?;
- let right = right.to_scalar()?;
- Ok(match float_ty {
- FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
- FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
- })
-}
-
-fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
- // SIMD uses all-1 as pattern for "true"
- let val = if b { -1 } else { 0 };
- Scalar::from_int(val, size)
-}
-
-fn simd_element_to_bool(elem: ImmTy<'_, Tag>) -> InterpResult<'_, bool> {
- let val = elem.to_scalar()?.to_int(elem.layout.size)?;
- Ok(match val {
- 0 => false,
- -1 => true,
- _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
- })
-}
-
-fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
- assert!(idx < vec_len);
- match endianess {
- Endian::Little => idx,
- Endian::Big => vec_len - 1 - idx, // reverse order of bits
- }
-}
diff --git a/src/shims/intrinsics/atomic.rs b/src/shims/intrinsics/atomic.rs
new file mode 100644
index 0000000000..78e13a498c
--- /dev/null
+++ b/src/shims/intrinsics/atomic.rs
@@ -0,0 +1,340 @@
+use rustc_middle::{mir, mir::BinOp, ty};
+use rustc_target::abi::Align;
+
+use crate::*;
+use helpers::check_arg_count;
+
+pub enum AtomicOp {
+ /// The `bool` indicates whether the result of the operation should be negated
+ /// (must be a boolean-typed operation).
+ MirOp(mir::BinOp, bool),
+ Max,
+ Min,
+}
+
+impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
+pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
+ /// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed.
+ fn emulate_atomic_intrinsic(
+ &mut self,
+ intrinsic_name: &str,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();
+
+ fn read_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicReadOrd> {
+ Ok(match ord {
+ "seqcst" => AtomicReadOrd::SeqCst,
+ "acquire" => AtomicReadOrd::Acquire,
+ "relaxed" => AtomicReadOrd::Relaxed,
+ _ => throw_unsup_format!("unsupported read ordering `{ord}`"),
+ })
+ }
+
+ fn write_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicWriteOrd> {
+ Ok(match ord {
+ "seqcst" => AtomicWriteOrd::SeqCst,
+ "release" => AtomicWriteOrd::Release,
+ "relaxed" => AtomicWriteOrd::Relaxed,
+ _ => throw_unsup_format!("unsupported write ordering `{ord}`"),
+ })
+ }
+
+ fn rw_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicRwOrd> {
+ Ok(match ord {
+ "seqcst" => AtomicRwOrd::SeqCst,
+ "acqrel" => AtomicRwOrd::AcqRel,
+ "acquire" => AtomicRwOrd::Acquire,
+ "release" => AtomicRwOrd::Release,
+ "relaxed" => AtomicRwOrd::Relaxed,
+ _ => throw_unsup_format!("unsupported read-write ordering `{ord}`"),
+ })
+ }
+
+ fn fence_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicFenceOrd> {
+ Ok(match ord {
+ "seqcst" => AtomicFenceOrd::SeqCst,
+ "acqrel" => AtomicFenceOrd::AcqRel,
+ "acquire" => AtomicFenceOrd::Acquire,
+ "release" => AtomicFenceOrd::Release,
+ _ => throw_unsup_format!("unsupported fence ordering `{ord}`"),
+ })
+ }
+
+ match &*intrinsic_structure {
+ ["load", ord] => this.atomic_load(args, dest, read_ord(ord)?)?,
+ ["store", ord] => this.atomic_store(args, write_ord(ord)?)?,
+
+ ["fence", ord] => this.atomic_fence(args, fence_ord(ord)?)?,
+ ["singlethreadfence", ord] => this.compiler_fence(args, fence_ord(ord)?)?,
+
+ ["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord)?)?,
+ ["cxchg", ord1, ord2] =>
+ this.atomic_compare_exchange(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?,
+ ["cxchgweak", ord1, ord2] =>
+ this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?,
+
+ ["or", ord] =>
+ this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord)?)?,
+ ["xor", ord] =>
+ this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord)?)?,
+ ["and", ord] =>
+ this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord)?)?,
+ ["nand", ord] =>
+ this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord)?)?,
+ ["xadd", ord] =>
+ this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord)?)?,
+ ["xsub", ord] =>
+ this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord)?)?,
+ ["min", ord] => {
+ // Later we will use the type to indicate signed vs unsigned,
+ // so make sure it matches the intrinsic name.
+ assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
+ this.atomic_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?;
+ }
+ ["umin", ord] => {
+ // Later we will use the type to indicate signed vs unsigned,
+ // so make sure it matches the intrinsic name.
+ assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
+ this.atomic_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?;
+ }
+ ["max", ord] => {
+ // Later we will use the type to indicate signed vs unsigned,
+ // so make sure it matches the intrinsic name.
+ assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
+ this.atomic_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?;
+ }
+ ["umax", ord] => {
+ // Later we will use the type to indicate signed vs unsigned,
+ // so make sure it matches the intrinsic name.
+ assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
+ this.atomic_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?;
+ }
+
+ _ => throw_unsup_format!("unimplemented intrinsic: `atomic_{intrinsic_name}`"),
+ }
+ Ok(())
+ }
+
+ fn atomic_load(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ atomic: AtomicReadOrd,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ let [place] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+
+ // make sure it fits into a scalar; otherwise it cannot be atomic
+ let val = this.read_scalar_atomic(&place, atomic)?;
+
+ // Check alignment requirements. Atomics must always be aligned to their size,
+ // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
+ // be 8-aligned).
+ let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
+ this.check_ptr_access_align(
+ place.ptr,
+ place.layout.size,
+ align,
+ CheckInAllocMsg::MemoryAccessTest,
+ )?;
+ // Perform regular access.
+ this.write_scalar(val, dest)?;
+ Ok(())
+ }
+
+ fn atomic_store(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ atomic: AtomicWriteOrd,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ let [place, val] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+ let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
+
+ // Check alignment requirements. Atomics must always be aligned to their size,
+ // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
+ // be 8-aligned).
+ let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
+ this.check_ptr_access_align(
+ place.ptr,
+ place.layout.size,
+ align,
+ CheckInAllocMsg::MemoryAccessTest,
+ )?;
+
+ // Perform atomic store
+ this.write_scalar_atomic(val, &place, atomic)?;
+ Ok(())
+ }
+
+ fn compiler_fence(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ atomic: AtomicFenceOrd,
+ ) -> InterpResult<'tcx> {
+ let [] = check_arg_count(args)?;
+ let _ = atomic;
+ //FIXME: compiler fences are currently ignored
+ Ok(())
+ }
+
+ fn atomic_fence(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ atomic: AtomicFenceOrd,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+ let [] = check_arg_count(args)?;
+ this.validate_atomic_fence(atomic)?;
+ Ok(())
+ }
+
+ fn atomic_op(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ atomic_op: AtomicOp,
+ atomic: AtomicRwOrd,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ let [place, rhs] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+ let rhs = this.read_immediate(rhs)?;
+
+ if !place.layout.ty.is_integral() && !place.layout.ty.is_unsafe_ptr() {
+ span_bug!(
+ this.cur_span(),
+ "atomic arithmetic operations only work on integer and raw pointer types",
+ );
+ }
+ if rhs.layout.ty != place.layout.ty {
+ span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch");
+ }
+
+ // Check alignment requirements. Atomics must always be aligned to their size,
+ // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
+ // be 8-aligned).
+ let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
+ this.check_ptr_access_align(
+ place.ptr,
+ place.layout.size,
+ align,
+ CheckInAllocMsg::MemoryAccessTest,
+ )?;
+
+ match atomic_op {
+ AtomicOp::Min => {
+ let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
+ this.write_immediate(*old, dest)?; // old value is returned
+ Ok(())
+ }
+ AtomicOp::Max => {
+ let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
+ this.write_immediate(*old, dest)?; // old value is returned
+ Ok(())
+ }
+ AtomicOp::MirOp(op, neg) => {
+ let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
+ this.write_immediate(*old, dest)?; // old value is returned
+ Ok(())
+ }
+ }
+ }
+
+ fn atomic_exchange(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ atomic: AtomicRwOrd,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ let [place, new] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+ let new = this.read_scalar(new)?;
+
+ // Check alignment requirements. Atomics must always be aligned to their size,
+ // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
+ // be 8-aligned).
+ let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
+ this.check_ptr_access_align(
+ place.ptr,
+ place.layout.size,
+ align,
+ CheckInAllocMsg::MemoryAccessTest,
+ )?;
+
+ let old = this.atomic_exchange_scalar(&place, new, atomic)?;
+ this.write_scalar(old, dest)?; // old value is returned
+ Ok(())
+ }
+
+ fn atomic_compare_exchange_impl(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ success: AtomicRwOrd,
+ fail: AtomicReadOrd,
+ can_fail_spuriously: bool,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ let [place, expect_old, new] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+ let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
+ let new = this.read_scalar(new)?;
+
+ // Check alignment requirements. Atomics must always be aligned to their size,
+ // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
+ // be 8-aligned).
+ let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
+ this.check_ptr_access_align(
+ place.ptr,
+ place.layout.size,
+ align,
+ CheckInAllocMsg::MemoryAccessTest,
+ )?;
+
+ let old = this.atomic_compare_exchange_scalar(
+ &place,
+ &expect_old,
+ new,
+ success,
+ fail,
+ can_fail_spuriously,
+ )?;
+
+ // Return old value.
+ this.write_immediate(old, dest)?;
+ Ok(())
+ }
+
+ fn atomic_compare_exchange(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ success: AtomicRwOrd,
+ fail: AtomicReadOrd,
+ ) -> InterpResult<'tcx> {
+ self.atomic_compare_exchange_impl(args, dest, success, fail, false)
+ }
+
+ fn atomic_compare_exchange_weak(
+ &mut self,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ success: AtomicRwOrd,
+ fail: AtomicReadOrd,
+ ) -> InterpResult<'tcx> {
+ self.atomic_compare_exchange_impl(args, dest, success, fail, true)
+ }
+}
diff --git a/src/shims/intrinsics/mod.rs b/src/shims/intrinsics/mod.rs
new file mode 100644
index 0000000000..9ffa40f333
--- /dev/null
+++ b/src/shims/intrinsics/mod.rs
@@ -0,0 +1,426 @@
+mod atomic;
+mod simd;
+
+use std::iter;
+
+use log::trace;
+
+use rustc_apfloat::{Float, Round};
+use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
+use rustc_middle::{mir, ty, ty::FloatTy};
+use rustc_target::abi::Integer;
+
+use crate::*;
+use atomic::EvalContextExt as _;
+use helpers::check_arg_count;
+use simd::EvalContextExt as _;
+
+impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
+pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
+ fn call_intrinsic(
+ &mut self,
+ instance: ty::Instance<'tcx>,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ ret: Option<mir::BasicBlock>,
+ _unwind: StackPopUnwind,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ // See if the core engine can handle this intrinsic.
+ if this.emulate_intrinsic(instance, args, dest, ret)? {
+ return Ok(());
+ }
+
+ // All remaining supported intrinsics have a return place.
+ let intrinsic_name = this.tcx.item_name(instance.def_id());
+ let intrinsic_name = intrinsic_name.as_str();
+ let ret = match ret {
+ None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"),
+ Some(p) => p,
+ };
+
+ // Some intrinsics are special and need the "ret".
+ match intrinsic_name {
+ "try" => return this.handle_try(args, dest, ret),
+ _ => {}
+ }
+
+ // The rest jumps to `ret` immediately.
+ this.emulate_intrinsic_by_name(intrinsic_name, args, dest)?;
+
+ trace!("{:?}", this.dump_place(**dest));
+ this.go_to_block(ret);
+ Ok(())
+ }
+
+ /// Emulates a Miri-supported intrinsic (not supported by the core engine).
+ fn emulate_intrinsic_by_name(
+ &mut self,
+ intrinsic_name: &str,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+
+ if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
+ return this.emulate_atomic_intrinsic(name, args, dest);
+ }
+ if let Some(name) = intrinsic_name.strip_prefix("simd_") {
+ return this.emulate_simd_intrinsic(name, args, dest);
+ }
+
+ match intrinsic_name {
+ // Miri overwriting CTFE intrinsics.
+ "ptr_guaranteed_eq" => {
+ let [left, right] = check_arg_count(args)?;
+ let left = this.read_immediate(left)?;
+ let right = this.read_immediate(right)?;
+ this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
+ }
+ "ptr_guaranteed_ne" => {
+ let [left, right] = check_arg_count(args)?;
+ let left = this.read_immediate(left)?;
+ let right = this.read_immediate(right)?;
+ this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
+ }
+ "const_allocate" => {
+ // For now, for compatibility with the run-time implementation of this, we just return null.
+ // See <https://github.com/rust-lang/rust/issues/93935>.
+ this.write_null(dest)?;
+ }
+ "const_deallocate" => {
+ // complete NOP
+ }
+
+ // Raw memory accesses
+ "volatile_load" => {
+ let [place] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+ this.copy_op(&place.into(), dest, /*allow_transmute*/ false)?;
+ }
+ "volatile_store" => {
+ let [place, dest] = check_arg_count(args)?;
+ let place = this.deref_operand(place)?;
+ this.copy_op(dest, &place.into(), /*allow_transmute*/ false)?;
+ }
+
+ "write_bytes" | "volatile_set_memory" => {
+ let [ptr, val_byte, count] = check_arg_count(args)?;
+ let ty = ptr.layout.ty.builtin_deref(true).unwrap().ty;
+ let ty_layout = this.layout_of(ty)?;
+ let val_byte = this.read_scalar(val_byte)?.to_u8()?;
+ let ptr = this.read_pointer(ptr)?;
+ let count = this.read_scalar(count)?.to_machine_usize(this)?;
+ // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
+ // but no actual allocation can be big enough for the difference to be noticeable.
+ let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
+ err_ub_format!("overflow computing total size of `{intrinsic_name}`")
+ })?;
+ this.write_bytes_ptr(
+ ptr,
+ iter::repeat(val_byte).take(byte_count.bytes() as usize),
+ )?;
+ }
+
+ // Floating-point operations
+ "fabsf32" => {
+ let [f] = check_arg_count(args)?;
+ let f = this.read_scalar(f)?.to_f32()?;
+ // Can be implemented in soft-floats.
+ this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
+ }
+ "fabsf64" => {
+ let [f] = check_arg_count(args)?;
+ let f = this.read_scalar(f)?.to_f64()?;
+ // Can be implemented in soft-floats.
+ this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
+ }
+ #[rustfmt::skip]
+ | "sinf32"
+ | "cosf32"
+ | "sqrtf32"
+ | "expf32"
+ | "exp2f32"
+ | "logf32"
+ | "log10f32"
+ | "log2f32"
+ | "floorf32"
+ | "ceilf32"
+ | "truncf32"
+ | "roundf32"
+ => {
+ let [f] = check_arg_count(args)?;
+ // FIXME: Using host floats.
+ let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
+ let f = match intrinsic_name {
+ "sinf32" => f.sin(),
+ "cosf32" => f.cos(),
+ "sqrtf32" => f.sqrt(),
+ "expf32" => f.exp(),
+ "exp2f32" => f.exp2(),
+ "logf32" => f.ln(),
+ "log10f32" => f.log10(),
+ "log2f32" => f.log2(),
+ "floorf32" => f.floor(),
+ "ceilf32" => f.ceil(),
+ "truncf32" => f.trunc(),
+ "roundf32" => f.round(),
+ _ => bug!(),
+ };
+ this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
+ }
+
+ #[rustfmt::skip]
+ | "sinf64"
+ | "cosf64"
+ | "sqrtf64"
+ | "expf64"
+ | "exp2f64"
+ | "logf64"
+ | "log10f64"
+ | "log2f64"
+ | "floorf64"
+ | "ceilf64"
+ | "truncf64"
+ | "roundf64"
+ => {
+ let [f] = check_arg_count(args)?;
+ // FIXME: Using host floats.
+ let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
+ let f = match intrinsic_name {
+ "sinf64" => f.sin(),
+ "cosf64" => f.cos(),
+ "sqrtf64" => f.sqrt(),
+ "expf64" => f.exp(),
+ "exp2f64" => f.exp2(),
+ "logf64" => f.ln(),
+ "log10f64" => f.log10(),
+ "log2f64" => f.log2(),
+ "floorf64" => f.floor(),
+ "ceilf64" => f.ceil(),
+ "truncf64" => f.trunc(),
+ "roundf64" => f.round(),
+ _ => bug!(),
+ };
+ this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
+ }
+
+ #[rustfmt::skip]
+ | "fadd_fast"
+ | "fsub_fast"
+ | "fmul_fast"
+ | "fdiv_fast"
+ | "frem_fast"
+ => {
+ let [a, b] = check_arg_count(args)?;
+ let a = this.read_immediate(a)?;
+ let b = this.read_immediate(b)?;
+ let op = match intrinsic_name {
+ "fadd_fast" => mir::BinOp::Add,
+ "fsub_fast" => mir::BinOp::Sub,
+ "fmul_fast" => mir::BinOp::Mul,
+ "fdiv_fast" => mir::BinOp::Div,
+ "frem_fast" => mir::BinOp::Rem,
+ _ => bug!(),
+ };
+ let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
+ Ok(match x.layout.ty.kind() {
+ ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
+ ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
+ _ => bug!(
+ "`{intrinsic_name}` called with non-float input type {ty:?}",
+ ty = x.layout.ty,
+ ),
+ })
+ };
+ match (float_finite(a)?, float_finite(b)?) {
+ (false, false) => throw_ub_format!(
+ "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
+ ),
+ (false, _) => throw_ub_format!(
+ "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
+ ),
+ (_, false) => throw_ub_format!(
+ "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
+ ),
+ _ => {}
+ }
+ this.binop_ignore_overflow(op, &a, &b, dest)?;
+ }
+
+ #[rustfmt::skip]
+ | "minnumf32"
+ | "maxnumf32"
+ | "copysignf32"
+ => {
+ let [a, b] = check_arg_count(args)?;
+ let a = this.read_scalar(a)?.to_f32()?;
+ let b = this.read_scalar(b)?.to_f32()?;
+ let res = match intrinsic_name {
+ "minnumf32" => a.min(b),
+ "maxnumf32" => a.max(b),
+ "copysignf32" => a.copy_sign(b),
+ _ => bug!(),
+ };
+ this.write_scalar(Scalar::from_f32(res), dest)?;
+ }
+
+ #[rustfmt::skip]
+ | "minnumf64"
+ | "maxnumf64"
+ | "copysignf64"
+ => {
+ let [a, b] = check_arg_count(args)?;
+ let a = this.read_scalar(a)?.to_f64()?;
+ let b = this.read_scalar(b)?.to_f64()?;
+ let res = match intrinsic_name {
+ "minnumf64" => a.min(b),
+ "maxnumf64" => a.max(b),
+ "copysignf64" => a.copy_sign(b),
+ _ => bug!(),
+ };
+ this.write_scalar(Scalar::from_f64(res), dest)?;
+ }
+
+ "powf32" => {
+ let [f, f2] = check_arg_count(args)?;
+ // FIXME: Using host floats.
+ let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
+ let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
+ this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
+ }
+
+ "powf64" => {
+ let [f, f2] = check_arg_count(args)?;
+ // FIXME: Using host floats.
+ let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
+ let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
+ this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
+ }
+
+ "fmaf32" => {
+ let [a, b, c] = check_arg_count(args)?;
+ let a = this.read_scalar(a)?.to_f32()?;
+ let b = this.read_scalar(b)?.to_f32()?;
+ let c = this.read_scalar(c)?.to_f32()?;
+ let res = a.mul_add(b, c).value;
+ this.write_scalar(Scalar::from_f32(res), dest)?;
+ }
+
+ "fmaf64" => {
+ let [a, b, c] = check_arg_count(args)?;
+ let a = this.read_scalar(a)?.to_f64()?;
+ let b = this.read_scalar(b)?.to_f64()?;
+ let c = this.read_scalar(c)?.to_f64()?;
+ let res = a.mul_add(b, c).value;
+ this.write_scalar(Scalar::from_f64(res), dest)?;
+ }
+
+ "powif32" => {
+ let [f, i] = check_arg_count(args)?;
+ // FIXME: Using host floats.
+ let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
+ let i = this.read_scalar(i)?.to_i32()?;
+ this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
+ }
+
+ "powif64" => {
+ let [f, i] = check_arg_count(args)?;
+ // FIXME: Using host floats.
+ let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
+ let i = this.read_scalar(i)?.to_i32()?;
+ this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
+ }
+
+ "float_to_int_unchecked" => {
+ let [val] = check_arg_count(args)?;
+ let val = this.read_immediate(val)?;
+
+ let res = match val.layout.ty.kind() {
+ ty::Float(FloatTy::F32) =>
+ this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
+ ty::Float(FloatTy::F64) =>
+ this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
+ _ =>
+ span_bug!(
+ this.cur_span(),
+ "`float_to_int_unchecked` called with non-float input type {:?}",
+ val.layout.ty
+ ),
+ };
+
+ this.write_scalar(res, dest)?;
+ }
+
+ // Other
+ "exact_div" => {
+ let [num, denom] = check_arg_count(args)?;
+ this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
+ }
+
+ "breakpoint" => {
+ let [] = check_arg_count(args)?;
+ // normally this would raise a SIGTRAP, which aborts if no debugger is connected
+ throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
+ }
+
+ name => throw_unsup_format!("unimplemented intrinsic: `{name}`"),
+ }
+
+ Ok(())
+ }
+
+ fn float_to_int_unchecked<F>(
+ &self,
+ f: F,
+ dest_ty: ty::Ty<'tcx>,
+ ) -> InterpResult<'tcx, Scalar<Tag>>
+ where
+ F: Float + Into<Scalar<Tag>>,
+ {
+ let this = self.eval_context_ref();
+
+ // Step 1: cut off the fractional part of `f`. The result of this is
+ // guaranteed to be precisely representable in IEEE floats.
+ let f = f.round_to_integral(Round::TowardZero).value;
+
+ // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
+ Ok(match dest_ty.kind() {
+ // Unsigned
+ ty::Uint(t) => {
+ let size = Integer::from_uint_ty(this, *t).size();
+ let res = f.to_u128(size.bits_usize());
+ if res.status.is_empty() {
+ // No status flags means there was no further rounding or other loss of precision.
+ Scalar::from_uint(res.value, size)
+ } else {
+ // `f` was not representable in this integer type.
+ throw_ub_format!(
+ "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
+ );
+ }
+ }
+ // Signed
+ ty::Int(t) => {
+ let size = Integer::from_int_ty(this, *t).size();
+ let res = f.to_i128(size.bits_usize());
+ if res.status.is_empty() {
+ // No status flags means there was no further rounding or other loss of precision.
+ Scalar::from_int(res.value, size)
+ } else {
+ // `f` was not representable in this integer type.
+ throw_ub_format!(
+ "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
+ );
+ }
+ }
+ // Nothing else
+ _ =>
+ span_bug!(
+ this.cur_span(),
+ "`float_to_int_unchecked` called with non-int output type {dest_ty:?}"
+ ),
+ })
+ }
+}
diff --git a/src/shims/intrinsics/simd.rs b/src/shims/intrinsics/simd.rs
new file mode 100644
index 0000000000..fe5250ed08
--- /dev/null
+++ b/src/shims/intrinsics/simd.rs
@@ -0,0 +1,613 @@
+use rustc_apfloat::Float;
+use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
+use rustc_middle::{mir, ty, ty::FloatTy};
+use rustc_target::abi::{Endian, HasDataLayout, Size};
+
+use crate::*;
+use helpers::check_arg_count;
+
+impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
+pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
+ /// Calls the simd intrinsic `intrinsic`; the `simd_` prefix has already been removed.
+ fn emulate_simd_intrinsic(
+ &mut self,
+ intrinsic_name: &str,
+ args: &[OpTy<'tcx, Tag>],
+ dest: &PlaceTy<'tcx, Tag>,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+ match intrinsic_name {
+ #[rustfmt::skip]
+ | "neg"
+ | "fabs"
+ | "ceil"
+ | "floor"
+ | "round"
+ | "trunc"
+ | "fsqrt" => {
+ let [op] = check_arg_count(args)?;
+ let (op, op_len) = this.operand_to_simd(op)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ assert_eq!(dest_len, op_len);
+
+ #[derive(Copy, Clone)]
+ enum HostFloatOp {
+ Ceil,
+ Floor,
+ Round,
+ Trunc,
+ Sqrt,
+ }
+ #[derive(Copy, Clone)]
+ enum Op {
+ MirOp(mir::UnOp),
+ Abs,
+ HostOp(HostFloatOp),
+ }
+ let which = match intrinsic_name {
+ "neg" => Op::MirOp(mir::UnOp::Neg),
+ "fabs" => Op::Abs,
+ "ceil" => Op::HostOp(HostFloatOp::Ceil),
+ "floor" => Op::HostOp(HostFloatOp::Floor),
+ "round" => Op::HostOp(HostFloatOp::Round),
+ "trunc" => Op::HostOp(HostFloatOp::Trunc),
+ "fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
+ _ => unreachable!(),
+ };
+
+ for i in 0..dest_len {
+ let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
+ let dest = this.mplace_index(&dest, i)?;
+ let val = match which {
+ Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
+ Op::Abs => {
+ // Works for f32 and f64.
+ let ty::Float(float_ty) = op.layout.ty.kind() else {
+ span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
+ };
+ let op = op.to_scalar()?;
+ match float_ty {
+ FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
+ FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
+ }
+ }
+ Op::HostOp(host_op) => {
+ let ty::Float(float_ty) = op.layout.ty.kind() else {
+ span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
+ };
+ // FIXME using host floats
+ match float_ty {
+ FloatTy::F32 => {
+ let f = f32::from_bits(op.to_scalar()?.to_u32()?);
+ let res = match host_op {
+ HostFloatOp::Ceil => f.ceil(),
+ HostFloatOp::Floor => f.floor(),
+ HostFloatOp::Round => f.round(),
+ HostFloatOp::Trunc => f.trunc(),
+ HostFloatOp::Sqrt => f.sqrt(),
+ };
+ Scalar::from_u32(res.to_bits())
+ }
+ FloatTy::F64 => {
+ let f = f64::from_bits(op.to_scalar()?.to_u64()?);
+ let res = match host_op {
+ HostFloatOp::Ceil => f.ceil(),
+ HostFloatOp::Floor => f.floor(),
+ HostFloatOp::Round => f.round(),
+ HostFloatOp::Trunc => f.trunc(),
+ HostFloatOp::Sqrt => f.sqrt(),
+ };
+ Scalar::from_u64(res.to_bits())
+ }
+ }
+
+ }
+ };
+ this.write_scalar(val, &dest.into())?;
+ }
+ }
+ #[rustfmt::skip]
+ | "add"
+ | "sub"
+ | "mul"
+ | "div"
+ | "rem"
+ | "shl"
+ | "shr"
+ | "and"
+ | "or"
+ | "xor"
+ | "eq"
+ | "ne"
+ | "lt"
+ | "le"
+ | "gt"
+ | "ge"
+ | "fmax"
+ | "fmin"
+ | "saturating_add"
+ | "saturating_sub"
+ | "arith_offset" => {
+ use mir::BinOp;
+
+ let [left, right] = check_arg_count(args)?;
+ let (left, left_len) = this.operand_to_simd(left)?;
+ let (right, right_len) = this.operand_to_simd(right)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ assert_eq!(dest_len, left_len);
+ assert_eq!(dest_len, right_len);
+
+ enum Op {
+ MirOp(BinOp),
+ SaturatingOp(BinOp),
+ FMax,
+ FMin,
+ WrappingOffset,
+ }
+ let which = match intrinsic_name {
+ "add" => Op::MirOp(BinOp::Add),
+ "sub" => Op::MirOp(BinOp::Sub),
+ "mul" => Op::MirOp(BinOp::Mul),
+ "div" => Op::MirOp(BinOp::Div),
+ "rem" => Op::MirOp(BinOp::Rem),
+ "shl" => Op::MirOp(BinOp::Shl),
+ "shr" => Op::MirOp(BinOp::Shr),
+ "and" => Op::MirOp(BinOp::BitAnd),
+ "or" => Op::MirOp(BinOp::BitOr),
+ "xor" => Op::MirOp(BinOp::BitXor),
+ "eq" => Op::MirOp(BinOp::Eq),
+ "ne" => Op::MirOp(BinOp::Ne),
+ "lt" => Op::MirOp(BinOp::Lt),
+ "le" => Op::MirOp(BinOp::Le),
+ "gt" => Op::MirOp(BinOp::Gt),
+ "ge" => Op::MirOp(BinOp::Ge),
+ "fmax" => Op::FMax,
+ "fmin" => Op::FMin,
+ "saturating_add" => Op::SaturatingOp(BinOp::Add),
+ "saturating_sub" => Op::SaturatingOp(BinOp::Sub),
+ "arith_offset" => Op::WrappingOffset,
+ _ => unreachable!(),
+ };
+
+ for i in 0..dest_len {
+ let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
+ let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
+ let dest = this.mplace_index(&dest, i)?;
+ let val = match which {
+ Op::MirOp(mir_op) => {
+ let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
+ if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
+ // Shifts have extra UB as SIMD operations that the MIR binop does not have.
+ // See <https://github.com/rust-lang/rust/issues/91237>.
+ if overflowed {
+ let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
+ throw_ub_format!("overflowing shift by {r_val} in `simd_{intrinsic_name}` in SIMD lane {i}");
+ }
+ }
+ if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
+ // Special handling for boolean-returning operations
+ assert_eq!(ty, this.tcx.types.bool);
+ let val = val.to_bool().unwrap();
+ bool_to_simd_element(val, dest.layout.size)
+ } else {
+ assert_ne!(ty, this.tcx.types.bool);
+ assert_eq!(ty, dest.layout.ty);
+ val
+ }
+ }
+ Op::SaturatingOp(mir_op) => {
+ this.saturating_arith(mir_op, &left, &right)?
+ }
+ Op::WrappingOffset => {
+ let ptr = this.scalar_to_ptr(left.to_scalar()?)?;
+ let offset_count = right.to_scalar()?.to_machine_isize(this)?;
+ let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty;
+
+ let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
+ let offset_bytes = offset_count.wrapping_mul(pointee_size);
+ let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this);
+ Scalar::from_maybe_pointer(offset_ptr, this)
+ }
+ Op::FMax => {
+ fmax_op(&left, &right)?
+ }
+ Op::FMin => {
+ fmin_op(&left, &right)?
+ }
+ };
+ this.write_scalar(val, &dest.into())?;
+ }
+ }
+ "fma" => {
+ let [a, b, c] = check_arg_count(args)?;
+ let (a, a_len) = this.operand_to_simd(a)?;
+ let (b, b_len) = this.operand_to_simd(b)?;
+ let (c, c_len) = this.operand_to_simd(c)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ assert_eq!(dest_len, a_len);
+ assert_eq!(dest_len, b_len);
+ assert_eq!(dest_len, c_len);
+
+ for i in 0..dest_len {
+ let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
+ let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
+ let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
+ let dest = this.mplace_index(&dest, i)?;
+
+ // Works for f32 and f64.
+ let ty::Float(float_ty) = dest.layout.ty.kind() else {
+ span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
+ };
+ let val = match float_ty {
+ FloatTy::F32 =>
+ Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
+ FloatTy::F64 =>
+ Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
+ };
+ this.write_scalar(val, &dest.into())?;
+ }
+ }
+ #[rustfmt::skip]
+ | "reduce_and"
+ | "reduce_or"
+ | "reduce_xor"
+ | "reduce_any"
+ | "reduce_all"
+ | "reduce_max"
+ | "reduce_min" => {
+ use mir::BinOp;
+
+ let [op] = check_arg_count(args)?;
+ let (op, op_len) = this.operand_to_simd(op)?;
+
+ let imm_from_bool =
+ |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
+
+ enum Op {
+ MirOp(BinOp),
+ MirOpBool(BinOp),
+ Max,
+ Min,
+ }
+ let which = match intrinsic_name {
+ "reduce_and" => Op::MirOp(BinOp::BitAnd),
+ "reduce_or" => Op::MirOp(BinOp::BitOr),
+ "reduce_xor" => Op::MirOp(BinOp::BitXor),
+ "reduce_any" => Op::MirOpBool(BinOp::BitOr),
+ "reduce_all" => Op::MirOpBool(BinOp::BitAnd),
+ "reduce_max" => Op::Max,
+ "reduce_min" => Op::Min,
+ _ => unreachable!(),
+ };
+
+ // Initialize with first lane, then proceed with the rest.
+ let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
+ if matches!(which, Op::MirOpBool(_)) {
+ // Convert to `bool` scalar.
+ res = imm_from_bool(simd_element_to_bool(res)?);
+ }
+ for i in 1..op_len {
+ let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
+ res = match which {
+ Op::MirOp(mir_op) => {
+ this.binary_op(mir_op, &res, &op)?
+ }
+ Op::MirOpBool(mir_op) => {
+ let op = imm_from_bool(simd_element_to_bool(op)?);
+ this.binary_op(mir_op, &res, &op)?
+ }
+ Op::Max => {
+ if matches!(res.layout.ty.kind(), ty::Float(_)) {
+ ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
+ } else {
+ // Just boring integers, so NaNs to worry about
+ if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
+ res
+ } else {
+ op
+ }
+ }
+ }
+ Op::Min => {
+ if matches!(res.layout.ty.kind(), ty::Float(_)) {
+ ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
+ } else {
+ // Just boring integers, so NaNs to worry about
+ if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
+ res
+ } else {
+ op
+ }
+ }
+ }
+ };
+ }
+ this.write_immediate(*res, dest)?;
+ }
+ #[rustfmt::skip]
+ | "reduce_add_ordered"
+ | "reduce_mul_ordered" => {
+ use mir::BinOp;
+
+ let [op, init] = check_arg_count(args)?;
+ let (op, op_len) = this.operand_to_simd(op)?;
+ let init = this.read_immediate(init)?;
+
+ let mir_op = match intrinsic_name {
+ "reduce_add_ordered" => BinOp::Add,
+ "reduce_mul_ordered" => BinOp::Mul,
+ _ => unreachable!(),
+ };
+
+ let mut res = init;
+ for i in 0..op_len {
+ let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
+ res = this.binary_op(mir_op, &res, &op)?;
+ }
+ this.write_immediate(*res, dest)?;
+ }
+ "select" => {
+ let [mask, yes, no] = check_arg_count(args)?;
+ let (mask, mask_len) = this.operand_to_simd(mask)?;
+ let (yes, yes_len) = this.operand_to_simd(yes)?;
+ let (no, no_len) = this.operand_to_simd(no)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ assert_eq!(dest_len, mask_len);
+ assert_eq!(dest_len, yes_len);
+ assert_eq!(dest_len, no_len);
+
+ for i in 0..dest_len {
+ let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
+ let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
+ let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
+ let dest = this.mplace_index(&dest, i)?;
+
+ let val = if simd_element_to_bool(mask)? { yes } else { no };
+ this.write_immediate(*val, &dest.into())?;
+ }
+ }
+ "select_bitmask" => {
+ let [mask, yes, no] = check_arg_count(args)?;
+ let (yes, yes_len) = this.operand_to_simd(yes)?;
+ let (no, no_len) = this.operand_to_simd(no)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+ let bitmask_len = dest_len.max(8);
+
+ assert!(mask.layout.ty.is_integral());
+ assert!(bitmask_len <= 64);
+ assert_eq!(bitmask_len, mask.layout.size.bits());
+ assert_eq!(dest_len, yes_len);
+ assert_eq!(dest_len, no_len);
+
+ let mask: u64 = this
+ .read_scalar(mask)?
+ .check_init()?
+ .to_bits(mask.layout.size)?
+ .try_into()
+ .unwrap();
+ for i in 0..dest_len {
+ let mask =
+ mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
+ let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
+ let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
+ let dest = this.mplace_index(&dest, i)?;
+
+ let val = if mask != 0 { yes } else { no };
+ this.write_immediate(*val, &dest.into())?;
+ }
+ for i in dest_len..bitmask_len {
+ // If the mask is "padded", ensure that padding is all-zero.
+ let mask = mask & (1 << i);
+ if mask != 0 {
+ throw_ub_format!(
+ "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
+ );
+ }
+ }
+ }
+ #[rustfmt::skip]
+ "cast" | "as" => {
+ let [op] = check_arg_count(args)?;
+ let (op, op_len) = this.operand_to_simd(op)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ assert_eq!(dest_len, op_len);
+
+ let safe_cast = intrinsic_name == "as";
+
+ for i in 0..dest_len {
+ let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
+ let dest = this.mplace_index(&dest, i)?;
+
+ let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
+ // Int-to-(int|float): always safe
+ (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
+ this.misc_cast(&op, dest.layout.ty)?,
+ // Float-to-float: always safe
+ (ty::Float(_), ty::Float(_)) =>
+ this.misc_cast(&op, dest.layout.ty)?,
+ // Float-to-int in safe mode
+ (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
+ this.misc_cast(&op, dest.layout.ty)?,
+ // Float-to-int in unchecked mode
+ (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
+ this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
+ (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
+ this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
+ _ =>
+ throw_unsup_format!(
+ "Unsupported SIMD cast from element type {from_ty} to {to_ty}",
+ from_ty = op.layout.ty,
+ to_ty = dest.layout.ty,
+ ),
+ };
+ this.write_immediate(val, &dest.into())?;
+ }
+ }
+ "shuffle" => {
+ let [left, right, index] = check_arg_count(args)?;
+ let (left, left_len) = this.operand_to_simd(left)?;
+ let (right, right_len) = this.operand_to_simd(right)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ // `index` is an array, not a SIMD type
+ let ty::Array(_, index_len) = index.layout.ty.kind() else {
+ span_bug!(this.cur_span(), "simd_shuffle index argument has non-array type {}", index.layout.ty)
+ };
+ let index_len = index_len.eval_usize(*this.tcx, this.param_env());
+
+ assert_eq!(left_len, right_len);
+ assert_eq!(index_len, dest_len);
+
+ for i in 0..dest_len {
+ let src_index: u64 = this
+ .read_immediate(&this.operand_index(index, i)?)?
+ .to_scalar()?
+ .to_u32()?
+ .into();
+ let dest = this.mplace_index(&dest, i)?;
+
+ let val = if src_index < left_len {
+ this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
+ } else if src_index < left_len.checked_add(right_len).unwrap() {
+ this.read_immediate(
+ &this.mplace_index(&right, src_index - left_len)?.into(),
+ )?
+ } else {
+ span_bug!(
+ this.cur_span(),
+ "simd_shuffle index {src_index} is out of bounds for 2 vectors of size {left_len}",
+ );
+ };
+ this.write_immediate(*val, &dest.into())?;
+ }
+ }
+ "gather" => {
+ let [passthru, ptrs, mask] = check_arg_count(args)?;
+ let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
+ let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
+ let (mask, mask_len) = this.operand_to_simd(mask)?;
+ let (dest, dest_len) = this.place_to_simd(dest)?;
+
+ assert_eq!(dest_len, passthru_len);
+ assert_eq!(dest_len, ptrs_len);
+ assert_eq!(dest_len, mask_len);
+
+ for i in 0..dest_len {
+ let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
+ let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
+ let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
+ let dest = this.mplace_index(&dest, i)?;
+
+ let val = if simd_element_to_bool(mask)? {
+ let place = this.deref_operand(&ptr.into())?;
+ this.read_immediate(&place.into())?
+ } else {
+ passthru
+ };
+ this.write_immediate(*val, &dest.into())?;
+ }
+ }
+ "scatter" => {
+ let [value, ptrs, mask] = check_arg_count(args)?;
+ let (value, value_len) = this.operand_to_simd(value)?;
+ let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
+ let (mask, mask_len) = this.operand_to_simd(mask)?;
+
+ assert_eq!(ptrs_len, value_len);
+ assert_eq!(ptrs_len, mask_len);
+
+ for i in 0..ptrs_len {
+ let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
+ let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
+ let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
+
+ if simd_element_to_bool(mask)? {
+ let place = this.deref_operand(&ptr.into())?;
+ this.write_immediate(*value, &place.into())?;
+ }
+ }
+ }
+ "bitmask" => {
+ let [op] = check_arg_count(args)?;
+ let (op, op_len) = this.operand_to_simd(op)?;
+ let bitmask_len = op_len.max(8);
+
+ assert!(dest.layout.ty.is_integral());
+ assert!(bitmask_len <= 64);
+ assert_eq!(bitmask_len, dest.layout.size.bits());
+
+ let mut res = 0u64;
+ for i in 0..op_len {
+ let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
+ if simd_element_to_bool(op)? {
+ res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
+ }
+ }
+ this.write_int(res, dest)?;
+ }
+
+ name => throw_unsup_format!("unimplemented intrinsic: `simd_{name}`"),
+ }
+ Ok(())
+ }
+}
+
+fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
+ // SIMD uses all-1 as pattern for "true"
+ let val = if b { -1 } else { 0 };
+ Scalar::from_int(val, size)
+}
+
+fn simd_element_to_bool(elem: ImmTy<'_, Tag>) -> InterpResult<'_, bool> {
+ let val = elem.to_scalar()?.to_int(elem.layout.size)?;
+ Ok(match val {
+ 0 => false,
+ -1 => true,
+ _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
+ })
+}
+
+fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
+ assert!(idx < vec_len);
+ match endianess {
+ Endian::Little => idx,
+ Endian::Big => vec_len - 1 - idx, // reverse order of bits
+ }
+}
+
+fn fmax_op<'tcx>(
+ left: &ImmTy<'tcx, Tag>,
+ right: &ImmTy<'tcx, Tag>,
+) -> InterpResult<'tcx, Scalar<Tag>> {
+ assert_eq!(left.layout.ty, right.layout.ty);
+ let ty::Float(float_ty) = left.layout.ty.kind() else {
+ bug!("fmax operand is not a float")
+ };
+ let left = left.to_scalar()?;
+ let right = right.to_scalar()?;
+ Ok(match float_ty {
+ FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
+ FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
+ })
+}
+
+fn fmin_op<'tcx>(
+ left: &ImmTy<'tcx, Tag>,
+ right: &ImmTy<'tcx, Tag>,
+) -> InterpResult<'tcx, Scalar<Tag>> {
+ assert_eq!(left.layout.ty, right.layout.ty);
+ let ty::Float(float_ty) = left.layout.ty.kind() else {
+ bug!("fmin operand is not a float")
+ };
+ let left = left.to_scalar()?;
+ let right = right.to_scalar()?;
+ Ok(match float_ty {
+ FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
+ FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
+ })
+}
diff --git a/src/shims/unix/foreign_items.rs b/src/shims/unix/foreign_items.rs
index 57dfd6f181..cf34b4baec 100644
--- a/src/shims/unix/foreign_items.rs
+++ b/src/shims/unix/foreign_items.rs
@@ -2,7 +2,6 @@ use std::ffi::OsStr;
use log::trace;
-use rustc_middle::mir;
use rustc_middle::ty::layout::LayoutOf;
use rustc_span::Symbol;
use rustc_target::abi::{Align, Size};
@@ -22,7 +21,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
- ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
@@ -533,9 +531,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
// Platform-specific shims
_ => {
match this.tcx.sess.target.os.as_ref() {
- "linux" => return shims::unix::linux::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
- "macos" => return shims::unix::macos::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
- "freebsd" => return shims::unix::freebsd::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
+ "linux" => return shims::unix::linux::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
+ "macos" => return shims::unix::macos::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
+ "freebsd" => return shims::unix::freebsd::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
_ => unreachable!(),
}
}
diff --git a/src/shims/unix/freebsd/foreign_items.rs b/src/shims/unix/freebsd/foreign_items.rs
index 66e339cc4a..2350e5a12c 100644
--- a/src/shims/unix/freebsd/foreign_items.rs
+++ b/src/shims/unix/freebsd/foreign_items.rs
@@ -1,4 +1,3 @@
-use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
@@ -14,7 +13,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
- _ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
match link_name.as_str() {
diff --git a/src/shims/unix/linux/foreign_items.rs b/src/shims/unix/linux/foreign_items.rs
index bbdb1b8a31..efd4e2a8c0 100644
--- a/src/shims/unix/linux/foreign_items.rs
+++ b/src/shims/unix/linux/foreign_items.rs
@@ -1,4 +1,3 @@
-use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
@@ -17,7 +16,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
- _ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
diff --git a/src/shims/unix/linux/sync.rs b/src/shims/unix/linux/sync.rs
index a0e35c730c..a81fdb5e99 100644
--- a/src/shims/unix/linux/sync.rs
+++ b/src/shims/unix/linux/sync.rs
@@ -169,7 +169,7 @@ pub fn futex<'tcx>(
//
// Thankfully, preemptions cannot happen inside a Miri shim, so we do not need to
// do anything special to guarantee fence-load-comparison atomicity.
- this.atomic_fence(&[], AtomicFenceOrd::SeqCst)?;
+ this.validate_atomic_fence(AtomicFenceOrd::SeqCst)?;
// Read an `i32` through the pointer, regardless of any wrapper types.
// It's not uncommon for `addr` to be passed as another type than `*mut i32`, such as `*const AtomicI32`.
let futex_val = this
@@ -240,7 +240,7 @@ pub fn futex<'tcx>(
// Together with the SeqCst fence in futex_wait, this makes sure that futex_wait
// will see the latest value on addr which could be changed by our caller
// before doing the syscall.
- this.atomic_fence(&[], AtomicFenceOrd::SeqCst)?;
+ this.validate_atomic_fence(AtomicFenceOrd::SeqCst)?;
let mut n = 0;
for _ in 0..val {
if let Some(thread) = this.futex_wake(addr_usize, bitset) {
diff --git a/src/shims/unix/macos/foreign_items.rs b/src/shims/unix/macos/foreign_items.rs
index 089a082fa3..58dd40cda3 100644
--- a/src/shims/unix/macos/foreign_items.rs
+++ b/src/shims/unix/macos/foreign_items.rs
@@ -1,4 +1,3 @@
-use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
@@ -15,7 +14,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
- _ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
diff --git a/src/shims/windows/foreign_items.rs b/src/shims/windows/foreign_items.rs
index 28e60a6804..6563434241 100644
--- a/src/shims/windows/foreign_items.rs
+++ b/src/shims/windows/foreign_items.rs
@@ -1,6 +1,5 @@
use std::iter;
-use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::abi::Size;
use rustc_target::spec::abi::Abi;
@@ -18,7 +17,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
- _ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
diff --git a/tests/pass/atomic.rs b/tests/pass/atomic.rs
index 9b82e006fa..e3d80a7891 100644
--- a/tests/pass/atomic.rs
+++ b/tests/pass/atomic.rs
@@ -51,18 +51,22 @@ fn atomic_all_ops() {
static ATOMIC: AtomicIsize = AtomicIsize::new(0);
static ATOMIC_UNSIGNED: AtomicU64 = AtomicU64::new(0);
+ let load_orders = [Relaxed, Acquire, SeqCst];
+ let stored_orders = [Relaxed, Release, SeqCst];
+ let rmw_orders = [Relaxed, Release, Acquire, AcqRel, SeqCst];
+
// loads
- for o in [Relaxed, Acquire, SeqCst] {
+ for o in load_orders {
ATOMIC.load(o);
}
// stores
- for o in [Relaxed, Release, SeqCst] {
+ for o in stored_orders {
ATOMIC.store(1, o);
}
// most RMWs
- for o in [Relaxed, Release, Acquire, AcqRel, SeqCst] {
+ for o in rmw_orders {
ATOMIC.swap(0, o);
ATOMIC.fetch_or(0, o);
ATOMIC.fetch_xor(0, o);
@@ -76,29 +80,13 @@ fn atomic_all_ops() {
ATOMIC_UNSIGNED.fetch_max(0, o);
}
- // RMWs with deparate failure ordering
- ATOMIC.store(0, SeqCst);
- assert_eq!(ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed), Ok(0));
- assert_eq!(ATOMIC.compare_exchange(0, 2, Acquire, Relaxed), Err(1));
- assert_eq!(ATOMIC.compare_exchange(0, 1, Release, Relaxed), Err(1));
- assert_eq!(ATOMIC.compare_exchange(1, 0, AcqRel, Relaxed), Ok(1));
- ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed).ok();
- ATOMIC.compare_exchange(0, 1, Acquire, Acquire).ok();
- ATOMIC.compare_exchange(0, 1, AcqRel, Acquire).ok();
- ATOMIC.compare_exchange(0, 1, SeqCst, Acquire).ok();
- ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst).ok();
-
- ATOMIC.store(0, SeqCst);
- compare_exchange_weak_loop!(ATOMIC, 0, 1, Relaxed, Relaxed);
- assert_eq!(ATOMIC.compare_exchange_weak(0, 2, Acquire, Relaxed), Err(1));
- assert_eq!(ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed), Err(1));
- compare_exchange_weak_loop!(ATOMIC, 1, 0, AcqRel, Relaxed);
- assert_eq!(ATOMIC.load(Relaxed), 0);
- ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed).ok();
- ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire).ok();
- ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire).ok();
- ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire).ok();
- ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst).ok();
+ // RMWs with separate failure ordering
+ for o1 in rmw_orders {
+ for o2 in load_orders {
+ let _res = ATOMIC.compare_exchange(0, 0, o1, o2);
+ let _res = ATOMIC.compare_exchange_weak(0, 0, o1, o2);
+ }
+ }
}
fn atomic_u64() {
@@ -106,7 +94,12 @@ fn atomic_u64() {
ATOMIC.store(1, SeqCst);
assert_eq!(ATOMIC.compare_exchange(0, 0x100, AcqRel, Acquire), Err(1));
+ assert_eq!(ATOMIC.compare_exchange(0, 1, Release, Relaxed), Err(1));
+ assert_eq!(ATOMIC.compare_exchange(1, 0, AcqRel, Relaxed), Ok(1));
+ assert_eq!(ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed), Ok(0));
compare_exchange_weak_loop!(ATOMIC, 1, 0x100, AcqRel, Acquire);
+ assert_eq!(ATOMIC.compare_exchange_weak(0, 2, Acquire, Relaxed), Err(0x100));
+ assert_eq!(ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed), Err(0x100));
assert_eq!(ATOMIC.load(Relaxed), 0x100);
assert_eq!(ATOMIC.fetch_max(0x10, SeqCst), 0x100);