diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 72a4adba8d41..0b8bfe50e092 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -928,6 +928,28 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.write_scalar(Scalar::from_f64(res), dest)?; } + "llvm.prefetch" => { + let [_, rw, loc, ty] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let rw = this.read_scalar(rw)?.to_i32()?; + let loc = this.read_scalar(loc)?.to_i32()?; + let ty = this.read_scalar(ty)?.to_i32()?; + + if ty == 1 { + // Data cache prefetch + + if !matches!(rw, 0 | 1) { + throw_ub_format!("invalid `rw` value passed to `llvm.prefetch`: {}", rw); + } + if !matches!(loc, 0..=3) { + throw_ub_format!("invalid `loc` value passed to `llvm.prefetch`: {}", loc); + } + } else { + throw_unsup_format!("unsupported `llvm.prefetch` type argument: {}", ty); + } + } + // Architecture-specific shims "llvm.x86.addcarry.64" if this.tcx.sess.target.arch == "x86_64" => { // Computes u8+u64+u64, returning tuple (u8,u64) comprising the output carry and truncated sum. @@ -980,6 +1002,12 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } + name if name.starts_with("llvm.x86.sse.") => { + return shims::x86::sse::EvalContextExt::emulate_x86_sse_intrinsic( + this, link_name, abi, args, dest, + ); + } + // Platform-specific shims _ => return match this.tcx.sess.target.os.as_ref() { diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/shims/intrinsics/mod.rs index ca2c1652dc19..524b205aabec 100644 --- a/src/tools/miri/src/shims/intrinsics/mod.rs +++ b/src/tools/miri/src/shims/intrinsics/mod.rs @@ -120,6 +120,12 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.write_bytes_ptr(ptr, iter::repeat(val_byte).take(byte_count.bytes_usize()))?; } + "nontemporal_store" => { + let [place, dest] = check_arg_count(args)?; + let place = this.deref_operand(place)?; + this.copy_op(dest, &place.into(), /*allow_transmute*/ false)?; + } + "ptr_mask" => { let [ptr, mask] = check_arg_count(args)?; diff --git a/src/tools/miri/src/shims/mod.rs b/src/tools/miri/src/shims/mod.rs index 1027b24e3011..5a9574766f3d 100644 --- a/src/tools/miri/src/shims/mod.rs +++ b/src/tools/miri/src/shims/mod.rs @@ -7,6 +7,7 @@ pub mod foreign_items; pub mod intrinsics; pub mod unix; pub mod windows; +mod x86; pub mod dlsym; pub mod env; diff --git a/src/tools/miri/src/shims/x86/mod.rs b/src/tools/miri/src/shims/x86/mod.rs new file mode 100644 index 000000000000..36e673129ded --- /dev/null +++ b/src/tools/miri/src/shims/x86/mod.rs @@ -0,0 +1 @@ +pub(super) mod sse; diff --git a/src/tools/miri/src/shims/x86/sse.rs b/src/tools/miri/src/shims/x86/sse.rs new file mode 100644 index 000000000000..24a06d31f7c1 --- /dev/null +++ b/src/tools/miri/src/shims/x86/sse.rs @@ -0,0 +1,466 @@ +use rustc_apfloat::{ieee::Single, Float as _}; +use rustc_middle::mir; +use rustc_span::Symbol; +use rustc_target::spec::abi::Abi; + +use crate::*; +use shims::foreign_items::EmulateByNameResult; + +impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} +pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { + fn emulate_x86_sse_intrinsic( + &mut self, + link_name: Symbol, + abi: Abi, + args: &[OpTy<'tcx, Provenance>], + dest: &PlaceTy<'tcx, Provenance>, + ) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> { + let this = self.eval_context_mut(); + // Prefix should have already been checked. + let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.sse.").unwrap(); + match unprefixed_name { + "add.ss" | "sub.ss" | "mul.ss" | "div.ss" | "min.ss" | "max.ss" => { + let [left, right] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let which = match unprefixed_name { + "add.ss" => FloatBinOp::Arith(mir::BinOp::Add), + "sub.ss" => FloatBinOp::Arith(mir::BinOp::Sub), + "mul.ss" => FloatBinOp::Arith(mir::BinOp::Mul), + "div.ss" => FloatBinOp::Arith(mir::BinOp::Div), + "min.ss" => FloatBinOp::Min, + "max.ss" => FloatBinOp::Max, + _ => unreachable!(), + }; + + bin_op_ss(this, which, left, right, dest)?; + } + "min.ps" | "max.ps" => { + let [left, right] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let which = match unprefixed_name { + "min.ps" => FloatBinOp::Min, + "max.ps" => FloatBinOp::Max, + _ => unreachable!(), + }; + + bin_op_ps(this, which, left, right, dest)?; + } + "sqrt.ss" | "rcp.ss" | "rsqrt.ss" => { + let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let which = match unprefixed_name { + "sqrt.ss" => FloatUnaryOp::Sqrt, + "rcp.ss" => FloatUnaryOp::Rcp, + "rsqrt.ss" => FloatUnaryOp::Rsqrt, + _ => unreachable!(), + }; + + unary_op_ss(this, which, op, dest)?; + } + "sqrt.ps" | "rcp.ps" | "rsqrt.ps" => { + let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let which = match unprefixed_name { + "sqrt.ps" => FloatUnaryOp::Sqrt, + "rcp.ps" => FloatUnaryOp::Rcp, + "rsqrt.ps" => FloatUnaryOp::Rsqrt, + _ => unreachable!(), + }; + + unary_op_ps(this, which, op, dest)?; + } + "cmp.ss" => { + let [left, right, imm] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let which = match this.read_scalar(imm)?.to_i8()? { + 0 => FloatBinOp::Cmp(FloatCmpOp::Eq), + 1 => FloatBinOp::Cmp(FloatCmpOp::Lt), + 2 => FloatBinOp::Cmp(FloatCmpOp::Le), + 3 => FloatBinOp::Cmp(FloatCmpOp::Unord), + 4 => FloatBinOp::Cmp(FloatCmpOp::Neq), + 5 => FloatBinOp::Cmp(FloatCmpOp::Nlt), + 6 => FloatBinOp::Cmp(FloatCmpOp::Nle), + 7 => FloatBinOp::Cmp(FloatCmpOp::Ord), + imm => { + throw_ub_format!("invalid 3rd parameter of llvm.x86.sse.cmp.ps: {}", imm); + } + }; + + bin_op_ss(this, which, left, right, dest)?; + } + "cmp.ps" => { + let [left, right, imm] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let which = match this.read_scalar(imm)?.to_i8()? { + 0 => FloatBinOp::Cmp(FloatCmpOp::Eq), + 1 => FloatBinOp::Cmp(FloatCmpOp::Lt), + 2 => FloatBinOp::Cmp(FloatCmpOp::Le), + 3 => FloatBinOp::Cmp(FloatCmpOp::Unord), + 4 => FloatBinOp::Cmp(FloatCmpOp::Neq), + 5 => FloatBinOp::Cmp(FloatCmpOp::Nlt), + 6 => FloatBinOp::Cmp(FloatCmpOp::Nle), + 7 => FloatBinOp::Cmp(FloatCmpOp::Ord), + imm => { + throw_ub_format!("invalid 3rd parameter of llvm.x86.sse.cmp.ps: {}", imm); + } + }; + + bin_op_ps(this, which, left, right, dest)?; + } + "comieq.ss" | "comilt.ss" | "comile.ss" | "comigt.ss" | "comige.ss" | "comineq.ss" + | "ucomieq.ss" | "ucomilt.ss" | "ucomile.ss" | "ucomigt.ss" | "ucomige.ss" + | "ucomineq.ss" => { + let [left, right] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + + assert_eq!(left_len, right_len); + + let left = this.read_scalar(&this.mplace_index(&left, 0)?.into())?.to_f32()?; + let right = this.read_scalar(&this.mplace_index(&right, 0)?.into())?.to_f32()?; + let res = match unprefixed_name { + "comieq.ss" | "ucomieq.ss" => left == right, + "comilt.ss" | "ucomilt.ss" => left < right, + "comile.ss" | "ucomile.ss" => left <= right, + "comigt.ss" | "ucomigt.ss" => left > right, + "comige.ss" | "ucomige.ss" => left >= right, + "comineq.ss" | "ucomineq.ss" => left != right, + _ => unreachable!(), + }; + this.write_scalar(Scalar::from_i32(i32::from(res)), dest)?; + } + "cvtss2si" | "cvttss2si" => { + let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + let (op, _) = this.operand_to_simd(op)?; + + let op = this.read_scalar(&this.mplace_index(&op, 0)?.into())?.to_f32()?; + + let rnd = match unprefixed_name { + "cvtss2si" => rustc_apfloat::Round::NearestTiesToEven, + "cvttss2si" => rustc_apfloat::Round::TowardZero, + _ => unreachable!(), + }; + + let mut exact = false; + let cvt = op.to_i128_r(32, rnd, &mut exact); + let res = if cvt.status.intersects( + rustc_apfloat::Status::INVALID_OP + | rustc_apfloat::Status::OVERFLOW + | rustc_apfloat::Status::UNDERFLOW, + ) { + i32::MIN + } else { + cvt.value as i32 + }; + + this.write_scalar(Scalar::from_i32(res), dest)?; + } + "cvtss2si64" | "cvttss2si64" => { + let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + let (op, _) = this.operand_to_simd(op)?; + + let op = this.read_scalar(&this.mplace_index(&op, 0)?.into())?.to_f32()?; + + let rnd = match unprefixed_name { + "cvtss2si64" => rustc_apfloat::Round::NearestTiesToEven, + "cvttss2si64" => rustc_apfloat::Round::TowardZero, + _ => unreachable!(), + }; + + let mut exact = false; + let cvt = op.to_i128_r(64, rnd, &mut exact); + let res = if cvt.status.intersects( + rustc_apfloat::Status::INVALID_OP + | rustc_apfloat::Status::OVERFLOW + | rustc_apfloat::Status::UNDERFLOW, + ) { + i64::MIN + } else { + cvt.value as i64 + }; + + this.write_scalar(Scalar::from_i64(res), dest)?; + } + "cvtsi2ss" => { + let [left, right] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let (left, left_len) = this.operand_to_simd(left)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, left_len); + + let right = this.read_scalar(right)?.to_i32()?; + + let res0 = Scalar::from_f32(Single::from_i128(right.into()).value); + this.write_scalar(res0, &this.mplace_index(&dest, 0)?.into())?; + + for i in 1..dest_len { + let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + this.write_immediate(*left, &dest.into())?; + } + } + "cvtsi642ss" => { + let [left, right] = + this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let (left, left_len) = this.operand_to_simd(left)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, left_len); + + let right = this.read_scalar(right)?.to_i64()?; + + let res0 = Scalar::from_f32(Single::from_i128(right.into()).value); + this.write_scalar(res0, &this.mplace_index(&dest, 0)?.into())?; + + for i in 1..dest_len { + let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + this.write_immediate(*left, &dest.into())?; + } + } + "movmsk.ps" => { + let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + let (op, op_len) = this.operand_to_simd(op)?; + + let mut res = 0; + for i in 0..op_len { + let op = this.read_scalar(&this.mplace_index(&op, i)?.into())?; + let op = op.to_u32()?; + + res |= (op >> 31) << i; + } + + this.write_scalar(Scalar::from_u32(res), dest)?; + } + _ => return Ok(EmulateByNameResult::NotSupported), + } + Ok(EmulateByNameResult::NeedsJumping) + } +} + +#[derive(Copy, Clone)] +enum FloatCmpOp { + Eq, + Lt, + Le, + Unord, + Neq, + Nlt, + Nle, + Ord, +} + +#[derive(Copy, Clone)] +enum FloatBinOp { + Arith(mir::BinOp), + Cmp(FloatCmpOp), + Min, + Max, +} + +fn bin_op_f32<'mir, 'tcx>( + this: &mut crate::MiriInterpCx<'mir, 'tcx>, + which: FloatBinOp, + left: &ImmTy<'tcx, Provenance>, + right: &ImmTy<'tcx, Provenance>, +) -> InterpResult<'tcx, Scalar> { + match which { + FloatBinOp::Arith(which) => { + let (res, _, _) = this.overflowing_binary_op(which, left, right)?; + Ok(res) + } + FloatBinOp::Cmp(which) => { + let left = left.to_scalar().to_f32()?; + let right = right.to_scalar().to_f32()?; + // FIXME: Make sure that these operations match the semantics of cmpps + let res = match which { + FloatCmpOp::Eq => left == right, + FloatCmpOp::Lt => left < right, + FloatCmpOp::Le => left <= right, + FloatCmpOp::Unord => left.is_nan() || right.is_nan(), + FloatCmpOp::Neq => left != right, + FloatCmpOp::Nlt => !(left < right), + FloatCmpOp::Nle => !(left <= right), + FloatCmpOp::Ord => !left.is_nan() && !right.is_nan(), + }; + Ok(Scalar::from_u32(if res { u32::MAX } else { 0 })) + } + FloatBinOp::Min => { + let left = left.to_scalar().to_f32()?; + let right = right.to_scalar().to_f32()?; + if (left == Single::ZERO && right == Single::ZERO) + || left.is_nan() + || right.is_nan() + || left >= right + { + Ok(Scalar::from_f32(right)) + } else { + Ok(Scalar::from_f32(left)) + } + } + FloatBinOp::Max => { + let left = left.to_scalar().to_f32()?; + let right = right.to_scalar().to_f32()?; + if (left == Single::ZERO && right == Single::ZERO) + || left.is_nan() + || right.is_nan() + || left <= right + { + Ok(Scalar::from_f32(right)) + } else { + Ok(Scalar::from_f32(left)) + } + } + } +} + +fn bin_op_ss<'mir, 'tcx>( + this: &mut crate::MiriInterpCx<'mir, 'tcx>, + which: FloatBinOp, + left: &OpTy<'tcx, Provenance>, + right: &OpTy<'tcx, Provenance>, + dest: &PlaceTy<'tcx, Provenance>, +) -> InterpResult<'tcx, ()> { + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, left_len); + assert_eq!(dest_len, right_len); + + let res0 = bin_op_f32( + this, + which, + &this.read_immediate(&this.mplace_index(&left, 0)?.into())?, + &this.read_immediate(&this.mplace_index(&right, 0)?.into())?, + )?; + this.write_scalar(res0, &this.mplace_index(&dest, 0)?.into())?; + + for i in 1..dest_len { + let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + this.write_immediate(*left, &dest.into())?; + } + + Ok(()) +} + +fn bin_op_ps<'mir, 'tcx>( + this: &mut crate::MiriInterpCx<'mir, 'tcx>, + which: FloatBinOp, + left: &OpTy<'tcx, Provenance>, + right: &OpTy<'tcx, Provenance>, + dest: &PlaceTy<'tcx, Provenance>, +) -> InterpResult<'tcx, ()> { + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, left_len); + assert_eq!(dest_len, right_len); + + for i in 0..dest_len { + let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?; + let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let res = bin_op_f32(this, which, &left, &right)?; + this.write_scalar(res, &dest.into())?; + } + + Ok(()) +} + +#[derive(Copy, Clone)] +enum FloatUnaryOp { + Sqrt, + Rcp, + Rsqrt, +} + +fn unary_op_f32<'mir, 'tcx>( + _this: &mut crate::MiriInterpCx<'mir, 'tcx>, + which: FloatUnaryOp, + op: &ImmTy<'tcx, Provenance>, +) -> InterpResult<'tcx, Scalar> { + match which { + FloatUnaryOp::Sqrt => { + let op = op.to_scalar(); + // FIXME using host floats + Ok(Scalar::from_u32(f32::from_bits(op.to_u32()?).sqrt().to_bits())) + } + FloatUnaryOp::Rcp => { + let op = op.to_scalar(); + // Use 1.0001 as a crude way of simulating inaccuracy of rcp + let one = Single::from_bits(1.0001f32.to_bits().into()); + Ok(Scalar::from_f32((one / op.to_f32()?).value)) + } + FloatUnaryOp::Rsqrt => { + let op = op.to_scalar(); + // FIXME using host floats + let sqrt = Single::from_bits(f32::from_bits(op.to_u32()?).sqrt().to_bits().into()); + // Use 1.0001 as a crude way of simulating inaccuracy of rsqrt + let one = Single::from_bits(1.0001f32.to_bits().into()); + Ok(Scalar::from_f32((one / sqrt).value)) + } + } +} + +fn unary_op_ss<'mir, 'tcx>( + this: &mut crate::MiriInterpCx<'mir, 'tcx>, + which: FloatUnaryOp, + op: &OpTy<'tcx, Provenance>, + dest: &PlaceTy<'tcx, Provenance>, +) -> InterpResult<'tcx, ()> { + let (op, op_len) = this.operand_to_simd(op)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, op_len); + + let res0 = + unary_op_f32(this, which, &this.read_immediate(&this.mplace_index(&op, 0)?.into())?)?; + this.write_scalar(res0, &this.mplace_index(&dest, 0)?.into())?; + + for i in 1..dest_len { + let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + this.write_immediate(*op, &dest.into())?; + } + + Ok(()) +} + +fn unary_op_ps<'mir, 'tcx>( + this: &mut crate::MiriInterpCx<'mir, 'tcx>, + which: FloatUnaryOp, + op: &OpTy<'tcx, Provenance>, + dest: &PlaceTy<'tcx, Provenance>, +) -> InterpResult<'tcx, ()> { + let (op, op_len) = this.operand_to_simd(op)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, op_len); + + for i in 0..dest_len { + let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let res = unary_op_f32(this, which, &op)?; + this.write_scalar(res, &dest.into())?; + } + + Ok(()) +} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse.rs b/src/tools/miri/tests/pass/intrinsics-x86-sse.rs new file mode 100644 index 000000000000..ede6d0c524ed --- /dev/null +++ b/src/tools/miri/tests/pass/intrinsics-x86-sse.rs @@ -0,0 +1,1101 @@ +//@only-target-x86_64 + +use std::arch::x86_64::*; +use std::f32::NAN; +use std::mem::transmute; + +fn main() { + assert!(is_x86_feature_detected!("sse")); + + unsafe { + test_sse(); + } +} + +macro_rules! assert_approx_eq { + ($a:expr, $b:expr, $eps:expr) => {{ + let (a, b) = (&$a, &$b); + assert!( + (*a - *b).abs() < $eps, + "assertion failed: `(left !== right)` \ + (left: `{:?}`, right: `{:?}`, expect diff: `{:?}`, real diff: `{:?}`)", + *a, + *b, + $eps, + (*a - *b).abs() + ); + }}; +} + +#[target_feature(enable = "sse")] +unsafe fn test_sse() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86{,_64}/sse.rs + + #[target_feature(enable = "sse")] + unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } + } + + #[target_feature(enable = "sse")] + unsafe fn test_mm_add_ss() { + let a = _mm_set_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_set_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_add_ss(a, b); + assert_eq_m128(r, _mm_set_ps(-1.0, 5.0, 0.0, -15.0)); + } + test_mm_add_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_sub_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_sub_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(99.0, 5.0, 0.0, -10.0)); + } + test_mm_sub_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_mul_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_mul_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(100.0, 5.0, 0.0, -10.0)); + } + test_mm_mul_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_div_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_div_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(0.01, 5.0, 0.0, -10.0)); + } + test_mm_div_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_sqrt_ss() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_sqrt_ss(a); + let e = _mm_setr_ps(2.0, 13.0, 16.0, 100.0); + assert_eq_m128(r, e); + } + test_mm_sqrt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_sqrt_ps() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_sqrt_ps(a); + let e = _mm_setr_ps(2.0, 3.6055512, 4.0, 10.0); + assert_eq_m128(r, e); + } + test_mm_sqrt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rcp_ss() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rcp_ss(a); + let e = _mm_setr_ps(0.24993896, 13.0, 16.0, 100.0); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + assert_approx_eq!(r[0], e[0], 2. * rel_err); + for i in 1..4 { + assert_eq!(r[i], e[i]); + } + } + test_mm_rcp_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rcp_ps() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rcp_ps(a); + let e = _mm_setr_ps(0.24993896, 0.0769043, 0.06248474, 0.0099983215); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + for i in 0..4 { + assert_approx_eq!(r[i], e[i], 2. * rel_err); + } + } + test_mm_rcp_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rsqrt_ss() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rsqrt_ss(a); + let e = _mm_setr_ps(0.49987793, 13.0, 16.0, 100.0); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + assert_approx_eq!(r[0], e[0], 2. * rel_err); + for i in 1..4 { + assert_eq!(r[i], e[i]); + } + } + test_mm_rsqrt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rsqrt_ps() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rsqrt_ps(a); + let e = _mm_setr_ps(0.49987793, 0.2772827, 0.24993896, 0.099990845); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + for i in 0..4 { + assert_approx_eq!(r[i], e[i], 2. * rel_err); + } + } + test_mm_rsqrt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_min_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_min_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0)); + } + test_mm_min_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_min_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_min_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0)); + + // `_mm_min_ps` can **not** be implemented using the `simd_min` rust intrinsic. `simd_min` + // is lowered by the llvm codegen backend to `llvm.minnum.v*` llvm intrinsic. This intrinsic + // doesn't specify how -0.0 is handled. Unfortunately it happens to behave different from + // the `minps` x86 instruction on x86. The `llvm.minnum.v*` llvm intrinsic equals + // `r1` to `a` and `r2` to `b`. + let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); + let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); + let r1: [u8; 16] = transmute(_mm_min_ps(a, b)); + let r2: [u8; 16] = transmute(_mm_min_ps(b, a)); + let a: [u8; 16] = transmute(a); + let b: [u8; 16] = transmute(b); + assert_eq!(r1, b); + assert_eq!(r2, a); + assert_ne!(a, b); // sanity check that -0.0 is actually present + } + test_mm_min_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_max_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_max_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(-1.0, 5.0, 0.0, -10.0)); + } + test_mm_max_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_max_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_max_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(-1.0, 20.0, 0.0, -5.0)); + } + test_mm_max_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpeq_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(-1.0, 5.0, 6.0, 7.0); + let r: [u32; 4] = transmute(_mm_cmpeq_ss(a, b)); + let e: [u32; 4] = transmute(_mm_setr_ps(transmute(0u32), 2.0, 3.0, 4.0)); + assert_eq!(r, e); + + let b2 = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let r2: [u32; 4] = transmute(_mm_cmpeq_ss(a, b2)); + let e2: [u32; 4] = transmute(_mm_setr_ps(transmute(0xffffffffu32), 2.0, 3.0, 4.0)); + assert_eq!(r2, e2); + } + test_mm_cmpeq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmplt_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) < b.extract(0) + let c1 = 0u32; // a.extract(0) < c.extract(0) + let d1 = !0u32; // a.extract(0) < d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmplt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmplt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmplt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmplt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmple_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) <= b.extract(0) + let c1 = !0u32; // a.extract(0) <= c.extract(0) + let d1 = !0u32; // a.extract(0) <= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmple_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmple_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmple_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmple_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpgt_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) > b.extract(0) + let c1 = 0u32; // a.extract(0) > c.extract(0) + let d1 = 0u32; // a.extract(0) > d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpgt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpgt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpgt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpgt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpge_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) >= b.extract(0) + let c1 = !0u32; // a.extract(0) >= c.extract(0) + let d1 = 0u32; // a.extract(0) >= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpge_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpge_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpge_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpge_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpneq_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) != b.extract(0) + let c1 = 0u32; // a.extract(0) != c.extract(0) + let d1 = !0u32; // a.extract(0) != d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpneq_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpneq_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpneq_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpneq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnlt_ss() { + // TODO: this test is exactly the same as for `_mm_cmpge_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence of NaNs (signaling or quiet). If so, we should add tests + // for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) >= b.extract(0) + let c1 = !0u32; // a.extract(0) >= c.extract(0) + let d1 = 0u32; // a.extract(0) >= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpnlt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpnlt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpnlt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpnlt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnle_ss() { + // TODO: this test is exactly the same as for `_mm_cmpgt_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence + // of NaNs (signaling or quiet). If so, we should add tests for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) > b.extract(0) + let c1 = 0u32; // a.extract(0) > c.extract(0) + let d1 = 0u32; // a.extract(0) > d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpnle_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpnle_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpnle_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpnle_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpngt_ss() { + // TODO: this test is exactly the same as for `_mm_cmple_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence of NaNs (signaling or quiet). If so, we should add tests + // for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) <= b.extract(0) + let c1 = !0u32; // a.extract(0) <= c.extract(0) + let d1 = !0u32; // a.extract(0) <= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpngt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpngt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpngt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpngt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnge_ss() { + // TODO: this test is exactly the same as for `_mm_cmplt_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence of NaNs (signaling or quiet). If so, we should add tests + // for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) < b.extract(0) + let c1 = 0u32; // a.extract(0) < c.extract(0) + let d1 = !0u32; // a.extract(0) < d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpnge_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpnge_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpnge_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpnge_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpord_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) ord b.extract(0) + let c1 = 0u32; // a.extract(0) ord c.extract(0) + let d1 = !0u32; // a.extract(0) ord d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpord_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpord_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpord_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpord_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpunord_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) unord b.extract(0) + let c1 = !0u32; // a.extract(0) unord c.extract(0) + let d1 = 0u32; // a.extract(0) unord d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpunord_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpunord_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpunord_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpunord_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpeq_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, fls, tru, fls]; + let r: [u32; 4] = transmute(_mm_cmpeq_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpeq_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmplt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, fls, fls]; + let r: [u32; 4] = transmute(_mm_cmplt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmplt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmple_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, 4.0); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, tru, fls]; + let r: [u32; 4] = transmute(_mm_cmple_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmple_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpgt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, fls, fls]; + let r: [u32; 4] = transmute(_mm_cmpgt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpgt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpge_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, tru, fls]; + let r: [u32; 4] = transmute(_mm_cmpge_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpge_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpneq_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, tru, fls, tru]; + let r: [u32; 4] = transmute(_mm_cmpneq_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpneq_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnlt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, tru, tru]; + let r: [u32; 4] = transmute(_mm_cmpnlt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpnlt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnle_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, fls, tru]; + let r: [u32; 4] = transmute(_mm_cmpnle_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpnle_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpngt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, tru, tru]; + let r: [u32; 4] = transmute(_mm_cmpngt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpngt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnge_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, fls, tru]; + let r: [u32; 4] = transmute(_mm_cmpnge_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpnge_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpord_ps() { + let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); + let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, fls, fls]; + let r: [u32; 4] = transmute(_mm_cmpord_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpord_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpunord_ps() { + let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); + let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, tru, tru]; + let r: [u32; 4] = transmute(_mm_cmpunord_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpunord_ps(); + + + + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comieq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comieq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comieq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comieq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comilt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comilt_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comilt_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comilt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comile_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comile_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comile_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comile_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comigt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 1, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comige_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comige_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comigt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comineq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 1, 1]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comineq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comineq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comineq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomieq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomieq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomieq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomieq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomilt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomilt_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomilt_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomilt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomile_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomile_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomile_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomile_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomigt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 0, 1, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomigt_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomigt_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomigt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomige_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 1, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomige_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomige_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomige_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomineq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 1, 1]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomineq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomineq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomineq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtss_si32() { + let inputs = &[42.0f32, -3.1, 4.0e10, 4.0e-20, NAN, 2147483500.1]; + let result = &[42i32, -3, i32::MIN, 0, i32::MIN, 2147483520]; + for i in 0..inputs.len() { + let x = _mm_setr_ps(inputs[i], 1.0, 3.0, 4.0); + let e = result[i]; + let r = _mm_cvtss_si32(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvtss_si32({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + test_mm_cvtss_si32(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvttss_si32() { + let inputs = &[ + (42.0f32, 42i32), + (-31.4, -31), + (-33.5, -33), + (-34.5, -34), + (10.999, 10), + (-5.99, -5), + (4.0e10, i32::MIN), + (4.0e-10, 0), + (NAN, i32::MIN), + (2147483500.1, 2147483520), + ]; + for i in 0..inputs.len() { + let (xi, e) = inputs[i]; + let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); + let r = _mm_cvttss_si32(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvttss_si32({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + test_mm_cvttss_si32(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtss_f32() { + let a = _mm_setr_ps(312.0134, 5.0, 6.0, 7.0); + assert_eq!(_mm_cvtss_f32(a), 312.0134); + } + test_mm_cvtss_f32(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtsi32_ss() { + let inputs = &[ + (4555i32, 4555.0f32), + (322223333, 322223330.0), + (-432, -432.0), + (-322223333, -322223330.0), + ]; + + for i in 0..inputs.len() { + let (x, f) = inputs[i]; + let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); + let r = _mm_cvtsi32_ss(a, x); + let e = _mm_setr_ps(f, 6.0, 7.0, 8.0); + assert_eq_m128(e, r); + } + } + test_mm_cvtsi32_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtss_si64() { + let inputs = &[ + (42.0f32, 42i64), + (-31.4, -31), + (-33.5, -34), + (-34.5, -34), + (4.0e10, 40_000_000_000), + (4.0e-10, 0), + (f32::NAN, i64::MIN), + (2147483500.1, 2147483520), + (9.223371e18, 9223370937343148032), + ]; + for i in 0..inputs.len() { + let (xi, e) = inputs[i]; + let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); + let r = _mm_cvtss_si64(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvtss_si64({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + test_mm_cvtss_si64(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvttss_si64() { + let inputs = &[ + (42.0f32, 42i64), + (-31.4, -31), + (-33.5, -33), + (-34.5, -34), + (10.999, 10), + (-5.99, -5), + (4.0e10, 40_000_000_000), + (4.0e-10, 0), + (f32::NAN, i64::MIN), + (2147483500.1, 2147483520), + (9.223371e18, 9223370937343148032), + (9.223372e18, i64::MIN), + ]; + for i in 0..inputs.len() { + let (xi, e) = inputs[i]; + let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); + let r = _mm_cvttss_si64(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvttss_si64({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + test_mm_cvttss_si64(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtsi64_ss() { + let inputs = &[ + (4555i64, 4555.0f32), + (322223333, 322223330.0), + (-432, -432.0), + (-322223333, -322223330.0), + (9223372036854775807, 9.223372e18), + (-9223372036854775808, -9.223372e18), + ]; + + for i in 0..inputs.len() { + let (x, f) = inputs[i]; + let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); + let r = _mm_cvtsi64_ss(a, x); + let e = _mm_setr_ps(f, 6.0, 7.0, 8.0); + assert_eq_m128(e, r); + } + } + test_mm_cvtsi64_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_movemask_ps() { + let r = _mm_movemask_ps(_mm_setr_ps(-1.0, 5.0, -5.0, 0.0)); + assert_eq!(r, 0b0101); + + let r = _mm_movemask_ps(_mm_setr_ps(-1.0, -5.0, -5.0, 0.0)); + assert_eq!(r, 0b0111); + } + test_mm_movemask_ps(); + + #[repr(align(16))] + struct Memory { + data: [f32; 4], + } + + #[target_feature(enable = "sse")] + unsafe fn test_mm_stream_ps() { + let a = _mm_set1_ps(7.0); + let mut mem = Memory { data: [-1.0; 4] }; + + _mm_stream_ps(mem.data.as_mut_ptr(), a); + let a: [f32; 4] = transmute(a); + for i in 0..4 { + assert_eq!(mem.data[i], a[i]); + } + } + test_mm_stream_ps(); + + let x = 0i8; + _mm_prefetch(&x, _MM_HINT_T0); + _mm_prefetch(&x, _MM_HINT_T1); + _mm_prefetch(&x, _MM_HINT_T2); + _mm_prefetch(&x, _MM_HINT_NTA); + _mm_prefetch(&x, _MM_HINT_ET0); + _mm_prefetch(&x, _MM_HINT_ET1); +}