diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index b1dbcc744ac60..e323e88f26141 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -146,7 +146,6 @@
#![feature(const_size_of_val)]
#![feature(const_size_of_val_raw)]
#![feature(const_slice_from_ref)]
-#![feature(const_slice_split_at_mut)]
#![feature(const_strict_overflow_ops)]
#![feature(const_swap)]
#![feature(const_try)]
@@ -158,8 +157,6 @@
#![feature(coverage_attribute)]
#![feature(do_not_recommend)]
#![feature(duration_consts_float)]
-#![feature(f128_const)]
-#![feature(f16_const)]
#![feature(internal_impls_macro)]
#![feature(ip)]
#![feature(is_ascii_octdigit)]
diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs
index 133d6e3fc9ae2..764df4fe4b058 100644
--- a/library/core/src/num/f128.rs
+++ b/library/core/src/num/f128.rs
@@ -910,7 +910,7 @@ impl f128 {
/// ```
#[inline]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_bits(self) -> u128 {
// SAFETY: `u128` is a plain old datatype so we can always transmute to it.
@@ -959,7 +959,7 @@ impl f128 {
#[inline]
#[must_use]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
pub const fn from_bits(v: u128) -> Self {
// It turns out the safety issues with sNaN were overblown! Hooray!
// SAFETY: `u128` is a plain old datatype so we can always transmute from it.
@@ -986,7 +986,7 @@ impl f128 {
/// ```
#[inline]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_be_bytes(self) -> [u8; 16] {
self.to_bits().to_be_bytes()
@@ -1012,7 +1012,7 @@ impl f128 {
/// ```
#[inline]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_le_bytes(self) -> [u8; 16] {
self.to_bits().to_le_bytes()
@@ -1049,7 +1049,7 @@ impl f128 {
/// ```
#[inline]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_ne_bytes(self) -> [u8; 16] {
self.to_bits().to_ne_bytes()
@@ -1077,7 +1077,7 @@ impl f128 {
#[inline]
#[must_use]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
pub const fn from_be_bytes(bytes: [u8; 16]) -> Self {
Self::from_bits(u128::from_be_bytes(bytes))
}
@@ -1104,7 +1104,7 @@ impl f128 {
#[inline]
#[must_use]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
pub const fn from_le_bytes(bytes: [u8; 16]) -> Self {
Self::from_bits(u128::from_le_bytes(bytes))
}
@@ -1141,7 +1141,7 @@ impl f128 {
#[inline]
#[must_use]
#[unstable(feature = "f128", issue = "116909")]
- #[rustc_const_unstable(feature = "f128_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f128", issue = "116909")]
pub const fn from_ne_bytes(bytes: [u8; 16]) -> Self {
Self::from_bits(u128::from_ne_bytes(bytes))
}
diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs
index e50f5e7e8fbd5..897fc8c105d46 100644
--- a/library/core/src/num/f16.rs
+++ b/library/core/src/num/f16.rs
@@ -896,7 +896,7 @@ impl f16 {
/// ```
#[inline]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_bits(self) -> u16 {
// SAFETY: `u16` is a plain old datatype so we can always transmute to it.
@@ -944,7 +944,7 @@ impl f16 {
#[inline]
#[must_use]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
pub const fn from_bits(v: u16) -> Self {
// It turns out the safety issues with sNaN were overblown! Hooray!
// SAFETY: `u16` is a plain old datatype so we can always transmute from it.
@@ -970,7 +970,7 @@ impl f16 {
/// ```
#[inline]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_be_bytes(self) -> [u8; 2] {
self.to_bits().to_be_bytes()
@@ -995,7 +995,7 @@ impl f16 {
/// ```
#[inline]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_le_bytes(self) -> [u8; 2] {
self.to_bits().to_le_bytes()
@@ -1033,7 +1033,7 @@ impl f16 {
/// ```
#[inline]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
#[must_use = "this returns the result of the operation, without modifying the original"]
pub const fn to_ne_bytes(self) -> [u8; 2] {
self.to_bits().to_ne_bytes()
@@ -1057,7 +1057,7 @@ impl f16 {
#[inline]
#[must_use]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
pub const fn from_be_bytes(bytes: [u8; 2]) -> Self {
Self::from_bits(u16::from_be_bytes(bytes))
}
@@ -1080,7 +1080,7 @@ impl f16 {
#[inline]
#[must_use]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
pub const fn from_le_bytes(bytes: [u8; 2]) -> Self {
Self::from_bits(u16::from_le_bytes(bytes))
}
@@ -1114,7 +1114,7 @@ impl f16 {
#[inline]
#[must_use]
#[unstable(feature = "f16", issue = "116909")]
- #[rustc_const_unstable(feature = "f16_const", issue = "116909")]
+ #[rustc_const_unstable(feature = "f16", issue = "116909")]
pub const fn from_ne_bytes(bytes: [u8; 2]) -> Self {
Self::from_bits(u16::from_ne_bytes(bytes))
}
diff --git a/library/core/src/slice/mod.rs b/library/core/src/slice/mod.rs
index 754a736b15b3c..90ddc9c1d85d6 100644
--- a/library/core/src/slice/mod.rs
+++ b/library/core/src/slice/mod.rs
@@ -356,7 +356,8 @@ impl<T> [T] {
/// ```
#[inline]
#[stable(feature = "slice_first_last_chunk", since = "1.77.0")]
- #[rustc_const_unstable(feature = "const_slice_first_last_chunk", issue = "111774")]
+ #[rustc_const_stable(feature = "const_slice_first_last_chunk", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
pub const fn first_chunk_mut<const N: usize>(&mut self) -> Option<&mut [T; N]> {
if self.len() < N {
None
@@ -421,7 +422,8 @@ impl<T> [T] {
/// ```
#[inline]
#[stable(feature = "slice_first_last_chunk", since = "1.77.0")]
- #[rustc_const_unstable(feature = "const_slice_first_last_chunk", issue = "111774")]
+ #[rustc_const_stable(feature = "const_slice_first_last_chunk", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
pub const fn split_first_chunk_mut<const N: usize>(
&mut self,
) -> Option<(&mut [T; N], &mut [T])> {
@@ -491,7 +493,8 @@ impl<T> [T] {
/// ```
#[inline]
#[stable(feature = "slice_first_last_chunk", since = "1.77.0")]
- #[rustc_const_unstable(feature = "const_slice_first_last_chunk", issue = "111774")]
+ #[rustc_const_stable(feature = "const_slice_first_last_chunk", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
pub const fn split_last_chunk_mut<const N: usize>(
&mut self,
) -> Option<(&mut [T], &mut [T; N])> {
@@ -560,7 +563,8 @@ impl<T> [T] {
/// ```
#[inline]
#[stable(feature = "slice_first_last_chunk", since = "1.77.0")]
- #[rustc_const_unstable(feature = "const_slice_first_last_chunk", issue = "111774")]
+ #[rustc_const_stable(feature = "const_slice_first_last_chunk", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
pub const fn last_chunk_mut<const N: usize>(&mut self) -> Option<&mut [T; N]> {
if self.len() < N {
None
@@ -1903,7 +1907,8 @@ impl<T> [T] {
#[inline]
#[track_caller]
#[must_use]
- #[rustc_const_unstable(feature = "const_slice_split_at_mut", issue = "101804")]
+ #[rustc_const_stable(feature = "const_slice_split_at_mut", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
pub const fn split_at_mut(&mut self, mid: usize) -> (&mut [T], &mut [T]) {
match self.split_at_mut_checked(mid) {
Some(pair) => pair,
@@ -2005,7 +2010,8 @@ impl<T> [T] {
/// assert_eq!(v, [1, 2, 3, 4, 5, 6]);
/// ```
#[stable(feature = "slice_split_at_unchecked", since = "1.79.0")]
- #[rustc_const_unstable(feature = "const_slice_split_at_mut", issue = "101804")]
+ #[rustc_const_stable(feature = "const_slice_split_at_mut", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
#[inline]
#[must_use]
pub const unsafe fn split_at_mut_unchecked(&mut self, mid: usize) -> (&mut [T], &mut [T]) {
@@ -2105,7 +2111,8 @@ impl<T> [T] {
/// assert_eq!(None, v.split_at_mut_checked(7));
/// ```
#[stable(feature = "split_at_checked", since = "1.80.0")]
- #[rustc_const_unstable(feature = "const_slice_split_at_mut", issue = "101804")]
+ #[rustc_const_stable(feature = "const_slice_split_at_mut", since = "CURRENT_RUSTC_VERSION")]
+ #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))]
#[inline]
#[must_use]
pub const fn split_at_mut_checked(&mut self, mid: usize) -> Option<(&mut [T], &mut [T])> {
diff --git a/library/std/src/sys/sync/condvar/mod.rs b/library/std/src/sys/sync/condvar/mod.rs
index 6849cacf88e76..d0c998a559737 100644
--- a/library/std/src/sys/sync/condvar/mod.rs
+++ b/library/std/src/sys/sync/condvar/mod.rs
@@ -12,7 +12,10 @@ cfg_if::cfg_if! {
))] {
mod futex;
pub use futex::Condvar;
- } else if #[cfg(target_family = "unix")] {
+ } else if #[cfg(any(
+ target_family = "unix",
+ target_os = "teeos",
+ ))] {
mod pthread;
pub use pthread::Condvar;
} else if #[cfg(all(target_os = "windows", target_vendor = "win7"))] {
@@ -24,9 +27,6 @@ cfg_if::cfg_if! {
} else if #[cfg(target_os = "solid_asp3")] {
mod itron;
pub use itron::Condvar;
- } else if #[cfg(target_os = "teeos")] {
- mod teeos;
- pub use teeos::Condvar;
} else if #[cfg(target_os = "xous")] {
mod xous;
pub use xous::Condvar;
diff --git a/library/std/src/sys/sync/condvar/pthread.rs b/library/std/src/sys/sync/condvar/pthread.rs
index 5b5e7770b0627..986cd0cb7d188 100644
--- a/library/std/src/sys/sync/condvar/pthread.rs
+++ b/library/std/src/sys/sync/condvar/pthread.rs
@@ -2,31 +2,25 @@ use crate::cell::UnsafeCell;
use crate::ptr;
use crate::sync::atomic::AtomicPtr;
use crate::sync::atomic::Ordering::Relaxed;
-use crate::sys::sync::{Mutex, mutex};
+use crate::sys::sync::{Mutex, OnceBox};
#[cfg(not(target_os = "nto"))]
use crate::sys::time::TIMESPEC_MAX;
#[cfg(target_os = "nto")]
use crate::sys::time::TIMESPEC_MAX_CAPPED;
-use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::time::Duration;
struct AllocatedCondvar(UnsafeCell<libc::pthread_cond_t>);
pub struct Condvar {
- inner: LazyBox<AllocatedCondvar>,
+ inner: OnceBox<AllocatedCondvar>,
mutex: AtomicPtr<libc::pthread_mutex_t>,
}
-#[inline]
-fn raw(c: &Condvar) -> *mut libc::pthread_cond_t {
- c.inner.0.get()
-}
-
unsafe impl Send for AllocatedCondvar {}
unsafe impl Sync for AllocatedCondvar {}
-impl LazyInit for AllocatedCondvar {
- fn init() -> Box<Self> {
+impl AllocatedCondvar {
+ fn new() -> Box<Self> {
let condvar = Box::new(AllocatedCondvar(UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER)));
cfg_if::cfg_if! {
@@ -37,7 +31,7 @@ impl LazyInit for AllocatedCondvar {
target_vendor = "apple",
))] {
// `pthread_condattr_setclock` is unfortunately not supported on these platforms.
- } else if #[cfg(any(target_os = "espidf", target_os = "horizon"))] {
+ } else if #[cfg(any(target_os = "espidf", target_os = "horizon", target_os = "teeos"))] {
// NOTE: ESP-IDF's PTHREAD_COND_INITIALIZER support is not released yet
// So on that platform, init() should always be called
// Moreover, that platform does not have pthread_condattr_setclock support,
@@ -82,7 +76,11 @@ impl Drop for AllocatedCondvar {
impl Condvar {
pub const fn new() -> Condvar {
- Condvar { inner: LazyBox::new(), mutex: AtomicPtr::new(ptr::null_mut()) }
+ Condvar { inner: OnceBox::new(), mutex: AtomicPtr::new(ptr::null_mut()) }
+ }
+
+ fn get(&self) -> *mut libc::pthread_cond_t {
+ self.inner.get_or_init(AllocatedCondvar::new).0.get()
}
#[inline]
@@ -98,21 +96,21 @@ impl Condvar {
#[inline]
pub fn notify_one(&self) {
- let r = unsafe { libc::pthread_cond_signal(raw(self)) };
+ let r = unsafe { libc::pthread_cond_signal(self.get()) };
debug_assert_eq!(r, 0);
}
#[inline]
pub fn notify_all(&self) {
- let r = unsafe { libc::pthread_cond_broadcast(raw(self)) };
+ let r = unsafe { libc::pthread_cond_broadcast(self.get()) };
debug_assert_eq!(r, 0);
}
#[inline]
pub unsafe fn wait(&self, mutex: &Mutex) {
- let mutex = mutex::raw(mutex);
+ let mutex = mutex.get_assert_locked();
self.verify(mutex);
- let r = libc::pthread_cond_wait(raw(self), mutex);
+ let r = libc::pthread_cond_wait(self.get(), mutex);
debug_assert_eq!(r, 0);
}
@@ -129,7 +127,7 @@ impl Condvar {
pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool {
use crate::sys::time::Timespec;
- let mutex = mutex::raw(mutex);
+ let mutex = mutex.get_assert_locked();
self.verify(mutex);
#[cfg(not(target_os = "nto"))]
@@ -144,7 +142,7 @@ impl Condvar {
.and_then(|t| t.to_timespec_capped())
.unwrap_or(TIMESPEC_MAX_CAPPED);
- let r = libc::pthread_cond_timedwait(raw(self), mutex, &timeout);
+ let r = libc::pthread_cond_timedwait(self.get(), mutex, &timeout);
assert!(r == libc::ETIMEDOUT || r == 0);
r == 0
}
@@ -162,7 +160,7 @@ impl Condvar {
use crate::sys::time::SystemTime;
use crate::time::Instant;
- let mutex = mutex::raw(mutex);
+ let mutex = mutex.get_assert_locked();
self.verify(mutex);
// OSX implementation of `pthread_cond_timedwait` is buggy
@@ -188,7 +186,7 @@ impl Condvar {
.and_then(|t| t.to_timespec())
.unwrap_or(TIMESPEC_MAX);
- let r = libc::pthread_cond_timedwait(raw(self), mutex, &timeout);
+ let r = libc::pthread_cond_timedwait(self.get(), mutex, &timeout);
debug_assert!(r == libc::ETIMEDOUT || r == 0);
// ETIMEDOUT is not a totally reliable method of determining timeout due
diff --git a/library/std/src/sys/sync/condvar/sgx.rs b/library/std/src/sys/sync/condvar/sgx.rs
index ecb5872f60d90..e60715e4b592e 100644
--- a/library/std/src/sys/sync/condvar/sgx.rs
+++ b/library/std/src/sys/sync/condvar/sgx.rs
@@ -1,44 +1,39 @@
use crate::sys::pal::waitqueue::{SpinMutex, WaitQueue, WaitVariable};
-use crate::sys::sync::Mutex;
-use crate::sys_common::lazy_box::{LazyBox, LazyInit};
+use crate::sys::sync::{Mutex, OnceBox};
use crate::time::Duration;
-/// FIXME: `UnsafeList` is not movable.
-struct AllocatedCondvar(SpinMutex<WaitVariable<()>>);
-
pub struct Condvar {
- inner: LazyBox<AllocatedCondvar>,
-}
-
-impl LazyInit for AllocatedCondvar {
- fn init() -> Box<Self> {
- Box::new(AllocatedCondvar(SpinMutex::new(WaitVariable::new(()))))
- }
+ // FIXME: `UnsafeList` is not movable.
+ inner: OnceBox<SpinMutex<WaitVariable<()>>>,
}
impl Condvar {
pub const fn new() -> Condvar {
- Condvar { inner: LazyBox::new() }
+ Condvar { inner: OnceBox::new() }
+ }
+
+ fn get(&self) -> &SpinMutex<WaitVariable<()>> {
+ self.inner.get_or_init(|| Box::new(SpinMutex::new(WaitVariable::new(()))))
}
#[inline]
pub fn notify_one(&self) {
- let _ = WaitQueue::notify_one(self.inner.0.lock());
+ let _ = WaitQueue::notify_one(self.get().lock());
}
#[inline]
pub fn notify_all(&self) {
- let _ = WaitQueue::notify_all(self.inner.0.lock());
+ let _ = WaitQueue::notify_all(self.get().lock());
}
pub unsafe fn wait(&self, mutex: &Mutex) {
- let guard = self.inner.0.lock();
+ let guard = self.get().lock();
WaitQueue::wait(guard, || unsafe { mutex.unlock() });
mutex.lock()
}
pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool {
- let success = WaitQueue::wait_timeout(&self.inner.0, dur, || unsafe { mutex.unlock() });
+ let success = WaitQueue::wait_timeout(self.get(), dur, || unsafe { mutex.unlock() });
mutex.lock();
success
}
diff --git a/library/std/src/sys/sync/condvar/teeos.rs b/library/std/src/sys/sync/condvar/teeos.rs
deleted file mode 100644
index 943867cd76169..0000000000000
--- a/library/std/src/sys/sync/condvar/teeos.rs
+++ /dev/null
@@ -1,101 +0,0 @@
-use crate::cell::UnsafeCell;
-use crate::ptr;
-use crate::sync::atomic::AtomicPtr;
-use crate::sync::atomic::Ordering::Relaxed;
-use crate::sys::sync::mutex::{self, Mutex};
-use crate::sys::time::TIMESPEC_MAX;
-use crate::sys_common::lazy_box::{LazyBox, LazyInit};
-use crate::time::Duration;
-
-extern "C" {
- pub fn pthread_cond_timedwait(
- cond: *mut libc::pthread_cond_t,
- lock: *mut libc::pthread_mutex_t,
- adstime: *const libc::timespec,
- ) -> libc::c_int;
-}
-
-struct AllocatedCondvar(UnsafeCell<libc::pthread_cond_t>);
-
-pub struct Condvar {
- inner: LazyBox<AllocatedCondvar>,
- mutex: AtomicPtr<libc::pthread_mutex_t>,
-}
-
-#[inline]
-fn raw(c: &Condvar) -> *mut libc::pthread_cond_t {
- c.inner.0.get()
-}
-
-unsafe impl Send for AllocatedCondvar {}
-unsafe impl Sync for AllocatedCondvar {}
-
-impl LazyInit for AllocatedCondvar {
- fn init() -> Box<Self> {
- let condvar = Box::new(AllocatedCondvar(UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER)));
-
- let r = unsafe { libc::pthread_cond_init(condvar.0.get(), crate::ptr::null()) };
- assert_eq!(r, 0);
-
- condvar
- }
-}
-
-impl Drop for AllocatedCondvar {
- #[inline]
- fn drop(&mut self) {
- let r = unsafe { libc::pthread_cond_destroy(self.0.get()) };
- debug_assert_eq!(r, 0);
- }
-}
-
-impl Condvar {
- pub const fn new() -> Condvar {
- Condvar { inner: LazyBox::new(), mutex: AtomicPtr::new(ptr::null_mut()) }
- }
-
- #[inline]
- fn verify(&self, mutex: *mut libc::pthread_mutex_t) {
- match self.mutex.compare_exchange(ptr::null_mut(), mutex, Relaxed, Relaxed) {
- Ok(_) => {} // Stored the address
- Err(n) if n == mutex => {} // Lost a race to store the same address
- _ => panic!("attempted to use a condition variable with two mutexes"),
- }
- }
-
- #[inline]
- pub fn notify_one(&self) {
- let r = unsafe { libc::pthread_cond_signal(raw(self)) };
- debug_assert_eq!(r, 0);
- }
-
- #[inline]
- pub fn notify_all(&self) {
- let r = unsafe { libc::pthread_cond_broadcast(raw(self)) };
- debug_assert_eq!(r, 0);
- }
-
- #[inline]
- pub unsafe fn wait(&self, mutex: &Mutex) {
- let mutex = unsafe { mutex::raw(mutex) };
- self.verify(mutex);
- let r = unsafe { libc::pthread_cond_wait(raw(self), mutex) };
- debug_assert_eq!(r, 0);
- }
-
- pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool {
- use crate::sys::time::Timespec;
-
- let mutex = unsafe { mutex::raw(mutex) };
- self.verify(mutex);
-
- let timeout = Timespec::now(libc::CLOCK_MONOTONIC)
- .checked_add_duration(&dur)
- .and_then(|t| t.to_timespec())
- .unwrap_or(TIMESPEC_MAX);
-
- let r = unsafe { pthread_cond_timedwait(raw(self), mutex, &timeout) };
- assert!(r == libc::ETIMEDOUT || r == 0);
- r == 0
- }
-}
diff --git a/library/std/src/sys/sync/mod.rs b/library/std/src/sys/sync/mod.rs
index 52fac5902a296..0691e96785198 100644
--- a/library/std/src/sys/sync/mod.rs
+++ b/library/std/src/sys/sync/mod.rs
@@ -1,11 +1,14 @@
mod condvar;
mod mutex;
mod once;
+mod once_box;
mod rwlock;
mod thread_parking;
pub use condvar::Condvar;
pub use mutex::Mutex;
pub use once::{Once, OnceState};
+#[allow(unused)] // Only used on some platforms.
+use once_box::OnceBox;
pub use rwlock::RwLock;
pub use thread_parking::Parker;
diff --git a/library/std/src/sys/sync/mutex/mod.rs b/library/std/src/sys/sync/mutex/mod.rs
index 73d9bd273de17..360df3fc4b55d 100644
--- a/library/std/src/sys/sync/mutex/mod.rs
+++ b/library/std/src/sys/sync/mutex/mod.rs
@@ -19,7 +19,7 @@ cfg_if::cfg_if! {
target_os = "teeos",
))] {
mod pthread;
- pub use pthread::{Mutex, raw};
+ pub use pthread::Mutex;
} else if #[cfg(all(target_os = "windows", target_vendor = "win7"))] {
mod windows7;
pub use windows7::{Mutex, raw};
diff --git a/library/std/src/sys/sync/mutex/pthread.rs b/library/std/src/sys/sync/mutex/pthread.rs
index 1c407bc253776..87c95f45f964e 100644
--- a/library/std/src/sys/sync/mutex/pthread.rs
+++ b/library/std/src/sys/sync/mutex/pthread.rs
@@ -2,24 +2,19 @@ use crate::cell::UnsafeCell;
use crate::io::Error;
use crate::mem::{MaybeUninit, forget};
use crate::sys::cvt_nz;
-use crate::sys_common::lazy_box::{LazyBox, LazyInit};
+use crate::sys::sync::OnceBox;
struct AllocatedMutex(UnsafeCell<libc::pthread_mutex_t>);
pub struct Mutex {
- inner: LazyBox<AllocatedMutex>,
-}
-
-#[inline]
-pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
- m.inner.0.get()
+ inner: OnceBox<AllocatedMutex>,
}
unsafe impl Send for AllocatedMutex {}
unsafe impl Sync for AllocatedMutex {}
-impl LazyInit for AllocatedMutex {
- fn init() -> Box<Self> {
+impl AllocatedMutex {
+ fn new() -> Box<Self> {
let mutex = Box::new(AllocatedMutex(UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER)));
// Issue #33770
@@ -60,24 +55,6 @@ impl LazyInit for AllocatedMutex {
mutex
}
-
- fn destroy(mutex: Box<Self>) {
- // We're not allowed to pthread_mutex_destroy a locked mutex,
- // so check first if it's unlocked.
- if unsafe { libc::pthread_mutex_trylock(mutex.0.get()) == 0 } {
- unsafe { libc::pthread_mutex_unlock(mutex.0.get()) };
- drop(mutex);
- } else {
- // The mutex is locked. This happens if a MutexGuard is leaked.
- // In this case, we just leak the Mutex too.
- forget(mutex);
- }
- }
-
- fn cancel_init(_: Box<Self>) {
- // In this case, we can just drop it without any checks,
- // since it cannot have been locked yet.
- }
}
impl Drop for AllocatedMutex {
@@ -99,11 +76,33 @@ impl Drop for AllocatedMutex {
impl Mutex {
#[inline]
pub const fn new() -> Mutex {
- Mutex { inner: LazyBox::new() }
+ Mutex { inner: OnceBox::new() }
+ }
+
+ /// Gets access to the pthread mutex under the assumption that the mutex is
+ /// locked.
+ ///
+ /// This allows skipping the initialization check, as the mutex can only be
+ /// locked if it is already initialized, and allows relaxing the ordering
+ /// on the pointer load, since the allocation cannot have been modified
+ /// since the `lock` and the lock must have occurred on the current thread.
+ ///
+ /// # Safety
+ /// Causes undefined behaviour if the mutex is not locked.
+ #[inline]
+ pub(crate) unsafe fn get_assert_locked(&self) -> *mut libc::pthread_mutex_t {
+ unsafe { self.inner.get_unchecked().0.get() }
}
#[inline]
- pub unsafe fn lock(&self) {
+ fn get(&self) -> *mut libc::pthread_mutex_t {
+ // If initialization fails, the mutex is destroyed. This is always sound,
+ // however, as the mutex cannot have been locked yet.
+ self.inner.get_or_init(AllocatedMutex::new).0.get()
+ }
+
+ #[inline]
+ pub fn lock(&self) {
#[cold]
#[inline(never)]
fn fail(r: i32) -> ! {
@@ -111,7 +110,7 @@ impl Mutex {
panic!("failed to lock mutex: {error}");
}
- let r = libc::pthread_mutex_lock(raw(self));
+ let r = unsafe { libc::pthread_mutex_lock(self.get()) };
// As we set the mutex type to `PTHREAD_MUTEX_NORMAL` above, we expect
// the lock call to never fail. Unfortunately however, some platforms
// (Solaris) do not conform to the standard, and instead always provide
@@ -126,13 +125,29 @@ impl Mutex {
#[inline]
pub unsafe fn unlock(&self) {
- let r = libc::pthread_mutex_unlock(raw(self));
+ let r = libc::pthread_mutex_unlock(self.get_assert_locked());
debug_assert_eq!(r, 0);
}
#[inline]
- pub unsafe fn try_lock(&self) -> bool {
- libc::pthread_mutex_trylock(raw(self)) == 0
+ pub fn try_lock(&self) -> bool {
+ unsafe { libc::pthread_mutex_trylock(self.get()) == 0 }
+ }
+}
+
+impl Drop for Mutex {
+ fn drop(&mut self) {
+ let Some(mutex) = self.inner.take() else { return };
+ // We're not allowed to pthread_mutex_destroy a locked mutex,
+ // so check first if it's unlocked.
+ if unsafe { libc::pthread_mutex_trylock(mutex.0.get()) == 0 } {
+ unsafe { libc::pthread_mutex_unlock(mutex.0.get()) };
+ drop(mutex);
+ } else {
+ // The mutex is locked. This happens if a MutexGuard is leaked.
+ // In this case, we just leak the Mutex too.
+ forget(mutex);
+ }
}
}
diff --git a/library/std/src/sys/sync/mutex/sgx.rs b/library/std/src/sys/sync/mutex/sgx.rs
index 65d1e880f7baf..8529e85797043 100644
--- a/library/std/src/sys/sync/mutex/sgx.rs
+++ b/library/std/src/sys/sync/mutex/sgx.rs
@@ -1,28 +1,24 @@
use crate::sys::pal::waitqueue::{SpinMutex, WaitQueue, WaitVariable, try_lock_or_false};
-use crate::sys_common::lazy_box::{LazyBox, LazyInit};
-
-/// FIXME: `UnsafeList` is not movable.
-struct AllocatedMutex(SpinMutex<WaitVariable<bool>>);
+use crate::sys::sync::OnceBox;
pub struct Mutex {
- inner: LazyBox<AllocatedMutex>,
-}
-
-impl LazyInit for AllocatedMutex {
- fn init() -> Box<Self> {
- Box::new(AllocatedMutex(SpinMutex::new(WaitVariable::new(false))))
- }
+ // FIXME: `UnsafeList` is not movable.
+ inner: OnceBox<SpinMutex<WaitVariable<bool>>>,
}
// Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28
impl Mutex {
pub const fn new() -> Mutex {
- Mutex { inner: LazyBox::new() }
+ Mutex { inner: OnceBox::new() }
+ }
+
+ fn get(&self) -> &SpinMutex<WaitVariable<bool>> {
+ self.inner.get_or_init(|| Box::new(SpinMutex::new(WaitVariable::new(false))))
}
#[inline]
pub fn lock(&self) {
- let mut guard = self.inner.0.lock();
+ let mut guard = self.get().lock();
if *guard.lock_var() {
// Another thread has the lock, wait
WaitQueue::wait(guard, || {})
@@ -35,7 +31,9 @@ impl Mutex {
#[inline]
pub unsafe fn unlock(&self) {
- let guard = self.inner.0.lock();
+ // SAFETY: the mutex was locked by the current thread, so it has been
+ // initialized already.
+ let guard = unsafe { self.inner.get_unchecked().lock() };
if let Err(mut guard) = WaitQueue::notify_one(guard) {
// No other waiters, unlock
*guard.lock_var_mut() = false;
@@ -46,7 +44,7 @@ impl Mutex {
#[inline]
pub fn try_lock(&self) -> bool {
- let mut guard = try_lock_or_false!(self.inner.0);
+ let mut guard = try_lock_or_false!(self.get());
if *guard.lock_var() {
// Another thread has the lock
false
diff --git a/library/std/src/sys/sync/once_box.rs b/library/std/src/sys/sync/once_box.rs
new file mode 100644
index 0000000000000..1422b5a172162
--- /dev/null
+++ b/library/std/src/sys/sync/once_box.rs
@@ -0,0 +1,82 @@
+//! A racily-initialized alternative to `OnceLock<Box<T>>`.
+//!
+//! This is used to implement synchronization primitives that need allocation,
+//! like the pthread versions.
+
+#![allow(dead_code)] // Only used on some platforms.
+
+use crate::mem::replace;
+use crate::ptr::null_mut;
+use crate::sync::atomic::AtomicPtr;
+use crate::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed};
+
+pub(crate) struct OnceBox<T> {
+ ptr: AtomicPtr<T>,
+}
+
+impl<T> OnceBox<T> {
+ #[inline]
+ pub const fn new() -> Self {
+ Self { ptr: AtomicPtr::new(null_mut()) }
+ }
+
+ /// Gets access to the value, assuming it is already initialized and this
+ /// initialization has been observed by the current thread.
+ ///
+ /// Since all modifications to the pointer have already been observed, the
+ /// pointer load in this function can be performed with relaxed ordering,
+ /// potentially allowing the optimizer to turn code like this:
+ /// ```rust, ignore
+ /// once_box.get_or_init(|| Box::new(42));
+ /// unsafe { once_box.get_unchecked() }
+ /// ```
+ /// into
+ /// ```rust, ignore
+ /// once_box.get_or_init(|| Box::new(42))
+ /// ```
+ ///
+ /// # Safety
+ /// This causes undefined behaviour if the assumption above is violated.
+ #[inline]
+ pub unsafe fn get_unchecked(&self) -> &T {
+ unsafe { &*self.ptr.load(Relaxed) }
+ }
+
+ #[inline]
+ pub fn get_or_init(&self, f: impl FnOnce() -> Box<T>) -> &T {
+ let ptr = self.ptr.load(Acquire);
+ match unsafe { ptr.as_ref() } {
+ Some(val) => val,
+ None => self.initialize(f),
+ }
+ }
+
+ #[inline]
+ pub fn take(&mut self) -> Option<Box<T>> {
+ let ptr = replace(self.ptr.get_mut(), null_mut());
+ if !ptr.is_null() { Some(unsafe { Box::from_raw(ptr) }) } else { None }
+ }
+
+ #[cold]
+ fn initialize(&self, f: impl FnOnce() -> Box<T>) -> &T {
+ let new_ptr = Box::into_raw(f());
+ match self.ptr.compare_exchange(null_mut(), new_ptr, AcqRel, Acquire) {
+ Ok(_) => unsafe { &*new_ptr },
+ Err(ptr) => {
+ // Lost the race to another thread.
+ // Drop the value we created, and use the one from the other thread instead.
+ drop(unsafe { Box::from_raw(new_ptr) });
+ unsafe { &*ptr }
+ }
+ }
+ }
+}
+
+unsafe impl<T: Send> Send for OnceBox<T> {}
+unsafe impl<T: Send + Sync> Sync for OnceBox<T> {}
+
+impl<T> Drop for OnceBox<T> {
+ fn drop(&mut self) {
+ self.take();
+ }
+}
diff --git a/library/std/src/sys/sync/rwlock/teeos.rs b/library/std/src/sys/sync/rwlock/teeos.rs
index ef9b1ab51546c..763430223834b 100644
--- a/library/std/src/sys/sync/rwlock/teeos.rs
+++ b/library/std/src/sys/sync/rwlock/teeos.rs
@@ -14,22 +14,22 @@ impl RwLock {
#[inline]
pub fn read(&self) {
- unsafe { self.inner.lock() };
+ self.inner.lock()
}
#[inline]
pub fn try_read(&self) -> bool {
- unsafe { self.inner.try_lock() }
+ self.inner.try_lock()
}
#[inline]
pub fn write(&self) {
- unsafe { self.inner.lock() };
+ self.inner.lock()
}
#[inline]
pub unsafe fn try_write(&self) -> bool {
- unsafe { self.inner.try_lock() }
+ self.inner.try_lock()
}
#[inline]
diff --git a/library/std/src/sys_common/lazy_box.rs b/library/std/src/sys_common/lazy_box.rs
deleted file mode 100644
index b45b05f63baaa..0000000000000
--- a/library/std/src/sys_common/lazy_box.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-#![allow(dead_code)] // Only used on some platforms.
-
-// This is used to wrap pthread {Mutex, Condvar, RwLock} in.
-
-use crate::marker::PhantomData;
-use crate::ops::{Deref, DerefMut};
-use crate::ptr::null_mut;
-use crate::sync::atomic::AtomicPtr;
-use crate::sync::atomic::Ordering::{AcqRel, Acquire};
-
-pub(crate) struct LazyBox<T: LazyInit> {
- ptr: AtomicPtr<T>,
- _phantom: PhantomData<T>,
-}
-
-pub(crate) trait LazyInit {
- /// This is called before the box is allocated, to provide the value to
- /// move into the new box.
- ///
- /// It might be called more than once per LazyBox, as multiple threads
- /// might race to initialize it concurrently, each constructing and initializing
- /// their own box. All but one of them will be passed to `cancel_init` right after.
- fn init() -> Box<Self>;
-
- /// Any surplus boxes from `init()` that lost the initialization race
- /// are passed to this function for disposal.
- ///
- /// The default implementation calls destroy().
- fn cancel_init(x: Box<Self>) {
- Self::destroy(x);
- }
-
- /// This is called to destroy a used box.
- ///
- /// The default implementation just drops it.
- fn destroy(_: Box<Self>) {}
-}
-
-impl<T: LazyInit> LazyBox<T> {
- #[inline]
- pub const fn new() -> Self {
- Self { ptr: AtomicPtr::new(null_mut()), _phantom: PhantomData }
- }
-
- #[inline]
- fn get_pointer(&self) -> *mut T {
- let ptr = self.ptr.load(Acquire);
- if ptr.is_null() { self.initialize() } else { ptr }
- }
-
- #[cold]
- fn initialize(&self) -> *mut T {
- let new_ptr = Box::into_raw(T::init());
- match self.ptr.compare_exchange(null_mut(), new_ptr, AcqRel, Acquire) {
- Ok(_) => new_ptr,
- Err(ptr) => {
- // Lost the race to another thread.
- // Drop the box we created, and use the one from the other thread instead.
- T::cancel_init(unsafe { Box::from_raw(new_ptr) });
- ptr
- }
- }
- }
-}
-
-impl<T: LazyInit> Deref for LazyBox<T> {
- type Target = T;
- #[inline]
- fn deref(&self) -> &T {
- unsafe { &*self.get_pointer() }
- }
-}
-
-impl<T: LazyInit> DerefMut for LazyBox<T> {
- #[inline]
- fn deref_mut(&mut self) -> &mut T {
- unsafe { &mut *self.get_pointer() }
- }
-}
-
-impl<T: LazyInit> Drop for LazyBox<T> {
- fn drop(&mut self) {
- let ptr = *self.ptr.get_mut();
- if !ptr.is_null() {
- T::destroy(unsafe { Box::from_raw(ptr) });
- }
- }
-}
diff --git a/library/std/src/sys_common/mod.rs b/library/std/src/sys_common/mod.rs
index aa27886ff6f9c..4f7a131f6bb90 100644
--- a/library/std/src/sys_common/mod.rs
+++ b/library/std/src/sys_common/mod.rs
@@ -22,7 +22,6 @@ mod tests;
pub mod fs;
pub mod io;
-pub mod lazy_box;
pub mod process;
pub mod wstr;
pub mod wtf8;
diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md
index 0ac58645d2dfc..f036603ee707b 100644
--- a/src/bootstrap/README.md
+++ b/src/bootstrap/README.md
@@ -182,10 +182,8 @@ Some general areas that you may be interested in modifying are:
`Config` struct.
* Adding a sanity check? Take a look at `bootstrap/src/core/sanity.rs`.
-If you make a major change on bootstrap configuration, please remember to:
-
-+ Update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/utils/change_tracker.rs`.
-* Update `change-id = {pull-request-id}` in `config.example.toml`.
+If you make a major change on bootstrap configuration, please add a new entry to
+`CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/utils/change_tracker.rs`.
A 'major change' includes
diff --git a/src/tools/clippy/tests/ui/transmute_float_to_int.fixed b/src/tools/clippy/tests/ui/transmute_float_to_int.fixed
index 83814ca43b96d..075a198918a0d 100644
--- a/src/tools/clippy/tests/ui/transmute_float_to_int.fixed
+++ b/src/tools/clippy/tests/ui/transmute_float_to_int.fixed
@@ -1,7 +1,7 @@
#![warn(clippy::transmute_float_to_int)]
#![allow(clippy::missing_transmute_annotations)]
-#![feature(f128, f128_const)]
-#![feature(f16, f16_const)]
+#![feature(f128)]
+#![feature(f16)]
fn float_to_int() {
let _: u32 = unsafe { 1f32.to_bits() };
diff --git a/src/tools/clippy/tests/ui/transmute_float_to_int.rs b/src/tools/clippy/tests/ui/transmute_float_to_int.rs
index 64d6e9172039d..12541b2f7cf32 100644
--- a/src/tools/clippy/tests/ui/transmute_float_to_int.rs
+++ b/src/tools/clippy/tests/ui/transmute_float_to_int.rs
@@ -1,7 +1,7 @@
#![warn(clippy::transmute_float_to_int)]
#![allow(clippy::missing_transmute_annotations)]
-#![feature(f128, f128_const)]
-#![feature(f16, f16_const)]
+#![feature(f128)]
+#![feature(f16)]
fn float_to_int() {
let _: u32 = unsafe { std::mem::transmute(1f32) };
diff --git a/src/tools/miri/cargo-miri/src/phases.rs b/src/tools/miri/cargo-miri/src/phases.rs
index 52bc8e1a3b6d7..f1f76fd338cea 100644
--- a/src/tools/miri/cargo-miri/src/phases.rs
+++ b/src/tools/miri/cargo-miri/src/phases.rs
@@ -668,7 +668,6 @@ pub fn phase_runner(mut binary_args: impl Iterator<Item = String>, phase: Runner
RunnerPhase::Rustdoc => {
cmd.stdin(std::process::Stdio::piped());
// the warning is wrong, we have a `wait` inside the `scope` closure.
- #[expect(clippy::zombie_processes)]
let mut child = cmd.spawn().expect("failed to spawn process");
let child_stdin = child.stdin.take().unwrap();
// Write stdin in a background thread, as it may block.
diff --git a/src/tools/miri/rust-version b/src/tools/miri/rust-version
index b05c409f82319..eb4dfcf57cf00 100644
--- a/src/tools/miri/rust-version
+++ b/src/tools/miri/rust-version
@@ -1 +1 @@
-76ed7a1fa40c3f54d3fd3f834e12bf9c932d0146
+7067e4aee45c18cfa1c6af3bf79bd097684fb294
diff --git a/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs b/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
index 410f4a58ac530..15cefab1a68e7 100644
--- a/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
+++ b/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
@@ -859,14 +859,15 @@ impl Tree {
) -> Option<UniIndex> {
let node = self.nodes.get(idx).unwrap();
+ let [child_idx] = node.children[..] else { return None };
+
// We never want to replace the root node, as it is also kept in `root_ptr_tags`.
- if node.children.len() != 1 || live.contains(&node.tag) || node.parent.is_none() {
+ if live.contains(&node.tag) || node.parent.is_none() {
return None;
}
// Since protected nodes are never GC'd (see `borrow_tracker::FrameExtra::visit_provenance`),
// we know that `node` is not protected because otherwise `live` would
// have contained `node.tag`.
- let child_idx = node.children[0];
let child = self.nodes.get(child_idx).unwrap();
// Check that for that one child, `can_be_replaced_by_child` holds for the permission
// on all locations.
diff --git a/src/tools/miri/src/concurrency/init_once.rs b/src/tools/miri/src/concurrency/init_once.rs
index 8985135f4e892..7a9b12bbe82c9 100644
--- a/src/tools/miri/src/concurrency/init_once.rs
+++ b/src/tools/miri/src/concurrency/init_once.rs
@@ -39,7 +39,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
|ecx| &mut ecx.machine.sync.init_onces,
|_| interp_ok(Default::default()),
)?
- .ok_or_else(|| err_ub_format!("init_once has invalid ID")).into()
+ .ok_or_else(|| err_ub_format!("init_once has invalid ID"))
+ .into()
}
#[inline]
diff --git a/src/tools/miri/src/concurrency/sync.rs b/src/tools/miri/src/concurrency/sync.rs
index 3b57af641b589..5627ccdbbea27 100644
--- a/src/tools/miri/src/concurrency/sync.rs
+++ b/src/tools/miri/src/concurrency/sync.rs
@@ -307,7 +307,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
|ecx| &mut ecx.machine.sync.mutexes,
|ecx| initialize_data(ecx).map(|data| Mutex { data, ..Default::default() }),
)?
- .ok_or_else(|| err_ub_format!("mutex has invalid ID")).into()
+ .ok_or_else(|| err_ub_format!("mutex has invalid ID"))
+ .into()
}
/// Retrieve the additional data stored for a mutex.
@@ -334,7 +335,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
|ecx| &mut ecx.machine.sync.rwlocks,
|ecx| initialize_data(ecx).map(|data| RwLock { data, ..Default::default() }),
)?
- .ok_or_else(|| err_ub_format!("rwlock has invalid ID")).into()
+ .ok_or_else(|| err_ub_format!("rwlock has invalid ID"))
+ .into()
}
/// Retrieve the additional data stored for a rwlock.
@@ -375,7 +377,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
|ecx| &mut ecx.machine.sync.condvars,
|ecx| initialize_data(ecx).map(|data| Condvar { data, ..Default::default() }),
)?
- .ok_or_else(|| err_ub_format!("condvar has invalid ID")).into()
+ .ok_or_else(|| err_ub_format!("condvar has invalid ID"))
+ .into()
}
/// Retrieve the additional data stored for a condvar.
diff --git a/src/tools/miri/src/concurrency/vector_clock.rs b/src/tools/miri/src/concurrency/vector_clock.rs
index f9025e06c684e..345726634299b 100644
--- a/src/tools/miri/src/concurrency/vector_clock.rs
+++ b/src/tools/miri/src/concurrency/vector_clock.rs
@@ -151,7 +151,7 @@ impl VClock {
/// Load the internal timestamp slice in the vector clock
#[inline]
pub(super) fn as_slice(&self) -> &[VTimestamp] {
- debug_assert!(!self.0.last().is_some_and(|t| t.time() == 0));
+ debug_assert!(self.0.last().is_none_or(|t| t.time() != 0));
self.0.as_slice()
}
diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs
index 70ebca0f32961..013bfe03aafa7 100644
--- a/src/tools/miri/src/helpers.rs
+++ b/src/tools/miri/src/helpers.rs
@@ -31,65 +31,6 @@ pub enum AccessKind {
Write,
}
-// This mapping should match `decode_error_kind` in
-// <https://github.com/rust-lang/rust/blob/master/library/std/src/sys/pal/unix/mod.rs>.
-const UNIX_IO_ERROR_TABLE: &[(&str, std::io::ErrorKind)] = {
- use std::io::ErrorKind::*;
- &[
- ("E2BIG", ArgumentListTooLong),
- ("EADDRINUSE", AddrInUse),
- ("EADDRNOTAVAIL", AddrNotAvailable),
- ("EBUSY", ResourceBusy),
- ("ECONNABORTED", ConnectionAborted),
- ("ECONNREFUSED", ConnectionRefused),
- ("ECONNRESET", ConnectionReset),
- ("EDEADLK", Deadlock),
- ("EDQUOT", FilesystemQuotaExceeded),
- ("EEXIST", AlreadyExists),
- ("EFBIG", FileTooLarge),
- ("EHOSTUNREACH", HostUnreachable),
- ("EINTR", Interrupted),
- ("EINVAL", InvalidInput),
- ("EISDIR", IsADirectory),
- ("ELOOP", FilesystemLoop),
- ("ENOENT", NotFound),
- ("ENOMEM", OutOfMemory),
- ("ENOSPC", StorageFull),
- ("ENOSYS", Unsupported),
- ("EMLINK", TooManyLinks),
- ("ENAMETOOLONG", InvalidFilename),
- ("ENETDOWN", NetworkDown),
- ("ENETUNREACH", NetworkUnreachable),
- ("ENOTCONN", NotConnected),
- ("ENOTDIR", NotADirectory),
- ("ENOTEMPTY", DirectoryNotEmpty),
- ("EPIPE", BrokenPipe),
- ("EROFS", ReadOnlyFilesystem),
- ("ESPIPE", NotSeekable),
- ("ESTALE", StaleNetworkFileHandle),
- ("ETIMEDOUT", TimedOut),
- ("ETXTBSY", ExecutableFileBusy),
- ("EXDEV", CrossesDevices),
- // The following have two valid options. We have both for the forwards mapping; only the
- // first one will be used for the backwards mapping.
- ("EPERM", PermissionDenied),
- ("EACCES", PermissionDenied),
- ("EWOULDBLOCK", WouldBlock),
- ("EAGAIN", WouldBlock),
- ]
-};
-// This mapping should match `decode_error_kind` in
-// <https://github.com/rust-lang/rust/blob/master/library/std/src/sys/pal/windows/mod.rs>.
-const WINDOWS_IO_ERROR_TABLE: &[(&str, std::io::ErrorKind)] = {
- use std::io::ErrorKind::*;
- // FIXME: this is still incomplete.
- &[
- ("ERROR_ACCESS_DENIED", PermissionDenied),
- ("ERROR_FILE_NOT_FOUND", NotFound),
- ("ERROR_INVALID_PARAMETER", InvalidInput),
- ]
-};
-
/// Gets an instance for a path.
///
/// A `None` namespace indicates we are looking for a module.
@@ -745,119 +686,6 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
self.eval_context_ref().tcx.sess.target.families.iter().any(|f| f == "unix")
}
- /// Get last error variable as a place, lazily allocating thread-local storage for it if
- /// necessary.
- fn last_error_place(&mut self) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
- let this = self.eval_context_mut();
- if let Some(errno_place) = this.active_thread_ref().last_error.as_ref() {
- interp_ok(errno_place.clone())
- } else {
- // Allocate new place, set initial value to 0.
- let errno_layout = this.machine.layouts.u32;
- let errno_place = this.allocate(errno_layout, MiriMemoryKind::Machine.into())?;
- this.write_scalar(Scalar::from_u32(0), &errno_place)?;
- this.active_thread_mut().last_error = Some(errno_place.clone());
- interp_ok(errno_place)
- }
- }
-
- /// Sets the last error variable.
- fn set_last_error(&mut self, scalar: Scalar) -> InterpResult<'tcx> {
- let this = self.eval_context_mut();
- let errno_place = this.last_error_place()?;
- this.write_scalar(scalar, &errno_place)
- }
-
- /// Gets the last error variable.
- fn get_last_error(&mut self) -> InterpResult<'tcx, Scalar> {
- let this = self.eval_context_mut();
- let errno_place = this.last_error_place()?;
- this.read_scalar(&errno_place)
- }
-
- /// This function tries to produce the most similar OS error from the `std::io::ErrorKind`
- /// as a platform-specific errnum.
- fn io_error_to_errnum(&self, err: std::io::Error) -> InterpResult<'tcx, Scalar> {
- let this = self.eval_context_ref();
- let target = &this.tcx.sess.target;
- if target.families.iter().any(|f| f == "unix") {
- for &(name, kind) in UNIX_IO_ERROR_TABLE {
- if err.kind() == kind {
- return interp_ok(this.eval_libc(name));
- }
- }
- throw_unsup_format!("unsupported io error: {err}")
- } else if target.families.iter().any(|f| f == "windows") {
- for &(name, kind) in WINDOWS_IO_ERROR_TABLE {
- if err.kind() == kind {
- return interp_ok(this.eval_windows("c", name));
- }
- }
- throw_unsup_format!("unsupported io error: {err}");
- } else {
- throw_unsup_format!(
- "converting io::Error into errnum is unsupported for OS {}",
- target.os
- )
- }
- }
-
- /// The inverse of `io_error_to_errnum`.
- #[allow(clippy::needless_return)]
- fn try_errnum_to_io_error(
- &self,
- errnum: Scalar,
- ) -> InterpResult<'tcx, Option<std::io::ErrorKind>> {
- let this = self.eval_context_ref();
- let target = &this.tcx.sess.target;
- if target.families.iter().any(|f| f == "unix") {
- let errnum = errnum.to_i32()?;
- for &(name, kind) in UNIX_IO_ERROR_TABLE {
- if errnum == this.eval_libc_i32(name) {
- return interp_ok(Some(kind));
- }
- }
- return interp_ok(None);
- } else if target.families.iter().any(|f| f == "windows") {
- let errnum = errnum.to_u32()?;
- for &(name, kind) in WINDOWS_IO_ERROR_TABLE {
- if errnum == this.eval_windows("c", name).to_u32()? {
- return interp_ok(Some(kind));
- }
- }
- return interp_ok(None);
- } else {
- throw_unsup_format!(
- "converting errnum into io::Error is unsupported for OS {}",
- target.os
- )
- }
- }
-
- /// Sets the last OS error using a `std::io::ErrorKind`.
- fn set_last_error_from_io_error(&mut self, err: std::io::Error) -> InterpResult<'tcx> {
- self.set_last_error(self.io_error_to_errnum(err)?)
- }
-
- /// Helper function that consumes a `std::io::Result<T>` and returns a
- /// `InterpResult<'tcx, T>` instead. In case the result is an error, this function returns
- /// `Ok(-1)` and sets the last OS error accordingly.
- ///
- /// This function uses `T: From<i32>` instead of `i32` directly because some IO related
- /// functions return different integer types (like `read`, that returns an `i64`).
- fn try_unwrap_io_result<T: From<i32>>(
- &mut self,
- result: std::io::Result<T>,
- ) -> InterpResult<'tcx, T> {
- match result {
- Ok(ok) => interp_ok(ok),
- Err(e) => {
- self.eval_context_mut().set_last_error_from_io_error(e)?;
- interp_ok((-1).into())
- }
- }
- }
-
/// Dereference a pointer operand to a place using `layout` instead of the pointer's declared type
fn deref_pointer_as(
&self,
@@ -924,17 +752,19 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let nanoseconds_scalar = this.read_scalar(&nanoseconds_place)?;
let nanoseconds = nanoseconds_scalar.to_target_isize(this)?;
- interp_ok(try {
- // tv_sec must be non-negative.
- let seconds: u64 = seconds.try_into().ok()?;
- // tv_nsec must be non-negative.
- let nanoseconds: u32 = nanoseconds.try_into().ok()?;
- if nanoseconds >= 1_000_000_000 {
- // tv_nsec must not be greater than 999,999,999.
- None?
- }
- Duration::new(seconds, nanoseconds)
- })
+ interp_ok(
+ try {
+ // tv_sec must be non-negative.
+ let seconds: u64 = seconds.try_into().ok()?;
+ // tv_nsec must be non-negative.
+ let nanoseconds: u32 = nanoseconds.try_into().ok()?;
+ if nanoseconds >= 1_000_000_000 {
+ // tv_nsec must not be greater than 999,999,999.
+ None?
+ }
+ Duration::new(seconds, nanoseconds)
+ },
+ )
}
/// Read bytes from a byte slice.
diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs
index 78e7bf704552d..330147c8f1cf8 100644
--- a/src/tools/miri/src/lib.rs
+++ b/src/tools/miri/src/lib.rs
@@ -150,6 +150,7 @@ pub use crate::range_map::RangeMap;
pub use crate::shims::EmulateItemResult;
pub use crate::shims::env::{EnvVars, EvalContextExt as _};
pub use crate::shims::foreign_items::{DynSym, EvalContextExt as _};
+pub use crate::shims::io_error::{EvalContextExt as _, LibcError};
pub use crate::shims::os_str::EvalContextExt as _;
pub use crate::shims::panic::{CatchUnwindData, EvalContextExt as _};
pub use crate::shims::time::EvalContextExt as _;
diff --git a/src/tools/miri/src/shims/io_error.rs b/src/tools/miri/src/shims/io_error.rs
new file mode 100644
index 0000000000000..38aa181cb4f05
--- /dev/null
+++ b/src/tools/miri/src/shims/io_error.rs
@@ -0,0 +1,228 @@
+use std::io;
+
+use crate::*;
+
+/// A representation of an IO error: either a libc error name,
+/// or a host error.
+#[derive(Debug)]
+pub enum IoError {
+ LibcError(&'static str),
+ HostError(io::Error),
+ Raw(Scalar),
+}
+pub use self::IoError::*;
+
+impl From<io::Error> for IoError {
+ fn from(value: io::Error) -> Self {
+ IoError::HostError(value)
+ }
+}
+
+impl From<io::ErrorKind> for IoError {
+ fn from(value: io::ErrorKind) -> Self {
+ IoError::HostError(value.into())
+ }
+}
+
+impl From<Scalar> for IoError {
+ fn from(value: Scalar) -> Self {
+ IoError::Raw(value)
+ }
+}
+
+// This mapping should match `decode_error_kind` in
+// <https://github.com/rust-lang/rust/blob/master/library/std/src/sys/pal/unix/mod.rs>.
+const UNIX_IO_ERROR_TABLE: &[(&str, std::io::ErrorKind)] = {
+ use std::io::ErrorKind::*;
+ &[
+ ("E2BIG", ArgumentListTooLong),
+ ("EADDRINUSE", AddrInUse),
+ ("EADDRNOTAVAIL", AddrNotAvailable),
+ ("EBUSY", ResourceBusy),
+ ("ECONNABORTED", ConnectionAborted),
+ ("ECONNREFUSED", ConnectionRefused),
+ ("ECONNRESET", ConnectionReset),
+ ("EDEADLK", Deadlock),
+ ("EDQUOT", FilesystemQuotaExceeded),
+ ("EEXIST", AlreadyExists),
+ ("EFBIG", FileTooLarge),
+ ("EHOSTUNREACH", HostUnreachable),
+ ("EINTR", Interrupted),
+ ("EINVAL", InvalidInput),
+ ("EISDIR", IsADirectory),
+ ("ELOOP", FilesystemLoop),
+ ("ENOENT", NotFound),
+ ("ENOMEM", OutOfMemory),
+ ("ENOSPC", StorageFull),
+ ("ENOSYS", Unsupported),
+ ("EMLINK", TooManyLinks),
+ ("ENAMETOOLONG", InvalidFilename),
+ ("ENETDOWN", NetworkDown),
+ ("ENETUNREACH", NetworkUnreachable),
+ ("ENOTCONN", NotConnected),
+ ("ENOTDIR", NotADirectory),
+ ("ENOTEMPTY", DirectoryNotEmpty),
+ ("EPIPE", BrokenPipe),
+ ("EROFS", ReadOnlyFilesystem),
+ ("ESPIPE", NotSeekable),
+ ("ESTALE", StaleNetworkFileHandle),
+ ("ETIMEDOUT", TimedOut),
+ ("ETXTBSY", ExecutableFileBusy),
+ ("EXDEV", CrossesDevices),
+ // The following have two valid options. We have both for the forwards mapping; only the
+ // first one will be used for the backwards mapping.
+ ("EPERM", PermissionDenied),
+ ("EACCES", PermissionDenied),
+ ("EWOULDBLOCK", WouldBlock),
+ ("EAGAIN", WouldBlock),
+ ]
+};
+// This mapping should match `decode_error_kind` in
+// <https://github.com/rust-lang/rust/blob/master/library/std/src/sys/pal/windows/mod.rs>.
+const WINDOWS_IO_ERROR_TABLE: &[(&str, std::io::ErrorKind)] = {
+ use std::io::ErrorKind::*;
+ // FIXME: this is still incomplete.
+ &[
+ ("ERROR_ACCESS_DENIED", PermissionDenied),
+ ("ERROR_FILE_NOT_FOUND", NotFound),
+ ("ERROR_INVALID_PARAMETER", InvalidInput),
+ ]
+};
+
+impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
+pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
+ /// Get last error variable as a place, lazily allocating thread-local storage for it if
+ /// necessary.
+ fn last_error_place(&mut self) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
+ let this = self.eval_context_mut();
+ if let Some(errno_place) = this.active_thread_ref().last_error.as_ref() {
+ interp_ok(errno_place.clone())
+ } else {
+ // Allocate new place, set initial value to 0.
+ let errno_layout = this.machine.layouts.u32;
+ let errno_place = this.allocate(errno_layout, MiriMemoryKind::Machine.into())?;
+ this.write_scalar(Scalar::from_u32(0), &errno_place)?;
+ this.active_thread_mut().last_error = Some(errno_place.clone());
+ interp_ok(errno_place)
+ }
+ }
+
+ /// Sets the last error variable.
+ fn set_last_error(&mut self, err: impl Into<IoError>) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+ let errno = match err.into() {
+ HostError(err) => this.io_error_to_errnum(err)?,
+ LibcError(name) => this.eval_libc(name),
+ Raw(val) => val,
+ };
+ let errno_place = this.last_error_place()?;
+ this.write_scalar(errno, &errno_place)
+ }
+
+ /// Sets the last OS error and writes -1 to dest place.
+ fn set_last_error_and_return(
+ &mut self,
+ err: impl Into<IoError>,
+ dest: &MPlaceTy<'tcx>,
+ ) -> InterpResult<'tcx> {
+ let this = self.eval_context_mut();
+ this.set_last_error(err)?;
+ this.write_int(-1, dest)?;
+ interp_ok(())
+ }
+
+ /// Sets the last OS error and return `-1` as a `i32`-typed Scalar
+ fn set_last_error_and_return_i32(
+ &mut self,
+ err: impl Into<IoError>,
+ ) -> InterpResult<'tcx, Scalar> {
+ let this = self.eval_context_mut();
+ this.set_last_error(err)?;
+ interp_ok(Scalar::from_i32(-1))
+ }
+
+ /// Gets the last error variable.
+ fn get_last_error(&mut self) -> InterpResult<'tcx, Scalar> {
+ let this = self.eval_context_mut();
+ let errno_place = this.last_error_place()?;
+ this.read_scalar(&errno_place)
+ }
+
+ /// This function tries to produce the most similar OS error from the `std::io::ErrorKind`
+ /// as a platform-specific errnum.
+ fn io_error_to_errnum(&self, err: std::io::Error) -> InterpResult<'tcx, Scalar> {
+ let this = self.eval_context_ref();
+ let target = &this.tcx.sess.target;
+ if target.families.iter().any(|f| f == "unix") {
+ for &(name, kind) in UNIX_IO_ERROR_TABLE {
+ if err.kind() == kind {
+ return interp_ok(this.eval_libc(name));
+ }
+ }
+ throw_unsup_format!("unsupported io error: {err}")
+ } else if target.families.iter().any(|f| f == "windows") {
+ for &(name, kind) in WINDOWS_IO_ERROR_TABLE {
+ if err.kind() == kind {
+ return interp_ok(this.eval_windows("c", name));
+ }
+ }
+ throw_unsup_format!("unsupported io error: {err}");
+ } else {
+ throw_unsup_format!(
+ "converting io::Error into errnum is unsupported for OS {}",
+ target.os
+ )
+ }
+ }
+
+ /// The inverse of `io_error_to_errnum`.
+ #[allow(clippy::needless_return)]
+ fn try_errnum_to_io_error(
+ &self,
+ errnum: Scalar,
+ ) -> InterpResult<'tcx, Option<std::io::ErrorKind>> {
+ let this = self.eval_context_ref();
+ let target = &this.tcx.sess.target;
+ if target.families.iter().any(|f| f == "unix") {
+ let errnum = errnum.to_i32()?;
+ for &(name, kind) in UNIX_IO_ERROR_TABLE {
+ if errnum == this.eval_libc_i32(name) {
+ return interp_ok(Some(kind));
+ }
+ }
+ return interp_ok(None);
+ } else if target.families.iter().any(|f| f == "windows") {
+ let errnum = errnum.to_u32()?;
+ for &(name, kind) in WINDOWS_IO_ERROR_TABLE {
+ if errnum == this.eval_windows("c", name).to_u32()? {
+ return interp_ok(Some(kind));
+ }
+ }
+ return interp_ok(None);
+ } else {
+ throw_unsup_format!(
+ "converting errnum into io::Error is unsupported for OS {}",
+ target.os
+ )
+ }
+ }
+
+ /// Helper function that consumes an `std::io::Result<T>` and returns an
+ /// `InterpResult<'tcx,T>::Ok` instead. In case the result is an error, this function returns
+ /// `Ok(-1)` and sets the last OS error accordingly.
+ ///
+ /// This function uses `T: From<i32>` instead of `i32` directly because some IO related
+ /// functions return different integer types (like `read`, that returns an `i64`).
+ fn try_unwrap_io_result<T: From<i32>>(
+ &mut self,
+ result: std::io::Result<T>,
+ ) -> InterpResult<'tcx, T> {
+ match result {
+ Ok(ok) => interp_ok(ok),
+ Err(e) => {
+ self.eval_context_mut().set_last_error(e)?;
+ interp_ok((-1).into())
+ }
+ }
+ }
+}
diff --git a/src/tools/miri/src/shims/mod.rs b/src/tools/miri/src/shims/mod.rs
index a689ac2b3784e..b9317ac1a15fa 100644
--- a/src/tools/miri/src/shims/mod.rs
+++ b/src/tools/miri/src/shims/mod.rs
@@ -12,6 +12,7 @@ mod x86;
pub mod env;
pub mod extern_static;
pub mod foreign_items;
+pub mod io_error;
pub mod os_str;
pub mod panic;
pub mod time;
diff --git a/src/tools/miri/src/shims/time.rs b/src/tools/miri/src/shims/time.rs
index 21c5421f10941..12c7679608ded 100644
--- a/src/tools/miri/src/shims/time.rs
+++ b/src/tools/miri/src/shims/time.rs
@@ -11,7 +11,8 @@ use crate::*;
/// Returns the time elapsed between the provided time and the unix epoch as a `Duration`.
pub fn system_time_to_duration<'tcx>(time: &SystemTime) -> InterpResult<'tcx, Duration> {
time.duration_since(SystemTime::UNIX_EPOCH)
- .map_err(|_| err_unsup_format!("times before the Unix epoch are not supported")).into()
+ .map_err(|_| err_unsup_format!("times before the Unix epoch are not supported"))
+ .into()
}
impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
diff --git a/src/tools/miri/src/shims/unix/env.rs b/src/tools/miri/src/shims/unix/env.rs
index 75721814c9948..96c5a9fad9b35 100644
--- a/src/tools/miri/src/shims/unix/env.rs
+++ b/src/tools/miri/src/shims/unix/env.rs
@@ -177,9 +177,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
interp_ok(Scalar::from_i32(0)) // return zero on success
} else {
// name argument is a null pointer, points to an empty string, or points to a string containing an '=' character.
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
- interp_ok(Scalar::from_i32(-1))
+ this.set_last_error_and_return_i32(LibcError("EINVAL"))
}
}
@@ -203,9 +201,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
interp_ok(Scalar::from_i32(0))
} else {
// name argument is a null pointer, points to an empty string, or points to a string containing an '=' character.
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
- interp_ok(Scalar::from_i32(-1))
+ this.set_last_error_and_return_i32(LibcError("EINVAL"))
}
}
@@ -218,7 +214,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`getcwd`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Pointer::null());
}
@@ -228,10 +224,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
if this.write_path_to_c_str(&cwd, buf, size)?.0 {
return interp_ok(buf);
}
- let erange = this.eval_libc("ERANGE");
- this.set_last_error(erange)?;
+ this.set_last_error(LibcError("ERANGE"))?;
}
- Err(e) => this.set_last_error_from_io_error(e)?,
+ Err(e) => this.set_last_error(e)?,
}
interp_ok(Pointer::null())
@@ -245,9 +240,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`chdir`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
-
- return interp_ok(Scalar::from_i32(-1));
+ return this.set_last_error_and_return_i32(ErrorKind::PermissionDenied);
}
let result = env::set_current_dir(path).map(|()| 0);
diff --git a/src/tools/miri/src/shims/unix/fd.rs b/src/tools/miri/src/shims/unix/fd.rs
index 24e5d5579c3f1..34e29760da783 100644
--- a/src/tools/miri/src/shims/unix/fd.rs
+++ b/src/tools/miri/src/shims/unix/fd.rs
@@ -150,7 +150,10 @@ impl FileDescription for io::Stdin {
helpers::isolation_abort_error("`read` from stdin")?;
}
let result = Read::read(&mut { self }, &mut bytes);
- ecx.return_read_bytes_and_count(ptr, &bytes, result, dest)
+ match result {
+ Ok(read_size) => ecx.return_read_success(ptr, &bytes, read_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn is_tty(&self, communicate_allowed: bool) -> bool {
@@ -181,7 +184,10 @@ impl FileDescription for io::Stdout {
// the host -- there is no good in adding extra buffering
// here.
io::stdout().flush().unwrap();
- ecx.return_written_byte_count_or_error(result, dest)
+ match result {
+ Ok(write_size) => ecx.return_write_success(write_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn is_tty(&self, communicate_allowed: bool) -> bool {
@@ -207,7 +213,10 @@ impl FileDescription for io::Stderr {
// We allow writing to stderr even with isolation enabled.
// No need to flush, stderr is not buffered.
let result = Write::write(&mut { self }, bytes);
- ecx.return_written_byte_count_or_error(result, dest)
+ match result {
+ Ok(write_size) => ecx.return_write_success(write_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn is_tty(&self, communicate_allowed: bool) -> bool {
@@ -234,8 +243,7 @@ impl FileDescription for NullOutput {
ecx: &mut MiriInterpCx<'tcx>,
) -> InterpResult<'tcx> {
// We just don't write anything, but report to the user that we did.
- let result = Ok(len);
- ecx.return_written_byte_count_or_error(result, dest)
+ ecx.return_write_success(len, dest)
}
}
@@ -473,14 +481,14 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
fn fcntl(&mut self, args: &[OpTy<'tcx>]) -> InterpResult<'tcx, Scalar> {
let this = self.eval_context_mut();
- if args.len() < 2 {
+ let [fd_num, cmd, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for fcntl: got {}, expected at least 2",
args.len()
);
- }
- let fd_num = this.read_scalar(&args[0])?.to_i32()?;
- let cmd = this.read_scalar(&args[1])?.to_i32()?;
+ };
+ let fd_num = this.read_scalar(fd_num)?.to_i32()?;
+ let cmd = this.read_scalar(cmd)?.to_i32()?;
// We only support getting the flags for a descriptor.
if cmd == this.eval_libc_i32("F_GETFD") {
@@ -500,24 +508,24 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// because exec() isn't supported. The F_DUPFD and F_DUPFD_CLOEXEC commands only
// differ in whether the FD_CLOEXEC flag is pre-set on the new file descriptor,
// thus they can share the same implementation here.
- if args.len() < 3 {
+ let [_, _, start, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for fcntl with cmd=`F_DUPFD`/`F_DUPFD_CLOEXEC`: got {}, expected at least 3",
args.len()
);
- }
- let start = this.read_scalar(&args[2])?.to_i32()?;
+ };
+ let start = this.read_scalar(start)?.to_i32()?;
match this.machine.fds.get(fd_num) {
- Some(fd) => interp_ok(Scalar::from_i32(this.machine.fds.insert_with_min_num(fd, start))),
+ Some(fd) =>
+ interp_ok(Scalar::from_i32(this.machine.fds.insert_with_min_num(fd, start))),
None => interp_ok(Scalar::from_i32(this.fd_not_found()?)),
}
} else if this.tcx.sess.target.os == "macos" && cmd == this.eval_libc_i32("F_FULLFSYNC") {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`fcntl`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
- return interp_ok(Scalar::from_i32(-1));
+ return this.set_last_error_and_return_i32(ErrorKind::PermissionDenied);
}
this.ffullsync_fd(fd_num)
@@ -598,10 +606,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
None => fd.read(&fd, communicate, buf, count, dest, this)?,
Some(offset) => {
let Ok(offset) = u64::try_from(offset) else {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
- this.write_int(-1, dest)?;
- return interp_ok(());
+ return this.set_last_error_and_return(LibcError("EINVAL"), dest);
};
fd.pread(communicate, offset, buf, count, dest, this)?
}
@@ -643,10 +648,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
None => fd.write(&fd, communicate, buf, count, dest, this)?,
Some(offset) => {
let Ok(offset) = u64::try_from(offset) else {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
- this.write_int(-1, dest)?;
- return interp_ok(());
+ return this.set_last_error_and_return(LibcError("EINVAL"), dest);
};
fd.pwrite(communicate, buf, count, offset, dest, this)?
}
@@ -655,46 +657,39 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
/// Helper to implement `FileDescription::read`:
- /// `result` should be the return value of some underlying `read` call that used `bytes` as its output buffer.
+ /// This is only used when `read` is successful.
+ /// `actual_read_size` should be the return value of some underlying `read` call that used
+ /// `bytes` as its output buffer.
/// The length of `bytes` must not exceed either the host's or the target's `isize`.
- /// If `Result` indicates success, `bytes` is written to `buf` and the size is written to `dest`.
- /// Otherwise, `-1` is written to `dest` and the last libc error is set appropriately.
- fn return_read_bytes_and_count(
+ /// `bytes` is written to `buf` and the size is written to `dest`.
+ fn return_read_success(
&mut self,
buf: Pointer,
bytes: &[u8],
- result: io::Result<usize>,
+ actual_read_size: usize,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
- match result {
- Ok(read_bytes) => {
- // If reading to `bytes` did not fail, we write those bytes to the buffer.
- // Crucially, if fewer than `bytes.len()` bytes were read, only write
- // that much into the output buffer!
- this.write_bytes_ptr(buf, bytes[..read_bytes].iter().copied())?;
- // The actual read size is always less than what got originally requested so this cannot fail.
- this.write_int(u64::try_from(read_bytes).unwrap(), dest)?;
- interp_ok(())
- }
- Err(e) => {
- this.set_last_error_from_io_error(e)?;
- this.write_int(-1, dest)?;
- interp_ok(())
- }
- }
+ // If reading to `bytes` did not fail, we write those bytes to the buffer.
+ // Crucially, if fewer than `bytes.len()` bytes were read, only write
+ // that much into the output buffer!
+ this.write_bytes_ptr(buf, bytes[..actual_read_size].iter().copied())?;
+
+ // The actual read size is always less than what got originally requested so this cannot fail.
+ this.write_int(u64::try_from(actual_read_size).unwrap(), dest)?;
+ interp_ok(())
}
- /// This function writes the number of written bytes (given in `result`) to `dest`, or sets the
- /// last libc error and writes -1 to dest.
- fn return_written_byte_count_or_error(
+ /// Helper to implement `FileDescription::write`:
+ /// This function is only used when `write` is successful, and writes `actual_write_size` to `dest`
+ fn return_write_success(
&mut self,
- result: io::Result<usize>,
+ actual_write_size: usize,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
- let result = this.try_unwrap_io_result(result.map(|c| i64::try_from(c).unwrap()))?;
- this.write_int(result, dest)?;
+ // The actual write size is always less than what got originally requested so this cannot fail.
+ this.write_int(u64::try_from(actual_write_size).unwrap(), dest)?;
interp_ok(())
}
}
diff --git a/src/tools/miri/src/shims/unix/foreign_items.rs b/src/tools/miri/src/shims/unix/foreign_items.rs
index b35fe4487b27d..908f91a3bd6d5 100644
--- a/src/tools/miri/src/shims/unix/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/foreign_items.rs
@@ -355,8 +355,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// FreeBSD: https://man.freebsd.org/cgi/man.cgi?query=reallocarray
match this.compute_size_in_bytes(Size::from_bytes(size), nmemb) {
None => {
- let einval = this.eval_libc("ENOMEM");
- this.set_last_error(einval)?;
+ let enmem = this.eval_libc("ENOMEM");
+ this.set_last_error(enmem)?;
this.write_null(dest)?;
}
Some(len) => {
@@ -646,13 +646,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let chunk_size = CpuAffinityMask::chunk_size(this);
if this.ptr_is_null(mask)? {
- let einval = this.eval_libc("EFAULT");
- this.set_last_error(einval)?;
+ let efault = this.eval_libc("EFAULT");
+ this.set_last_error(efault)?;
this.write_int(-1, dest)?;
} else if cpusetsize == 0 || cpusetsize.checked_rem(chunk_size).unwrap() != 0 {
// we only copy whole chunks of size_of::<c_ulong>()
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
this.write_int(-1, dest)?;
} else if let Some(cpuset) = this.machine.thread_cpu_affinity.get(&thread_id) {
let cpuset = cpuset.clone();
@@ -662,8 +661,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
this.write_null(dest)?;
} else {
// The thread whose ID is pid could not be found
- let einval = this.eval_libc("ESRCH");
- this.set_last_error(einval)?;
+ let esrch = this.eval_libc("ESRCH");
+ this.set_last_error(esrch)?;
this.write_int(-1, dest)?;
}
}
@@ -689,8 +688,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
};
if this.ptr_is_null(mask)? {
- let einval = this.eval_libc("EFAULT");
- this.set_last_error(einval)?;
+ let efault = this.eval_libc("EFAULT");
+ this.set_last_error(efault)?;
this.write_int(-1, dest)?;
} else {
// NOTE: cpusetsize might be smaller than `CpuAffinityMask::CPU_MASK_BYTES`.
@@ -707,8 +706,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
None => {
// The intersection between the mask and the available CPUs was empty.
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
this.write_int(-1, dest)?;
}
}
diff --git a/src/tools/miri/src/shims/unix/fs.rs b/src/tools/miri/src/shims/unix/fs.rs
index 56280982e6ece..6c9a2beac2d2a 100644
--- a/src/tools/miri/src/shims/unix/fs.rs
+++ b/src/tools/miri/src/shims/unix/fs.rs
@@ -41,7 +41,10 @@ impl FileDescription for FileHandle {
assert!(communicate_allowed, "isolation should have prevented even opening a file");
let mut bytes = vec![0; len];
let result = (&mut &self.file).read(&mut bytes);
- ecx.return_read_bytes_and_count(ptr, &bytes, result, dest)
+ match result {
+ Ok(read_size) => ecx.return_read_success(ptr, &bytes, read_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn write<'tcx>(
@@ -56,7 +59,10 @@ impl FileDescription for FileHandle {
assert!(communicate_allowed, "isolation should have prevented even opening a file");
let bytes = ecx.read_bytes_ptr_strip_provenance(ptr, Size::from_bytes(len))?;
let result = (&mut &self.file).write(bytes);
- ecx.return_written_byte_count_or_error(result, dest)
+ match result {
+ Ok(write_size) => ecx.return_write_success(write_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn pread<'tcx>(
@@ -84,7 +90,10 @@ impl FileDescription for FileHandle {
res
};
let result = f();
- ecx.return_read_bytes_and_count(ptr, &bytes, result, dest)
+ match result {
+ Ok(read_size) => ecx.return_read_success(ptr, &bytes, read_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn pwrite<'tcx>(
@@ -112,7 +121,10 @@ impl FileDescription for FileHandle {
res
};
let result = f();
- ecx.return_written_byte_count_or_error(result, dest)
+ match result {
+ Ok(write_size) => ecx.return_write_success(write_size, dest),
+ Err(e) => ecx.set_last_error_and_return(e, dest),
+ }
}
fn seek<'tcx>(
@@ -421,18 +433,18 @@ fn maybe_sync_file(
impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
fn open(&mut self, args: &[OpTy<'tcx>]) -> InterpResult<'tcx, Scalar> {
- if args.len() < 2 {
+ let [path_raw, flag, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for `open`: got {}, expected at least 2",
args.len()
);
- }
+ };
let this = self.eval_context_mut();
- let path_raw = this.read_pointer(&args[0])?;
+ let path_raw = this.read_pointer(path_raw)?;
let path = this.read_path_from_c_str(path_raw)?;
- let flag = this.read_scalar(&args[1])?.to_i32()?;
+ let flag = this.read_scalar(flag)?.to_i32()?;
let mut options = OpenOptions::new();
@@ -526,8 +538,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let o_tmpfile = this.eval_libc_i32("O_TMPFILE");
if flag & o_tmpfile == o_tmpfile {
// if the flag contains `O_TMPFILE` then we return a graceful error
- let eopnotsupp = this.eval_libc("EOPNOTSUPP");
- this.set_last_error(eopnotsupp)?;
+ this.set_last_error(LibcError("EOPNOTSUPP"))?;
return interp_ok(Scalar::from_i32(-1));
}
}
@@ -564,7 +575,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`open`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -583,8 +594,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let seek_from = if whence == this.eval_libc_i32("SEEK_SET") {
if offset < 0 {
// Negative offsets return `EINVAL`.
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
return interp_ok(Scalar::from_i64(-1));
} else {
SeekFrom::Start(u64::try_from(offset).unwrap())
@@ -594,8 +604,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
} else if whence == this.eval_libc_i32("SEEK_END") {
SeekFrom::End(i64::try_from(offset).unwrap())
} else {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
return interp_ok(Scalar::from_i64(-1));
};
@@ -619,7 +628,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`unlink`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -650,7 +659,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`symlink`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -951,7 +960,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`rename`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -975,7 +984,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`mkdir`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -1003,7 +1012,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Reject if isolation is enabled.
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`rmdir`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -1037,7 +1046,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
interp_ok(Scalar::from_target_usize(id, this))
}
Err(e) => {
- this.set_last_error_from_io_error(e)?;
+ this.set_last_error(e)?;
interp_ok(Scalar::null_ptr(this))
}
}
@@ -1122,7 +1131,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
None
}
Some(Err(e)) => {
- this.set_last_error_from_io_error(e)?;
+ this.set_last_error(e)?;
None
}
};
@@ -1308,15 +1317,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
interp_ok(Scalar::from_i32(result))
} else {
drop(fd);
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
interp_ok(Scalar::from_i32(-1))
}
} else {
drop(fd);
// The file is not writable
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
interp_ok(Scalar::from_i32(-1))
}
}
@@ -1394,16 +1401,14 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let flags = this.read_scalar(flags_op)?.to_i32()?;
if offset < 0 || nbytes < 0 {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
return interp_ok(Scalar::from_i32(-1));
}
let allowed_flags = this.eval_libc_i32("SYNC_FILE_RANGE_WAIT_BEFORE")
| this.eval_libc_i32("SYNC_FILE_RANGE_WRITE")
| this.eval_libc_i32("SYNC_FILE_RANGE_WAIT_AFTER");
if flags & allowed_flags != flags {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -1465,7 +1470,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
interp_ok(path_bytes.len().try_into().unwrap())
}
Err(e) => {
- this.set_last_error_from_io_error(e)?;
+ this.set_last_error(e)?;
interp_ok(-1)
}
}
@@ -1545,7 +1550,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
interp_ok(Scalar::from_maybe_pointer(dest, this))
}
Err(e) => {
- this.set_last_error_from_io_error(e)?;
+ this.set_last_error(e)?;
interp_ok(Scalar::from_target_usize(0, this))
}
}
@@ -1597,8 +1602,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// If we don't find the suffix, it is an error.
if last_six_char_bytes != suffix_bytes {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -1664,7 +1668,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
_ => {
// "On error, -1 is returned, and errno is set to
// indicate the error"
- this.set_last_error_from_io_error(e)?;
+ this.set_last_error(e)?;
return interp_ok(Scalar::from_i32(-1));
}
},
@@ -1744,7 +1748,7 @@ impl FileMetadata {
let metadata = match metadata {
Ok(metadata) => metadata,
Err(e) => {
- ecx.set_last_error_from_io_error(e)?;
+ ecx.set_last_error(e)?;
return interp_ok(None);
}
};
diff --git a/src/tools/miri/src/shims/unix/linux/epoll.rs b/src/tools/miri/src/shims/unix/linux/epoll.rs
index 08f1381caef72..b57347abffa18 100644
--- a/src/tools/miri/src/shims/unix/linux/epoll.rs
+++ b/src/tools/miri/src/shims/unix/linux/epoll.rs
@@ -4,6 +4,7 @@ use std::io;
use std::rc::{Rc, Weak};
use std::time::Duration;
+use crate::concurrency::VClock;
use crate::shims::unix::fd::{FdId, FileDescriptionRef, WeakFileDescriptionRef};
use crate::shims::unix::*;
use crate::*;
@@ -19,7 +20,7 @@ struct Epoll {
/// and file descriptor value.
// This is an Rc because EpollInterest need to hold a reference to update
// it.
- ready_list: Rc<RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>>,
+ ready_list: Rc<ReadyList>,
/// A list of thread ids blocked on this epoll instance.
thread_id: RefCell<Vec<ThreadId>>,
}
@@ -63,7 +64,7 @@ pub struct EpollEventInterest {
/// <https://man7.org/linux/man-pages/man3/epoll_event.3type.html>
data: u64,
/// Ready list of the epoll instance under which this EpollEventInterest is registered.
- ready_list: Rc<RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>>,
+ ready_list: Rc<ReadyList>,
/// The epoll file description that this EpollEventInterest is registered under.
weak_epfd: WeakFileDescriptionRef,
}
@@ -88,6 +89,12 @@ pub struct EpollReadyEvents {
pub epollerr: bool,
}
+#[derive(Debug, Default)]
+struct ReadyList {
+ mapping: RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>,
+ clock: RefCell<VClock>,
+}
+
impl EpollReadyEvents {
pub fn new() -> Self {
EpollReadyEvents {
@@ -127,7 +134,7 @@ impl EpollReadyEvents {
}
impl Epoll {
- fn get_ready_list(&self) -> Rc<RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>> {
+ fn get_ready_list(&self) -> Rc<ReadyList> {
Rc::clone(&self.ready_list)
}
}
@@ -207,9 +214,6 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
);
}
- let mut epoll_instance = Epoll::default();
- epoll_instance.ready_list = Rc::new(RefCell::new(BTreeMap::new()));
-
let fd = this.machine.fds.insert_new(Epoll::default());
interp_ok(Scalar::from_i32(fd))
}
@@ -261,8 +265,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Throw EINVAL if epfd and fd have the same value.
if epfd_value == fd {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
return interp_ok(Scalar::from_i32(-1));
}
@@ -378,7 +381,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
drop(epoll_interest);
// Remove related epoll_interest from ready list.
- ready_list.borrow_mut().remove(&epoll_key);
+ ready_list.mapping.borrow_mut().remove(&epoll_key);
// Remove dangling EpollEventInterest from its global table.
// .unwrap() below should succeed because the file description id must have registered
@@ -443,8 +446,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let timeout = this.read_scalar(timeout)?.to_i32()?;
if epfd_value <= 0 || maxevents <= 0 {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
this.write_int(-1, dest)?;
return interp_ok(());
}
@@ -473,8 +475,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let epoll_file_description = epfd
.downcast::<Epoll>()
.ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?;
- let binding = epoll_file_description.get_ready_list();
- ready_list_empty = binding.borrow_mut().is_empty();
+ ready_list_empty = epoll_file_description.ready_list.mapping.borrow().is_empty();
thread_ids = epoll_file_description.thread_id.borrow_mut();
}
if timeout == 0 || !ready_list_empty {
@@ -563,9 +564,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// holds a strong ref to epoll_interest.
let epfd = epoll_interest.borrow().weak_epfd.upgrade().unwrap();
// FIXME: We can randomly pick a thread to unblock.
- if let Some(thread_id) =
- epfd.downcast::<Epoll>().unwrap().thread_id.borrow_mut().pop()
- {
+
+ let epoll = epfd.downcast::<Epoll>().unwrap();
+
+ // Synchronize running thread to the epoll ready list.
+ if let Some(clock) = &this.release_clock() {
+ epoll.ready_list.clock.borrow_mut().join(clock);
+ }
+
+ if let Some(thread_id) = epoll.thread_id.borrow_mut().pop() {
waiter.push(thread_id);
};
}
@@ -619,7 +626,7 @@ fn check_and_update_one_event_interest<'tcx>(
// insert an epoll_return to the ready list.
if flags != 0 {
let epoll_key = (id, epoll_event_interest.fd_num);
- let ready_list = &mut epoll_event_interest.ready_list.borrow_mut();
+ let ready_list = &mut epoll_event_interest.ready_list.mapping.borrow_mut();
let event_instance = EpollEventInstance::new(flags, epoll_event_interest.data);
// Triggers the notification by inserting it to the ready list.
ready_list.insert(epoll_key, event_instance);
@@ -646,7 +653,11 @@ fn blocking_epoll_callback<'tcx>(
.ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?;
let ready_list = epoll_file_description.get_ready_list();
- let mut ready_list = ready_list.borrow_mut();
+
+ // Synchronize waking thread from the epoll ready list.
+ ecx.acquire_clock(&ready_list.clock.borrow());
+
+ let mut ready_list = ready_list.mapping.borrow_mut();
let mut num_of_events: i32 = 0;
let mut array_iter = ecx.project_array_fields(events)?;
diff --git a/src/tools/miri/src/shims/unix/linux/eventfd.rs b/src/tools/miri/src/shims/unix/linux/eventfd.rs
index 12e99644357a6..910ab7e90f2a4 100644
--- a/src/tools/miri/src/shims/unix/linux/eventfd.rs
+++ b/src/tools/miri/src/shims/unix/linux/eventfd.rs
@@ -1,7 +1,7 @@
//! Linux `eventfd` implementation.
use std::cell::{Cell, RefCell};
use std::io;
-use std::io::{Error, ErrorKind};
+use std::io::ErrorKind;
use crate::concurrency::VClock;
use crate::shims::unix::fd::FileDescriptionRef;
@@ -66,9 +66,7 @@ impl FileDescription for Event {
let ty = ecx.machine.layouts.u64;
// Check the size of slice, and return error only if the size of the slice < 8.
if len < ty.size.bytes_usize() {
- ecx.set_last_error_from_io_error(Error::from(ErrorKind::InvalidInput))?;
- ecx.write_int(-1, dest)?;
- return interp_ok(());
+ return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest);
}
// eventfd read at the size of u64.
@@ -78,9 +76,7 @@ impl FileDescription for Event {
let counter = self.counter.get();
if counter == 0 {
if self.is_nonblock {
- ecx.set_last_error_from_io_error(Error::from(ErrorKind::WouldBlock))?;
- ecx.write_int(-1, dest)?;
- return interp_ok(());
+ return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest);
}
throw_unsup_format!("eventfd: blocking is unsupported");
@@ -128,8 +124,7 @@ impl FileDescription for Event {
let ty = ecx.machine.layouts.u64;
// Check the size of slice, and return error only if the size of the slice < 8.
if len < ty.layout.size.bytes_usize() {
- let result = Err(Error::from(ErrorKind::InvalidInput));
- return ecx.return_written_byte_count_or_error(result, dest);
+ return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest);
}
// Read the user supplied value from the pointer.
@@ -138,8 +133,7 @@ impl FileDescription for Event {
// u64::MAX as input is invalid because the maximum value of counter is u64::MAX - 1.
if num == u64::MAX {
- let result = Err(Error::from(ErrorKind::InvalidInput));
- return ecx.return_written_byte_count_or_error(result, dest);
+ return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest);
}
// If the addition does not let the counter to exceed the maximum value, update the counter.
// Else, block.
@@ -153,8 +147,7 @@ impl FileDescription for Event {
}
None | Some(u64::MAX) =>
if self.is_nonblock {
- let result = Err(Error::from(ErrorKind::WouldBlock));
- return ecx.return_written_byte_count_or_error(result, dest);
+ return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest);
} else {
throw_unsup_format!("eventfd: blocking is unsupported");
},
diff --git a/src/tools/miri/src/shims/unix/linux/foreign_items.rs b/src/tools/miri/src/shims/unix/linux/foreign_items.rs
index 9726dac7e516b..4b5f3b6c81bae 100644
--- a/src/tools/miri/src/shims/unix/linux/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/linux/foreign_items.rs
@@ -122,19 +122,19 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
id if id == sys_getrandom => {
// Used by getrandom 0.1
// The first argument is the syscall id, so skip over it.
- if args.len() < 4 {
+ let [_, ptr, len, flags, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for `getrandom` syscall: got {}, expected at least 4",
args.len()
);
- }
+ };
- let ptr = this.read_pointer(&args[1])?;
- let len = this.read_target_usize(&args[2])?;
+ let ptr = this.read_pointer(ptr)?;
+ let len = this.read_target_usize(len)?;
// The only supported flags are GRND_RANDOM and GRND_NONBLOCK,
// neither of which have any effect on our current PRNG.
// See <https://github.com/rust-lang/rust/pull/79196> for a discussion of argument sizes.
- let _flags = this.read_scalar(&args[3])?.to_i32()?;
+ let _flags = this.read_scalar(flags)?.to_i32()?;
this.gen_random(ptr, len)?;
this.write_scalar(Scalar::from_target_usize(len, this), dest)?;
diff --git a/src/tools/miri/src/shims/unix/linux/sync.rs b/src/tools/miri/src/shims/unix/linux/sync.rs
index 5108bc8299cbb..5833ec64fc68f 100644
--- a/src/tools/miri/src/shims/unix/linux/sync.rs
+++ b/src/tools/miri/src/shims/unix/linux/sync.rs
@@ -15,19 +15,19 @@ pub fn futex<'tcx>(
// may or may not be left out from the `syscall()` call.
// Therefore we don't use `check_arg_count` here, but only check for the
// number of arguments to fall within a range.
- if args.len() < 3 {
+ let [addr, op, val, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for `futex` syscall: got {}, expected at least 3",
args.len()
);
- }
+ };
// The first three arguments (after the syscall number itself) are the same to all futex operations:
// (int *addr, int op, int val).
// We checked above that these definitely exist.
- let addr = this.read_pointer(&args[0])?;
- let op = this.read_scalar(&args[1])?.to_i32()?;
- let val = this.read_scalar(&args[2])?.to_i32()?;
+ let addr = this.read_pointer(addr)?;
+ let op = this.read_scalar(op)?.to_i32()?;
+ let val = this.read_scalar(val)?.to_i32()?;
// This is a vararg function so we have to bring our own type for this pointer.
let addr = this.ptr_to_mplace(addr, this.machine.layouts.i32);
@@ -55,15 +55,15 @@ pub fn futex<'tcx>(
let wait_bitset = op & !futex_realtime == futex_wait_bitset;
let bitset = if wait_bitset {
- if args.len() < 6 {
+ let [_, _, _, timeout, uaddr2, bitset, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for `futex` syscall with `op=FUTEX_WAIT_BITSET`: got {}, expected at least 6",
args.len()
);
- }
- let _timeout = this.read_pointer(&args[3])?;
- let _uaddr2 = this.read_pointer(&args[4])?;
- this.read_scalar(&args[5])?.to_u32()?
+ };
+ let _timeout = this.read_pointer(timeout)?;
+ let _uaddr2 = this.read_pointer(uaddr2)?;
+ this.read_scalar(bitset)?.to_u32()?
} else {
if args.len() < 4 {
throw_ub_format!(
@@ -75,8 +75,7 @@ pub fn futex<'tcx>(
};
if bitset == 0 {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
this.write_scalar(Scalar::from_target_isize(-1, this), dest)?;
return interp_ok(());
}
@@ -88,8 +87,7 @@ pub fn futex<'tcx>(
let duration = match this.read_timespec(&timeout)? {
Some(duration) => duration,
None => {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
this.write_scalar(Scalar::from_target_isize(-1, this), dest)?;
return interp_ok(());
}
@@ -185,21 +183,20 @@ pub fn futex<'tcx>(
// Same as FUTEX_WAKE, but allows you to specify a bitset to select which threads to wake up.
op if op == futex_wake || op == futex_wake_bitset => {
let bitset = if op == futex_wake_bitset {
- if args.len() < 6 {
+ let [_, _, _, timeout, uaddr2, bitset, ..] = args else {
throw_ub_format!(
"incorrect number of arguments for `futex` syscall with `op=FUTEX_WAKE_BITSET`: got {}, expected at least 6",
args.len()
);
- }
- let _timeout = this.read_pointer(&args[3])?;
- let _uaddr2 = this.read_pointer(&args[4])?;
- this.read_scalar(&args[5])?.to_u32()?
+ };
+ let _timeout = this.read_pointer(timeout)?;
+ let _uaddr2 = this.read_pointer(uaddr2)?;
+ this.read_scalar(bitset)?.to_u32()?
} else {
u32::MAX
};
if bitset == 0 {
- let einval = this.eval_libc("EINVAL");
- this.set_last_error(einval)?;
+ this.set_last_error(LibcError("EINVAL"))?;
this.write_scalar(Scalar::from_target_isize(-1, this), dest)?;
return interp_ok(());
}
diff --git a/src/tools/miri/src/shims/unix/sync.rs b/src/tools/miri/src/shims/unix/sync.rs
index 017291f81a22e..b05f340861e78 100644
--- a/src/tools/miri/src/shims/unix/sync.rs
+++ b/src/tools/miri/src/shims/unix/sync.rs
@@ -483,7 +483,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Trying to acquire the same mutex again.
match kind {
MutexKind::Default =>
- throw_ub_format!("trying to acquire already locked default mutex"),
+ throw_ub_format!(
+ "trying to acquire default mutex already locked by the current thread"
+ ),
MutexKind::Normal => throw_machine_stop!(TerminationInfo::Deadlock),
MutexKind::ErrorCheck => this.eval_libc_i32("EDEADLK"),
MutexKind::Recursive => {
diff --git a/src/tools/miri/src/shims/unix/unnamed_socket.rs b/src/tools/miri/src/shims/unix/unnamed_socket.rs
index 763f9f2402768..faa54c6a75e93 100644
--- a/src/tools/miri/src/shims/unix/unnamed_socket.rs
+++ b/src/tools/miri/src/shims/unix/unnamed_socket.rs
@@ -5,7 +5,7 @@
use std::cell::{Cell, OnceCell, RefCell};
use std::collections::VecDeque;
use std::io;
-use std::io::{Error, ErrorKind, Read};
+use std::io::{ErrorKind, Read};
use rustc_target::abi::Size;
@@ -138,8 +138,7 @@ impl FileDescription for AnonSocket {
// Always succeed on read size 0.
if len == 0 {
- let result = Ok(0);
- return ecx.return_read_bytes_and_count(ptr, &bytes, result, dest);
+ return ecx.return_read_success(ptr, &bytes, 0, dest);
}
let Some(readbuf) = &self.readbuf else {
@@ -152,8 +151,7 @@ impl FileDescription for AnonSocket {
if self.peer_fd().upgrade().is_none() {
// Socketpair with no peer and empty buffer.
// 0 bytes successfully read indicates end-of-file.
- let result = Ok(0);
- return ecx.return_read_bytes_and_count(ptr, &bytes, result, dest);
+ return ecx.return_read_success(ptr, &bytes, 0, dest);
} else {
if self.is_nonblock {
// Non-blocking socketpair with writer and empty buffer.
@@ -161,8 +159,7 @@ impl FileDescription for AnonSocket {
// EAGAIN or EWOULDBLOCK can be returned for socket,
// POSIX.1-2001 allows either error to be returned for this case.
// Since there is no ErrorKind for EAGAIN, WouldBlock is used.
- let result = Err(Error::from(ErrorKind::WouldBlock));
- return ecx.return_read_bytes_and_count(ptr, &bytes, result, dest);
+ return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest);
} else {
// Blocking socketpair with writer and empty buffer.
// FIXME: blocking is currently not supported
@@ -194,8 +191,7 @@ impl FileDescription for AnonSocket {
ecx.check_and_update_readiness(&peer_fd)?;
}
- let result = Ok(actual_read_size);
- ecx.return_read_bytes_and_count(ptr, &bytes, result, dest)
+ ecx.return_read_success(ptr, &bytes, actual_read_size, dest)
}
fn write<'tcx>(
@@ -210,16 +206,14 @@ impl FileDescription for AnonSocket {
// Always succeed on write size 0.
// ("If count is zero and fd refers to a file other than a regular file, the results are not specified.")
if len == 0 {
- let result = Ok(0);
- return ecx.return_written_byte_count_or_error(result, dest);
+ return ecx.return_write_success(0, dest);
}
// We are writing to our peer's readbuf.
let Some(peer_fd) = self.peer_fd().upgrade() else {
// If the upgrade from Weak to Rc fails, it indicates that all read ends have been
// closed.
- let result = Err(Error::from(ErrorKind::BrokenPipe));
- return ecx.return_written_byte_count_or_error(result, dest);
+ return ecx.set_last_error_and_return(ErrorKind::BrokenPipe, dest);
};
let Some(writebuf) = &peer_fd.downcast::<AnonSocket>().unwrap().readbuf else {
@@ -233,8 +227,7 @@ impl FileDescription for AnonSocket {
if available_space == 0 {
if self.is_nonblock {
// Non-blocking socketpair with a full buffer.
- let result = Err(Error::from(ErrorKind::WouldBlock));
- return ecx.return_written_byte_count_or_error(result, dest);
+ return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest);
} else {
// Blocking socketpair with a full buffer.
throw_unsup_format!("socketpair write: blocking isn't supported yet");
@@ -256,8 +249,7 @@ impl FileDescription for AnonSocket {
// The kernel does this even if the fd was already readable before, so we follow suit.
ecx.check_and_update_readiness(&peer_fd)?;
- let result = Ok(actual_write_size);
- ecx.return_written_byte_count_or_error(result, dest)
+ ecx.return_write_success(actual_write_size, dest)
}
}
diff --git a/src/tools/miri/src/shims/windows/env.rs b/src/tools/miri/src/shims/windows/env.rs
index a6ace6f9bdc4c..72c1fb58023a8 100644
--- a/src/tools/miri/src/shims/windows/env.rs
+++ b/src/tools/miri/src/shims/windows/env.rs
@@ -150,7 +150,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`GetCurrentDirectoryW`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(Scalar::from_u32(0));
}
@@ -163,7 +163,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
this.write_path_to_wide_str(&cwd, buf, size)?,
)));
}
- Err(e) => this.set_last_error_from_io_error(e)?,
+ Err(e) => this.set_last_error(e)?,
}
interp_ok(Scalar::from_u32(0))
}
@@ -182,7 +182,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
if let IsolatedOp::Reject(reject_with) = this.machine.isolated_op {
this.reject_in_isolation("`SetCurrentDirectoryW`", reject_with)?;
- this.set_last_error_from_io_error(ErrorKind::PermissionDenied.into())?;
+ this.set_last_error(ErrorKind::PermissionDenied)?;
return interp_ok(this.eval_windows("c", "FALSE"));
}
@@ -190,7 +190,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
match env::set_current_dir(path) {
Ok(()) => interp_ok(this.eval_windows("c", "TRUE")),
Err(e) => {
- this.set_last_error_from_io_error(e)?;
+ this.set_last_error(e)?;
interp_ok(this.eval_windows("c", "FALSE"))
}
}
diff --git a/src/tools/miri/src/shims/windows/foreign_items.rs b/src/tools/miri/src/shims/windows/foreign_items.rs
index a4f40f9447e79..dee778876f6b0 100644
--- a/src/tools/miri/src/shims/windows/foreign_items.rs
+++ b/src/tools/miri/src/shims/windows/foreign_items.rs
@@ -227,7 +227,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let filename = this.read_path_from_wide_str(filename)?;
let result = match win_absolute(&filename)? {
Err(err) => {
- this.set_last_error_from_io_error(err)?;
+ this.set_last_error(err)?;
Scalar::from_u32(0) // return zero upon failure
}
Ok(abs_filename) => {
diff --git a/src/tools/miri/src/shims/x86/gfni.rs b/src/tools/miri/src/shims/x86/gfni.rs
new file mode 100644
index 0000000000000..c91b8c835f2e9
--- /dev/null
+++ b/src/tools/miri/src/shims/x86/gfni.rs
@@ -0,0 +1,196 @@
+use rustc_span::Symbol;
+use rustc_target::spec::abi::Abi;
+
+use crate::*;
+
+impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
+pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
+ fn emulate_x86_gfni_intrinsic(
+ &mut self,
+ link_name: Symbol,
+ abi: Abi,
+ args: &[OpTy<'tcx>],
+ dest: &MPlaceTy<'tcx>,
+ ) -> InterpResult<'tcx, EmulateItemResult> {
+ let this = self.eval_context_mut();
+
+ // Prefix should have already been checked.
+ let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.").unwrap();
+
+ this.expect_target_feature_for_intrinsic(link_name, "gfni")?;
+ if unprefixed_name.ends_with(".256") {
+ this.expect_target_feature_for_intrinsic(link_name, "avx")?;
+ } else if unprefixed_name.ends_with(".512") {
+ this.expect_target_feature_for_intrinsic(link_name, "avx512f")?;
+ }
+
+ match unprefixed_name {
+ // Used to implement the `_mm{, 256, 512}_gf2p8affine_epi64_epi8` functions.
+ // See `affine_transform` for details.
+ // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8affine_
+ "vgf2p8affineqb.128" | "vgf2p8affineqb.256" | "vgf2p8affineqb.512" => {
+ let [left, right, imm8] =
+ this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
+ affine_transform(this, left, right, imm8, dest, /* inverse */ false)?;
+ }
+ // Used to implement the `_mm{, 256, 512}_gf2p8affineinv_epi64_epi8` functions.
+ // See `affine_transform` for details.
+ // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8affineinv
+ "vgf2p8affineinvqb.128" | "vgf2p8affineinvqb.256" | "vgf2p8affineinvqb.512" => {
+ let [left, right, imm8] =
+ this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
+ affine_transform(this, left, right, imm8, dest, /* inverse */ true)?;
+ }
+ // Used to implement the `_mm{, 256, 512}_gf2p8mul_epi8` functions.
+ // Multiplies packed 8-bit integers in `left` and `right` in the finite field GF(2^8)
+ // and store the results in `dst`. The field GF(2^8) is represented in
+ // polynomial representation with the reduction polynomial x^8 + x^4 + x^3 + x + 1.
+ // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8mul
+ "vgf2p8mulb.128" | "vgf2p8mulb.256" | "vgf2p8mulb.512" => {
+ let [left, right] =
+ this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
+
+ let (left, left_len) = this.project_to_simd(left)?;
+ let (right, right_len) = this.project_to_simd(right)?;
+ let (dest, dest_len) = this.project_to_simd(dest)?;
+
+ assert_eq!(left_len, right_len);
+ assert_eq!(dest_len, right_len);
+
+ for i in 0..dest_len {
+ let left = this.read_scalar(&this.project_index(&left, i)?)?.to_u8()?;
+ let right = this.read_scalar(&this.project_index(&right, i)?)?.to_u8()?;
+ let dest = this.project_index(&dest, i)?;
+ this.write_scalar(Scalar::from_u8(gf2p8_mul(left, right)), &dest)?;
+ }
+ }
+ _ => return interp_ok(EmulateItemResult::NotSupported),
+ }
+ interp_ok(EmulateItemResult::NeedsReturn)
+ }
+}
+
+/// Calculates the affine transformation `right * left + imm8` inside the finite field GF(2^8).
+/// `right` is an 8x8 bit matrix, `left` and `imm8` are bit vectors.
+/// If `inverse` is set, then the inverse transformation with respect to the reduction polynomial
+/// x^8 + x^4 + x^3 + x + 1 is performed instead.
+fn affine_transform<'tcx>(
+ this: &mut MiriInterpCx<'tcx>,
+ left: &OpTy<'tcx>,
+ right: &OpTy<'tcx>,
+ imm8: &OpTy<'tcx>,
+ dest: &MPlaceTy<'tcx>,
+ inverse: bool,
+) -> InterpResult<'tcx, ()> {
+ let (left, left_len) = this.project_to_simd(left)?;
+ let (right, right_len) = this.project_to_simd(right)?;
+ let (dest, dest_len) = this.project_to_simd(dest)?;
+
+ assert_eq!(dest_len, right_len);
+ assert_eq!(dest_len, left_len);
+
+ let imm8 = this.read_scalar(imm8)?.to_u8()?;
+
+ // Each 8x8 bit matrix gets multiplied with eight bit vectors.
+ // Therefore, the iteration is done in chunks of eight.
+ for i in (0..dest_len).step_by(8) {
+ // Get the bit matrix.
+ let mut matrix = [0u8; 8];
+ for j in 0..8 {
+ matrix[usize::try_from(j).unwrap()] =
+ this.read_scalar(&this.project_index(&right, i.wrapping_add(j))?)?.to_u8()?;
+ }
+
+ // Multiply the matrix with the vector and perform the addition.
+ for j in 0..8 {
+ let index = i.wrapping_add(j);
+ let left = this.read_scalar(&this.project_index(&left, index)?)?.to_u8()?;
+ let left = if inverse { TABLE[usize::from(left)] } else { left };
+
+ let mut res = 0;
+
+ // Do the matrix multiplication.
+ for bit in 0u8..8 {
+ let mut b = matrix[usize::from(bit)] & left;
+
+ // Calculate the parity bit.
+ b = (b & 0b1111) ^ (b >> 4);
+ b = (b & 0b11) ^ (b >> 2);
+ b = (b & 0b1) ^ (b >> 1);
+
+ res |= b << 7u8.wrapping_sub(bit);
+ }
+
+ // Perform the addition.
+ res ^= imm8;
+
+ let dest = this.project_index(&dest, index)?;
+ this.write_scalar(Scalar::from_u8(res), &dest)?;
+ }
+ }
+
+ interp_ok(())
+}
+
+/// A lookup table for computing the inverse byte for the inverse affine transformation.
+// This is a evaluated at compile time. Trait based conversion is not available.
+/// See <https://www.corsix.org/content/galois-field-instructions-2021-cpus> for the
+/// definition of `gf_inv` which was used for the creation of this table.
+#[allow(clippy::cast_possible_truncation)]
+static TABLE: [u8; 256] = {
+ let mut array = [0; 256];
+
+ let mut i = 1;
+ while i < 256 {
+ let mut x = i as u8;
+ let mut y = gf2p8_mul(x, x);
+ x = y;
+ let mut j = 2;
+ while j < 8 {
+ x = gf2p8_mul(x, x);
+ y = gf2p8_mul(x, y);
+ j += 1;
+ }
+ array[i] = y;
+ i += 1;
+ }
+
+ array
+};
+
+/// Multiplies packed 8-bit integers in `left` and `right` in the finite field GF(2^8)
+/// and store the results in `dst`. The field GF(2^8) is represented in
+/// polynomial representation with the reduction polynomial x^8 + x^4 + x^3 + x + 1.
+/// See <https://www.corsix.org/content/galois-field-instructions-2021-cpus> for details.
+// This is a const function. Trait based conversion is not available.
+#[allow(clippy::cast_possible_truncation)]
+const fn gf2p8_mul(left: u8, right: u8) -> u8 {
+ // This implementation is based on the `gf2p8mul_byte` definition found inside the Intel intrinsics guide.
+ // See https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8mul
+ // for more information.
+
+ const POLYNOMIAL: u32 = 0x11b;
+
+ let left = left as u32;
+ let right = right as u32;
+
+ let mut result = 0u32;
+
+ let mut i = 0u32;
+ while i < 8 {
+ if left & (1 << i) != 0 {
+ result ^= right << i;
+ }
+ i = i.wrapping_add(1);
+ }
+
+ let mut i = 14u32;
+ while i >= 8 {
+ if result & (1 << i) != 0 {
+ result ^= POLYNOMIAL << i.wrapping_sub(8);
+ }
+ i = i.wrapping_sub(1);
+ }
+
+ result as u8
+}
diff --git a/src/tools/miri/src/shims/x86/mod.rs b/src/tools/miri/src/shims/x86/mod.rs
index 19c678cb7faca..9339d301aeecd 100644
--- a/src/tools/miri/src/shims/x86/mod.rs
+++ b/src/tools/miri/src/shims/x86/mod.rs
@@ -15,6 +15,7 @@ mod aesni;
mod avx;
mod avx2;
mod bmi;
+mod gfni;
mod sha;
mod sse;
mod sse2;
@@ -106,6 +107,13 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
this, link_name, abi, args, dest,
);
}
+ // The GFNI extension does not get its own namespace.
+ // Check for instruction names instead.
+ name if name.starts_with("vgf2p8affine") || name.starts_with("vgf2p8mulb") => {
+ return gfni::EvalContextExt::emulate_x86_gfni_intrinsic(
+ this, link_name, abi, args, dest,
+ );
+ }
name if name.starts_with("sha") => {
return sha::EvalContextExt::emulate_x86_sha_intrinsic(
this, link_name, abi, args, dest,
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.rs b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.rs
similarity index 72%
rename from src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.rs
rename to src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.rs
index a79abe65328e0..f2df8bdca12bd 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.rs
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.rs
@@ -1,12 +1,12 @@
//@ignore-target: windows # No pthreads on Windows
//
-// Check that if we pass NULL attribute, then we get the default mutex type.
+// Check that if we pass NULL attribute, then reentrant locking is UB.
fn main() {
unsafe {
let mut mutex: libc::pthread_mutex_t = std::mem::zeroed();
assert_eq!(libc::pthread_mutex_init(&mut mutex as *mut _, std::ptr::null() as *const _), 0);
assert_eq!(libc::pthread_mutex_lock(&mut mutex as *mut _), 0);
- libc::pthread_mutex_lock(&mut mutex as *mut _); //~ ERROR: Undefined Behavior: trying to acquire already locked default mutex
+ libc::pthread_mutex_lock(&mut mutex as *mut _); //~ ERROR: already locked by the current thread
}
}
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.stderr
similarity index 68%
rename from src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.stderr
rename to src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.stderr
index a57d10753d90d..9455e70437629 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.stderr
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.stderr
@@ -1,13 +1,13 @@
-error: Undefined Behavior: trying to acquire already locked default mutex
- --> tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.rs:LL:CC
+error: Undefined Behavior: trying to acquire default mutex already locked by the current thread
+ --> tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.rs:LL:CC
|
LL | libc::pthread_mutex_lock(&mut mutex as *mut _);
- | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ trying to acquire already locked default mutex
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ trying to acquire default mutex already locked by the current thread
|
= help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
= help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
= note: BACKTRACE:
- = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.rs:LL:CC
+ = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_NULL_reentrant.rs:LL:CC
note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.rs b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.rs
similarity index 52%
rename from src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.rs
rename to src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.rs
index d9293f938b6aa..d2d0ffff07a92 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_deadlock.rs
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.rs
@@ -1,6 +1,11 @@
//@ignore-target: windows # No pthreads on Windows
//
-// Check that if we do not set the mutex type, it is the default.
+// Check that if we do not set the mutex type, it is UB to do reentrant locking. glibc apparently
+// actually exploits this, see
+// <https://github.molgen.mpg.de/git-mirror/glibc/blob/master/nptl/pthread_mutexattr_settype.c#L31>:
+// one must actively call pthread_mutexattr_settype to disable lock elision. This means a call to
+// pthread_mutexattr_settype(PTHREAD_MUTEX_NORMAL) makes a difference even if
+// PTHREAD_MUTEX_NORMAL == PTHREAD_MUTEX_DEFAULT!
fn main() {
unsafe {
@@ -9,6 +14,6 @@ fn main() {
let mut mutex: libc::pthread_mutex_t = std::mem::zeroed();
assert_eq!(libc::pthread_mutex_init(&mut mutex as *mut _, &mutexattr as *const _), 0);
assert_eq!(libc::pthread_mutex_lock(&mut mutex as *mut _), 0);
- libc::pthread_mutex_lock(&mut mutex as *mut _); //~ ERROR: Undefined Behavior: trying to acquire already locked default mutex
+ libc::pthread_mutex_lock(&mut mutex as *mut _); //~ ERROR: already locked by the current thread
}
}
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.stderr
similarity index 68%
rename from src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.stderr
rename to src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.stderr
index e9961ed413d0c..a9ffbde1b65cf 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.stderr
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.stderr
@@ -1,13 +1,13 @@
-error: Undefined Behavior: trying to acquire already locked default mutex
- --> tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.rs:LL:CC
+error: Undefined Behavior: trying to acquire default mutex already locked by the current thread
+ --> tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.rs:LL:CC
|
LL | libc::pthread_mutex_lock(&mut mutex as *mut _);
- | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ trying to acquire already locked default mutex
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ trying to acquire default mutex already locked by the current thread
|
= help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
= help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
= note: BACKTRACE:
- = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_NULL_deadlock.rs:LL:CC
+ = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_default_reentrant.rs:LL:CC
note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.rs b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.rs
similarity index 81%
rename from src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.rs
rename to src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.rs
index b38582482b88b..9a88639edf795 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.rs
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.rs
@@ -10,6 +10,8 @@ fn main() {
let mut mutex: libc::pthread_mutex_t = std::mem::zeroed();
assert_eq!(libc::pthread_mutex_init(&mut mutex as *mut _, &mutexattr as *const _), 0);
assert_eq!(libc::pthread_mutex_lock(&mut mutex as *mut _), 0);
+ // A "normal" mutex properly tries to acquire the lock even if its is already held
+ // by the current thread -- and then we deadlock.
libc::pthread_mutex_lock(&mut mutex as *mut _); //~ ERROR: deadlock: the evaluated program deadlocked
}
}
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.stderr
similarity index 79%
rename from src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.stderr
rename to src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.stderr
index 4337475963e4a..f20b26297e274 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.stderr
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.stderr
@@ -1,11 +1,11 @@
error: deadlock: the evaluated program deadlocked
- --> tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.rs:LL:CC
+ --> tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.rs:LL:CC
|
LL | libc::pthread_mutex_lock(&mut mutex as *mut _);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program deadlocked
|
= note: BACKTRACE:
- = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_normal_deadlock.rs:LL:CC
+ = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_normal_reentrant.rs:LL:CC
note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.rs b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.rs
new file mode 100644
index 0000000000000..bd8aef787e6fb
--- /dev/null
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.rs
@@ -0,0 +1,12 @@
+//@ignore-target: windows # No pthreads on Windows
+//
+// Check that if we use PTHREAD_MUTEX_INITIALIZER, then reentrant locking is UB.
+// glibc apparently actually exploits this so we better catch it!
+
+fn main() {
+ unsafe {
+ let mut mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER;
+ assert_eq!(libc::pthread_mutex_lock(&mut mutex as *mut _), 0);
+ libc::pthread_mutex_lock(&mut mutex as *mut _); //~ ERROR: already locked by the current thread
+ }
+}
diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.stderr
new file mode 100644
index 0000000000000..984bb07b72895
--- /dev/null
+++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.stderr
@@ -0,0 +1,15 @@
+error: Undefined Behavior: trying to acquire default mutex already locked by the current thread
+ --> tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.rs:LL:CC
+ |
+LL | libc::pthread_mutex_lock(&mut mutex as *mut _);
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ trying to acquire default mutex already locked by the current thread
+ |
+ = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+ = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+ = note: BACKTRACE:
+ = note: inside `main` at tests/fail-dep/concurrency/libc_pthread_mutex_staticinit_reentrant.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/fail/data_race/mixed_size_read_read_write.rs b/src/tools/miri/tests/fail/data_race/mixed_size_read_read_write.rs
index ece5bd31274de..e76654806bb1e 100644
--- a/src/tools/miri/tests/fail/data_race/mixed_size_read_read_write.rs
+++ b/src/tools/miri/tests/fail/data_race/mixed_size_read_read_write.rs
@@ -4,7 +4,7 @@
// Two variants: the atomic store matches the size of the first or second atomic load.
//@revisions: match_first_load match_second_load
-use std::sync::atomic::{AtomicU16, AtomicU8, Ordering};
+use std::sync::atomic::{AtomicU8, AtomicU16, Ordering};
use std::thread;
fn convert(a: &AtomicU16) -> &[AtomicU8; 2] {
diff --git a/src/tools/miri/tests/fail/data_race/mixed_size_read_write.rs b/src/tools/miri/tests/fail/data_race/mixed_size_read_write.rs
index acc12427b3fdc..53016bab78045 100644
--- a/src/tools/miri/tests/fail/data_race/mixed_size_read_write.rs
+++ b/src/tools/miri/tests/fail/data_race/mixed_size_read_write.rs
@@ -4,7 +4,7 @@
// Two revisions, depending on which access goes first.
//@revisions: read_write write_read
-use std::sync::atomic::{AtomicU16, AtomicU8, Ordering};
+use std::sync::atomic::{AtomicU8, AtomicU16, Ordering};
use std::thread;
fn convert(a: &AtomicU16) -> &[AtomicU8; 2] {
diff --git a/src/tools/miri/tests/fail/data_race/mixed_size_write_write.fst.stderr b/src/tools/miri/tests/fail/data_race/mixed_size_write_write.fst.stderr
new file mode 100644
index 0000000000000..a353910dcc998
--- /dev/null
+++ b/src/tools/miri/tests/fail/data_race/mixed_size_write_write.fst.stderr
@@ -0,0 +1,22 @@
+error: Undefined Behavior: Race condition detected between (1) 2-byte atomic store on thread `unnamed-ID` and (2) 1-byte atomic store on thread `unnamed-ID` at ALLOC. (2) just happened here
+ --> tests/fail/data_race/mixed_size_write_write.rs:LL:CC
+ |
+LL | a8[idx].store(1, Ordering::SeqCst);
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Race condition detected between (1) 2-byte atomic store on thread `unnamed-ID` and (2) 1-byte atomic store on thread `unnamed-ID` at ALLOC. (2) just happened here
+ |
+help: and (1) occurred earlier here
+ --> tests/fail/data_race/mixed_size_write_write.rs:LL:CC
+ |
+LL | a16.store(1, Ordering::SeqCst);
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ = help: overlapping unsynchronized atomic accesses must use the same access size
+ = help: see https://doc.rust-lang.org/nightly/std/sync/atomic/index.html#memory-model-for-atomic-accesses for more information about the Rust memory model
+ = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+ = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+ = note: BACKTRACE (of the first span) on thread `unnamed-ID`:
+ = note: inside closure at tests/fail/data_race/mixed_size_write_write.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/fail/data_race/mixed_size_write_write.rs b/src/tools/miri/tests/fail/data_race/mixed_size_write_write.rs
index 89afda2fff5c2..545e354a0372c 100644
--- a/src/tools/miri/tests/fail/data_race/mixed_size_write_write.rs
+++ b/src/tools/miri/tests/fail/data_race/mixed_size_write_write.rs
@@ -1,6 +1,7 @@
//@compile-flags: -Zmiri-preemption-rate=0.0 -Zmiri-disable-weak-memory-emulation
// Avoid accidental synchronization via address reuse inside `thread::spawn`.
//@compile-flags: -Zmiri-address-reuse-cross-thread-rate=0
+//@revisions: fst snd
use std::sync::atomic::{AtomicU8, AtomicU16, Ordering};
use std::thread;
@@ -21,7 +22,8 @@ fn main() {
a16.store(1, Ordering::SeqCst);
});
s.spawn(|| {
- a8[0].store(1, Ordering::SeqCst);
+ let idx = if cfg!(fst) { 0 } else { 1 };
+ a8[idx].store(1, Ordering::SeqCst);
//~^ ERROR: Race condition detected between (1) 2-byte atomic store on thread `unnamed-1` and (2) 1-byte atomic store on thread `unnamed-2`
});
});
diff --git a/src/tools/miri/tests/fail/data_race/mixed_size_write_write.snd.stderr b/src/tools/miri/tests/fail/data_race/mixed_size_write_write.snd.stderr
new file mode 100644
index 0000000000000..3b9c0491502a1
--- /dev/null
+++ b/src/tools/miri/tests/fail/data_race/mixed_size_write_write.snd.stderr
@@ -0,0 +1,22 @@
+error: Undefined Behavior: Race condition detected between (1) 2-byte atomic store on thread `unnamed-ID` and (2) 1-byte atomic store on thread `unnamed-ID` at ALLOC+0x1. (2) just happened here
+ --> tests/fail/data_race/mixed_size_write_write.rs:LL:CC
+ |
+LL | a8[idx].store(1, Ordering::SeqCst);
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Race condition detected between (1) 2-byte atomic store on thread `unnamed-ID` and (2) 1-byte atomic store on thread `unnamed-ID` at ALLOC+0x1. (2) just happened here
+ |
+help: and (1) occurred earlier here
+ --> tests/fail/data_race/mixed_size_write_write.rs:LL:CC
+ |
+LL | a16.store(1, Ordering::SeqCst);
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ = help: overlapping unsynchronized atomic accesses must use the same access size
+ = help: see https://doc.rust-lang.org/nightly/std/sync/atomic/index.html#memory-model-for-atomic-accesses for more information about the Rust memory model
+ = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+ = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+ = note: BACKTRACE (of the first span) on thread `unnamed-ID`:
+ = note: inside closure at tests/fail/data_race/mixed_size_write_write.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs b/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs
index eb38529ae57df..d7675a40163c5 100644
--- a/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs
+++ b/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs
@@ -1,5 +1,5 @@
//@only-target: linux
-// test_epoll_block_then_unblock depends on a deterministic schedule.
+// test_epoll_block_then_unblock and test_epoll_race depend on a deterministic schedule.
//@compile-flags: -Zmiri-preemption-rate=0
use std::convert::TryInto;
@@ -12,6 +12,7 @@ fn main() {
test_epoll_block_without_notification();
test_epoll_block_then_unblock();
test_notification_after_timeout();
+ test_epoll_race();
}
// Using `as` cast since `EPOLLET` wraps around
@@ -137,3 +138,41 @@ fn test_notification_after_timeout() {
let expected_value = fds[0] as u64;
check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 10);
}
+
+// This test shows a data_race before epoll had vector clocks added.
+fn test_epoll_race() {
+ // Create an epoll instance.
+ let epfd = unsafe { libc::epoll_create1(0) };
+ assert_ne!(epfd, -1);
+
+ // Create an eventfd instance.
+ let flags = libc::EFD_NONBLOCK | libc::EFD_CLOEXEC;
+ let fd = unsafe { libc::eventfd(0, flags) };
+
+ // Register eventfd with the epoll instance.
+ let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd as u64 };
+ let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
+ assert_eq!(res, 0);
+
+ static mut VAL: u8 = 0;
+ let thread1 = thread::spawn(move || {
+ // Write to the static mut variable.
+ unsafe { VAL = 1 };
+ // Write to the eventfd instance.
+ let sized_8_data: [u8; 8] = 1_u64.to_ne_bytes();
+ let res = unsafe { libc::write(fd, sized_8_data.as_ptr() as *const libc::c_void, 8) };
+ // read returns number of bytes that have been read, which is always 8.
+ assert_eq!(res, 8);
+ });
+ thread::yield_now();
+ // epoll_wait for the event to happen.
+ let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap();
+ let expected_value = u64::try_from(fd).unwrap();
+ check_epoll_wait::<8>(epfd, &[(expected_event, expected_value)], -1);
+ // Read from the static mut variable.
+ #[allow(static_mut_refs)]
+ unsafe {
+ assert_eq!(VAL, 1)
+ };
+ thread1.join().unwrap();
+}
diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-gfni.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-gfni.rs
new file mode 100644
index 0000000000000..a629e2acfe998
--- /dev/null
+++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-gfni.rs
@@ -0,0 +1,518 @@
+// We're testing x86 target specific features
+//@only-target: x86_64 i686
+//@compile-flags: -C target-feature=+gfni,+avx512f
+
+// The constants in the tests below are just bit patterns. They should not
+// be interpreted as integers; signedness does not make sense for them, but
+// __mXXXi happens to be defined in terms of signed integers.
+#![allow(overflowing_literals)]
+#![feature(avx512_target_feature)]
+#![feature(stdarch_x86_avx512)]
+
+#[cfg(target_arch = "x86")]
+use std::arch::x86::*;
+#[cfg(target_arch = "x86_64")]
+use std::arch::x86_64::*;
+use std::hint::black_box;
+use std::mem::{size_of, transmute};
+
+const IDENTITY_BYTE: i32 = 0;
+const CONSTANT_BYTE: i32 = 0x63;
+
+fn main() {
+ // Mostly copied from library/stdarch/crates/core_arch/src/x86/gfni.rs
+
+ assert!(is_x86_feature_detected!("avx512f"));
+ assert!(is_x86_feature_detected!("gfni"));
+
+ unsafe {
+ let byte_mul_test_data = generate_byte_mul_test_data();
+ let affine_mul_test_data_identity = generate_affine_mul_test_data(IDENTITY_BYTE as u8);
+ let affine_mul_test_data_constant = generate_affine_mul_test_data(CONSTANT_BYTE as u8);
+ let inv_tests_data = generate_inv_tests_data();
+
+ test_mm512_gf2p8mul_epi8(&byte_mul_test_data);
+ test_mm256_gf2p8mul_epi8(&byte_mul_test_data);
+ test_mm_gf2p8mul_epi8(&byte_mul_test_data);
+ test_mm512_gf2p8affine_epi64_epi8(&byte_mul_test_data, &affine_mul_test_data_identity);
+ test_mm256_gf2p8affine_epi64_epi8(&byte_mul_test_data, &affine_mul_test_data_identity);
+ test_mm_gf2p8affine_epi64_epi8(&byte_mul_test_data, &affine_mul_test_data_identity);
+ test_mm512_gf2p8affineinv_epi64_epi8(&inv_tests_data, &affine_mul_test_data_constant);
+ test_mm256_gf2p8affineinv_epi64_epi8(&inv_tests_data, &affine_mul_test_data_constant);
+ test_mm_gf2p8affineinv_epi64_epi8(&inv_tests_data, &affine_mul_test_data_constant);
+ }
+}
+
+#[target_feature(enable = "gfni,avx512f")]
+unsafe fn test_mm512_gf2p8mul_epi8(
+ byte_mul_test_data: &([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]),
+) {
+ let (left, right, expected) = byte_mul_test_data;
+
+ for i in 0..NUM_TEST_WORDS_512 {
+ let left = load_m512i_word(left, i);
+ let right = load_m512i_word(right, i);
+ let expected = load_m512i_word(expected, i);
+ let result = _mm512_gf2p8mul_epi8(left, right);
+ assert_eq_m512i(result, expected);
+ }
+}
+
+#[target_feature(enable = "gfni,avx")]
+unsafe fn test_mm256_gf2p8mul_epi8(
+ byte_mul_test_data: &([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]),
+) {
+ let (left, right, expected) = byte_mul_test_data;
+
+ for i in 0..NUM_TEST_WORDS_256 {
+ let left = load_m256i_word(left, i);
+ let right = load_m256i_word(right, i);
+ let expected = load_m256i_word(expected, i);
+ let result = _mm256_gf2p8mul_epi8(left, right);
+ assert_eq_m256i(result, expected);
+ }
+}
+
+#[target_feature(enable = "gfni")]
+unsafe fn test_mm_gf2p8mul_epi8(
+ byte_mul_test_data: &([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]),
+) {
+ let (left, right, expected) = byte_mul_test_data;
+
+ for i in 0..NUM_TEST_WORDS_128 {
+ let left = load_m128i_word(left, i);
+ let right = load_m128i_word(right, i);
+ let expected = load_m128i_word(expected, i);
+ let result = _mm_gf2p8mul_epi8(left, right);
+ assert_eq_m128i(result, expected);
+ }
+}
+
+#[target_feature(enable = "gfni,avx512f")]
+unsafe fn test_mm512_gf2p8affine_epi64_epi8(
+ byte_mul_test_data: &([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]),
+ affine_mul_test_data_identity: &(
+ [u64; NUM_TEST_WORDS_64],
+ [u8; NUM_TEST_ENTRIES],
+ [u8; NUM_TEST_ENTRIES],
+ ),
+) {
+ let identity: i64 = 0x01_02_04_08_10_20_40_80;
+ let constant: i64 = 0;
+ let identity = _mm512_set1_epi64(identity);
+ let constant = _mm512_set1_epi64(constant);
+ let constant_reference = _mm512_set1_epi8(CONSTANT_BYTE as i8);
+
+ let (bytes, more_bytes, _) = byte_mul_test_data;
+ let (matrices, vectors, references) = affine_mul_test_data_identity;
+
+ for i in 0..NUM_TEST_WORDS_512 {
+ let data = load_m512i_word(bytes, i);
+ let result = _mm512_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(data, identity);
+ assert_eq_m512i(result, data);
+ let result = _mm512_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(data, constant);
+ assert_eq_m512i(result, constant_reference);
+ let data = load_m512i_word(more_bytes, i);
+ let result = _mm512_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(data, identity);
+ assert_eq_m512i(result, data);
+ let result = _mm512_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(data, constant);
+ assert_eq_m512i(result, constant_reference);
+
+ let matrix = load_m512i_word(matrices, i);
+ let vector = load_m512i_word(vectors, i);
+ let reference = load_m512i_word(references, i);
+
+ let result = _mm512_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(vector, matrix);
+ assert_eq_m512i(result, reference);
+ }
+}
+
+#[target_feature(enable = "gfni,avx")]
+unsafe fn test_mm256_gf2p8affine_epi64_epi8(
+ byte_mul_test_data: &([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]),
+ affine_mul_test_data_identity: &(
+ [u64; NUM_TEST_WORDS_64],
+ [u8; NUM_TEST_ENTRIES],
+ [u8; NUM_TEST_ENTRIES],
+ ),
+) {
+ let identity: i64 = 0x01_02_04_08_10_20_40_80;
+ let constant: i64 = 0;
+ let identity = _mm256_set1_epi64x(identity);
+ let constant = _mm256_set1_epi64x(constant);
+ let constant_reference = _mm256_set1_epi8(CONSTANT_BYTE as i8);
+
+ let (bytes, more_bytes, _) = byte_mul_test_data;
+ let (matrices, vectors, references) = affine_mul_test_data_identity;
+
+ for i in 0..NUM_TEST_WORDS_256 {
+ let data = load_m256i_word(bytes, i);
+ let result = _mm256_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(data, identity);
+ assert_eq_m256i(result, data);
+ let result = _mm256_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(data, constant);
+ assert_eq_m256i(result, constant_reference);
+ let data = load_m256i_word(more_bytes, i);
+ let result = _mm256_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(data, identity);
+ assert_eq_m256i(result, data);
+ let result = _mm256_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(data, constant);
+ assert_eq_m256i(result, constant_reference);
+
+ let matrix = load_m256i_word(matrices, i);
+ let vector = load_m256i_word(vectors, i);
+ let reference = load_m256i_word(references, i);
+
+ let result = _mm256_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(vector, matrix);
+ assert_eq_m256i(result, reference);
+ }
+}
+
+#[target_feature(enable = "gfni")]
+unsafe fn test_mm_gf2p8affine_epi64_epi8(
+ byte_mul_test_data: &([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]),
+ affine_mul_test_data_identity: &(
+ [u64; NUM_TEST_WORDS_64],
+ [u8; NUM_TEST_ENTRIES],
+ [u8; NUM_TEST_ENTRIES],
+ ),
+) {
+ let identity: i64 = 0x01_02_04_08_10_20_40_80;
+ let constant: i64 = 0;
+ let identity = _mm_set1_epi64x(identity);
+ let constant = _mm_set1_epi64x(constant);
+ let constant_reference = _mm_set1_epi8(CONSTANT_BYTE as i8);
+
+ let (bytes, more_bytes, _) = byte_mul_test_data;
+ let (matrices, vectors, references) = affine_mul_test_data_identity;
+
+ for i in 0..NUM_TEST_WORDS_128 {
+ let data = load_m128i_word(bytes, i);
+ let result = _mm_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(data, identity);
+ assert_eq_m128i(result, data);
+ let result = _mm_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(data, constant);
+ assert_eq_m128i(result, constant_reference);
+ let data = load_m128i_word(more_bytes, i);
+ let result = _mm_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(data, identity);
+ assert_eq_m128i(result, data);
+ let result = _mm_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(data, constant);
+ assert_eq_m128i(result, constant_reference);
+
+ let matrix = load_m128i_word(matrices, i);
+ let vector = load_m128i_word(vectors, i);
+ let reference = load_m128i_word(references, i);
+
+ let result = _mm_gf2p8affine_epi64_epi8::<IDENTITY_BYTE>(vector, matrix);
+ assert_eq_m128i(result, reference);
+ }
+}
+
+#[target_feature(enable = "gfni,avx512f")]
+unsafe fn test_mm512_gf2p8affineinv_epi64_epi8(
+ inv_tests_data: &([u8; NUM_BYTES], [u8; NUM_BYTES]),
+ affine_mul_test_data_constant: &(
+ [u64; NUM_TEST_WORDS_64],
+ [u8; NUM_TEST_ENTRIES],
+ [u8; NUM_TEST_ENTRIES],
+ ),
+) {
+ let identity: i64 = 0x01_02_04_08_10_20_40_80;
+ let identity = _mm512_set1_epi64(identity);
+
+ // validate inversion
+ let (inputs, results) = inv_tests_data;
+
+ for i in 0..NUM_BYTES_WORDS_512 {
+ let input = load_m512i_word(inputs, i);
+ let reference = load_m512i_word(results, i);
+ let result = _mm512_gf2p8affineinv_epi64_epi8::<IDENTITY_BYTE>(input, identity);
+ let remultiplied = _mm512_gf2p8mul_epi8(result, input);
+ assert_eq_m512i(remultiplied, reference);
+ }
+
+ // validate subsequent affine operation
+ let (matrices, vectors, _affine_expected) = affine_mul_test_data_constant;
+
+ for i in 0..NUM_TEST_WORDS_512 {
+ let vector = load_m512i_word(vectors, i);
+ let matrix = load_m512i_word(matrices, i);
+
+ let inv_vec = _mm512_gf2p8affineinv_epi64_epi8::<IDENTITY_BYTE>(vector, identity);
+ let reference = _mm512_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(inv_vec, matrix);
+ let result = _mm512_gf2p8affineinv_epi64_epi8::<CONSTANT_BYTE>(vector, matrix);
+ assert_eq_m512i(result, reference);
+ }
+
+ // validate everything by virtue of checking against the AES SBox
+ const AES_S_BOX_MATRIX: i64 = 0xF1_E3_C7_8F_1F_3E_7C_F8;
+ let sbox_matrix = _mm512_set1_epi64(AES_S_BOX_MATRIX);
+
+ for i in 0..NUM_BYTES_WORDS_512 {
+ let reference = load_m512i_word(&AES_S_BOX, i);
+ let input = load_m512i_word(inputs, i);
+ let result = _mm512_gf2p8affineinv_epi64_epi8::<CONSTANT_BYTE>(input, sbox_matrix);
+ assert_eq_m512i(result, reference);
+ }
+}
+
+#[target_feature(enable = "gfni,avx")]
+unsafe fn test_mm256_gf2p8affineinv_epi64_epi8(
+ inv_tests_data: &([u8; NUM_BYTES], [u8; NUM_BYTES]),
+ affine_mul_test_data_constant: &(
+ [u64; NUM_TEST_WORDS_64],
+ [u8; NUM_TEST_ENTRIES],
+ [u8; NUM_TEST_ENTRIES],
+ ),
+) {
+ let identity: i64 = 0x01_02_04_08_10_20_40_80;
+ let identity = _mm256_set1_epi64x(identity);
+
+ // validate inversion
+ let (inputs, results) = inv_tests_data;
+
+ for i in 0..NUM_BYTES_WORDS_256 {
+ let input = load_m256i_word(inputs, i);
+ let reference = load_m256i_word(results, i);
+ let result = _mm256_gf2p8affineinv_epi64_epi8::<IDENTITY_BYTE>(input, identity);
+ let remultiplied = _mm256_gf2p8mul_epi8(result, input);
+ assert_eq_m256i(remultiplied, reference);
+ }
+
+ // validate subsequent affine operation
+ let (matrices, vectors, _affine_expected) = affine_mul_test_data_constant;
+
+ for i in 0..NUM_TEST_WORDS_256 {
+ let vector = load_m256i_word(vectors, i);
+ let matrix = load_m256i_word(matrices, i);
+
+ let inv_vec = _mm256_gf2p8affineinv_epi64_epi8::<IDENTITY_BYTE>(vector, identity);
+ let reference = _mm256_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(inv_vec, matrix);
+ let result = _mm256_gf2p8affineinv_epi64_epi8::<CONSTANT_BYTE>(vector, matrix);
+ assert_eq_m256i(result, reference);
+ }
+
+ // validate everything by virtue of checking against the AES SBox
+ const AES_S_BOX_MATRIX: i64 = 0xF1_E3_C7_8F_1F_3E_7C_F8;
+ let sbox_matrix = _mm256_set1_epi64x(AES_S_BOX_MATRIX);
+
+ for i in 0..NUM_BYTES_WORDS_256 {
+ let reference = load_m256i_word(&AES_S_BOX, i);
+ let input = load_m256i_word(inputs, i);
+ let result = _mm256_gf2p8affineinv_epi64_epi8::<CONSTANT_BYTE>(input, sbox_matrix);
+ assert_eq_m256i(result, reference);
+ }
+}
+
+#[target_feature(enable = "gfni")]
+unsafe fn test_mm_gf2p8affineinv_epi64_epi8(
+ inv_tests_data: &([u8; NUM_BYTES], [u8; NUM_BYTES]),
+ affine_mul_test_data_constant: &(
+ [u64; NUM_TEST_WORDS_64],
+ [u8; NUM_TEST_ENTRIES],
+ [u8; NUM_TEST_ENTRIES],
+ ),
+) {
+ let identity: i64 = 0x01_02_04_08_10_20_40_80;
+ let identity = _mm_set1_epi64x(identity);
+
+ // validate inversion
+ let (inputs, results) = inv_tests_data;
+
+ for i in 0..NUM_BYTES_WORDS_128 {
+ let input = load_m128i_word(inputs, i);
+ let reference = load_m128i_word(results, i);
+ let result = _mm_gf2p8affineinv_epi64_epi8::<IDENTITY_BYTE>(input, identity);
+ let remultiplied = _mm_gf2p8mul_epi8(result, input);
+ assert_eq_m128i(remultiplied, reference);
+ }
+
+ // validate subsequent affine operation
+ let (matrices, vectors, _affine_expected) = affine_mul_test_data_constant;
+
+ for i in 0..NUM_TEST_WORDS_128 {
+ let vector = load_m128i_word(vectors, i);
+ let matrix = load_m128i_word(matrices, i);
+
+ let inv_vec = _mm_gf2p8affineinv_epi64_epi8::<IDENTITY_BYTE>(vector, identity);
+ let reference = _mm_gf2p8affine_epi64_epi8::<CONSTANT_BYTE>(inv_vec, matrix);
+ let result = _mm_gf2p8affineinv_epi64_epi8::<CONSTANT_BYTE>(vector, matrix);
+ assert_eq_m128i(result, reference);
+ }
+
+ // validate everything by virtue of checking against the AES SBox
+ const AES_S_BOX_MATRIX: i64 = 0xF1_E3_C7_8F_1F_3E_7C_F8;
+ let sbox_matrix = _mm_set1_epi64x(AES_S_BOX_MATRIX);
+
+ for i in 0..NUM_BYTES_WORDS_128 {
+ let reference = load_m128i_word(&AES_S_BOX, i);
+ let input = load_m128i_word(inputs, i);
+ let result = _mm_gf2p8affineinv_epi64_epi8::<CONSTANT_BYTE>(input, sbox_matrix);
+ assert_eq_m128i(result, reference);
+ }
+}
+
+/* Various utilities for processing SIMD values. */
+
+#[target_feature(enable = "sse2")]
+unsafe fn load_m128i_word<T>(data: &[T], word_index: usize) -> __m128i {
+ let byte_offset = word_index * 16 / size_of::<T>();
+ let pointer = data.as_ptr().add(byte_offset) as *const __m128i;
+ _mm_loadu_si128(black_box(pointer))
+}
+
+#[target_feature(enable = "avx")]
+unsafe fn load_m256i_word<T>(data: &[T], word_index: usize) -> __m256i {
+ let byte_offset = word_index * 32 / size_of::<T>();
+ let pointer = data.as_ptr().add(byte_offset) as *const __m256i;
+ _mm256_loadu_si256(black_box(pointer))
+}
+
+#[target_feature(enable = "avx512f")]
+unsafe fn load_m512i_word<T>(data: &[T], word_index: usize) -> __m512i {
+ let byte_offset = word_index * 64 / size_of::<T>();
+ let pointer = data.as_ptr().add(byte_offset) as *const i32;
+ _mm512_loadu_si512(black_box(pointer))
+}
+
+#[track_caller]
+#[target_feature(enable = "sse2")]
+unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) {
+ assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b))
+}
+
+#[track_caller]
+#[target_feature(enable = "avx")]
+unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) {
+ assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b))
+}
+
+#[track_caller]
+#[target_feature(enable = "avx512f")]
+unsafe fn assert_eq_m512i(a: __m512i, b: __m512i) {
+ assert_eq!(transmute::<_, [u64; 8]>(a), transmute::<_, [u64; 8]>(b))
+}
+
+/* Software implementation of the hardware intrinsics. */
+
+fn mulbyte(left: u8, right: u8) -> u8 {
+ // this implementation follows the description in
+ // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_gf2p8mul_epi8
+ const REDUCTION_POLYNOMIAL: u16 = 0x11b;
+ let left: u16 = left.into();
+ let right: u16 = right.into();
+ let mut carryless_product: u16 = 0;
+
+ // Carryless multiplication
+ for i in 0..8 {
+ if ((left >> i) & 0x01) != 0 {
+ carryless_product ^= right << i;
+ }
+ }
+
+ // reduction, adding in "0" where appropriate to clear out high bits
+ // note that REDUCTION_POLYNOMIAL is zero in this context
+ for i in (8..=14).rev() {
+ if ((carryless_product >> i) & 0x01) != 0 {
+ carryless_product ^= REDUCTION_POLYNOMIAL << (i - 8);
+ }
+ }
+
+ carryless_product as u8
+}
+
+/// Calculates the bitwise XOR of all bits inside a byte.
+fn parity(input: u8) -> u8 {
+ let mut accumulator = 0;
+ for i in 0..8 {
+ accumulator ^= (input >> i) & 0x01;
+ }
+ accumulator
+}
+
+/// Calculates `matrix * x + b` inside the finite field GF(2).
+fn mat_vec_multiply_affine(matrix: u64, x: u8, b: u8) -> u8 {
+ // this implementation follows the description in
+ // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_gf2p8affine_epi64_epi8
+ let mut accumulator = 0;
+
+ for bit in 0..8 {
+ accumulator |= parity(x & matrix.to_le_bytes()[bit]) << (7 - bit);
+ }
+
+ accumulator ^ b
+}
+
+/* Test data generation. */
+
+const NUM_TEST_WORDS_512: usize = 4;
+const NUM_TEST_WORDS_256: usize = NUM_TEST_WORDS_512 * 2;
+const NUM_TEST_WORDS_128: usize = NUM_TEST_WORDS_256 * 2;
+const NUM_TEST_ENTRIES: usize = NUM_TEST_WORDS_512 * 64;
+const NUM_TEST_WORDS_64: usize = NUM_TEST_WORDS_128 * 2;
+const NUM_BYTES: usize = 256;
+const NUM_BYTES_WORDS_128: usize = NUM_BYTES / 16;
+const NUM_BYTES_WORDS_256: usize = NUM_BYTES_WORDS_128 / 2;
+const NUM_BYTES_WORDS_512: usize = NUM_BYTES_WORDS_256 / 2;
+
+fn generate_affine_mul_test_data(
+ immediate: u8,
+) -> ([u64; NUM_TEST_WORDS_64], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]) {
+ let mut left: [u64; NUM_TEST_WORDS_64] = [0; NUM_TEST_WORDS_64];
+ let mut right: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES];
+ let mut result: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES];
+
+ for i in 0..NUM_TEST_WORDS_64 {
+ left[i] = (i as u64) * 103 * 101;
+ for j in 0..8 {
+ let j64 = j as u64;
+ right[i * 8 + j] = ((left[i] + j64) % 256) as u8;
+ result[i * 8 + j] = mat_vec_multiply_affine(left[i], right[i * 8 + j], immediate);
+ }
+ }
+
+ (left, right, result)
+}
+
+fn generate_inv_tests_data() -> ([u8; NUM_BYTES], [u8; NUM_BYTES]) {
+ let mut input: [u8; NUM_BYTES] = [0; NUM_BYTES];
+ let mut result: [u8; NUM_BYTES] = [0; NUM_BYTES];
+
+ for i in 0..NUM_BYTES {
+ input[i] = (i % 256) as u8;
+ result[i] = if i == 0 { 0 } else { 1 };
+ }
+
+ (input, result)
+}
+
+const AES_S_BOX: [u8; NUM_BYTES] = [
+ 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
+ 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
+ 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+ 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
+ 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
+ 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+ 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
+ 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
+ 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+ 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
+ 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
+ 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+ 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
+ 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
+ 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+ 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
+];
+
+fn generate_byte_mul_test_data()
+-> ([u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES], [u8; NUM_TEST_ENTRIES]) {
+ let mut left: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES];
+ let mut right: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES];
+ let mut result: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES];
+
+ for i in 0..NUM_TEST_ENTRIES {
+ left[i] = (i % 256) as u8;
+ right[i] = left[i].wrapping_mul(101);
+ result[i] = mulbyte(left[i], right[i]);
+ }
+
+ (left, right, result)
+}
diff --git a/tests/ui/float/classify-runtime-const.rs b/tests/ui/float/classify-runtime-const.rs
index 2a24e51cabbc4..ca852ea2468bc 100644
--- a/tests/ui/float/classify-runtime-const.rs
+++ b/tests/ui/float/classify-runtime-const.rs
@@ -6,8 +6,8 @@
// This tests the float classification functions, for regular runtime code and for const evaluation.
-#![feature(f16_const)]
-#![feature(f128_const)]
+#![feature(f16)]
+#![feature(f128)]
use std::num::FpCategory::*;
diff --git a/tests/ui/float/conv-bits-runtime-const.rs b/tests/ui/float/conv-bits-runtime-const.rs
index 60c45cc4cc194..3046728fe66ff 100644
--- a/tests/ui/float/conv-bits-runtime-const.rs
+++ b/tests/ui/float/conv-bits-runtime-const.rs
@@ -5,8 +5,6 @@
#![feature(f16)]
#![feature(f128)]
-#![feature(f16_const)]
-#![feature(f128_const)]
#![allow(unused_macro_rules)]
use std::hint::black_box;