Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 32 additions & 143 deletions library/core/src/sync/atomic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2337,10 +2337,7 @@ fn strongest_failure_ordering(order: Ordering) -> Ordering {
}

#[inline]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_store`.
unsafe {
match order {
Expand All @@ -2351,18 +2348,10 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
AcqRel => panic!("there is no such thing as an acquire/release store"),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_store`.
unsafe {
*dst = val;
}
}

#[inline]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
unsafe {
match order {
Expand All @@ -2373,19 +2362,11 @@ unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
AcqRel => panic!("there is no such thing as an acquire/release load"),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
unsafe {
*dst
}
}

#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
match order {
Expand All @@ -2396,22 +2377,12 @@ unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
SeqCst => intrinsics::atomic_xchg(dst, val),
}
}
#[cfg(any(target_arch = "bpf" ,target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
let old = *dst;
*dst = val;
old
}
}

/// Returns the previous value (like __sync_fetch_and_add).
#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_add<T: Copy + crate::ops::Add<Output = T>>(dst: *mut T, val: T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_add`.
unsafe {
match order {
Expand All @@ -2422,22 +2393,12 @@ unsafe fn atomic_add<T: Copy + crate::ops::Add<Output = T>>(dst: *mut T, val: T,
SeqCst => intrinsics::atomic_xadd(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_add`.
unsafe {
let old = *dst;
*dst = old + val;
old
}
}

/// Returns the previous value (like __sync_fetch_and_sub).
#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_sub<T: Copy + crate::ops::Sub<Output = T>>(dst: *mut T, val: T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_sub`.
unsafe {
match order {
Expand All @@ -2448,58 +2409,35 @@ unsafe fn atomic_sub<T: Copy + crate::ops::Sub<Output = T>>(dst: *mut T, val: T,
SeqCst => intrinsics::atomic_xsub(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_sub`.
unsafe {
let old = *dst;
*dst = old - val;
old
}
}

#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_compare_exchange<T: Copy + crate::cmp::PartialEq>(
dst: *mut T,
old: T,
new: T,
success: Ordering,
failure: Ordering,
) -> Result<T, T> {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
{
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
let (val, ok) = unsafe {
match (success, failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
};
if ok { Ok(val) } else { Err(val) }
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
unsafe {
let current = *dst;
if current == old {
*dst = new;
Ok(current)
} else {
Err(current)
let (val, ok) = unsafe {
match (success, failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
}
};
if ok { Ok(val) } else { Err(val) }
}

#[inline]
Expand All @@ -2511,38 +2449,24 @@ unsafe fn atomic_compare_exchange_weak<T: Copy + crate::cmp::PartialEq>(
_success: Ordering,
_failure: Ordering,
) -> Result<T, T> {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
{
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
let (val, ok) = unsafe {
match (_success, _failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
};
if ok { Ok(val) } else { Err(val) }
}
#[cfg(any(target_arch = "sbf", target_arch = "bpf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
unsafe {
let current = *dst;
if current == old {
*dst = new;
Ok(current)
} else {
Err(current)
let (val, ok) = unsafe {
match (_success, _failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
}
};
if ok { Ok(val) } else { Err(val) }
}

#[inline]
Expand Down Expand Up @@ -2609,7 +2533,6 @@ unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_max<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
match _order {
Expand All @@ -2620,18 +2543,12 @@ unsafe fn atomic_max<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ord
SeqCst => intrinsics::atomic_max(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
crate::cmp::max(*dst, val)
}
}

/// returns the min value (signed comparison)
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_min<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
match _order {
Expand All @@ -2642,18 +2559,12 @@ unsafe fn atomic_min<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ord
SeqCst => intrinsics::atomic_min(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
crate::cmp::min(*dst, val)
}
}

/// returns the max value (unsigned comparison)
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_umax<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_umax`
unsafe {
match _order {
Expand All @@ -2664,18 +2575,12 @@ unsafe fn atomic_umax<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Or
SeqCst => intrinsics::atomic_umax(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_umax`
unsafe {
crate::cmp::max(*dst, val)
}
}

/// returns the min value (unsigned comparison)
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_umin<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_umin`
unsafe {
match _order {
Expand All @@ -2686,11 +2591,6 @@ unsafe fn atomic_umin<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Or
SeqCst => intrinsics::atomic_umin(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_umin`
unsafe {
crate::cmp::min(*dst, val)
}
}

/// An atomic fence.
Expand Down Expand Up @@ -2770,16 +2670,8 @@ unsafe fn atomic_umin<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Or
#[inline]
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_diagnostic_item = "fence"]
#[cfg_attr(any(target_arch = "bpf", target_arch = "sbf", target_arch = "wasm32"), allow(unused_variables))]
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this remain for target_arch = "wasm32"?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Upstream doesn't seem to have it - is it something you somehow ran into? If it's something we added, it'd be great if we upstreamed it! But I don't think it makes much sense to keep it in our solana-* branches since we're not building wasm

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't check upstream. I thought we had from upstream. In this case, sure, let's remove it altogether.

pub fn fence(order: Ordering) {
#[cfg(not(any(target_arch = "wasm32", target_arch = "bpf", target_arch = "sbf")))]
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe keep the condition and comment for (not(target_arch = "wasm32"))?

// SAFETY: using an atomic fence is safe.
// On wasm32 and SBF it looks like fences aren't implemented in LLVM yet in that
// they will cause LLVM to abort. The wasm instruction set doesn't have
// fences right now. There's discussion online about the best way for tools
// to conventionally implement fences at
// https://github.com/WebAssembly/tool-conventions/issues/59. We should
// follow that discussion and implement a solution when one comes about!
unsafe {
match order {
Acquire => intrinsics::atomic_fence_acq(),
Expand Down Expand Up @@ -2860,10 +2752,7 @@ pub fn fence(order: Ordering) {
#[inline]
#[stable(feature = "compiler_fences", since = "1.21.0")]
#[rustc_diagnostic_item = "compiler_fence"]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
pub fn compiler_fence(order: Ordering) {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: using an atomic fence is safe.
unsafe {
match order {
Expand Down
Loading