Skip to content

Commit c3722ba

Browse files
bors[bot]taiki-e
bors[bot]
and
taiki-e
authored
Merge #17
17: Allow CAS failure ordering stronger than success ordering r=taiki-e a=taiki-e See rust-lang/rust#98383 and [p0418r2](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html). Co-authored-by: Taiki Endo <te316e89@gmail.com>
2 parents 1a23b74 + 6ffe826 commit c3722ba

File tree

9 files changed

+357
-218
lines changed

9 files changed

+357
-218
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com
1010

1111
## [Unreleased]
1212

13+
- Allow CAS failure ordering stronger than success ordering.
14+
1315
## [0.3.2] - 2022-06-19
1416

1517
- Optimize x86_64 128-bit atomic load/store on Intel CPU with AVX. ([#16](https://github.com/taiki-e/portable-atomic/pull/16))

‎src/imp/atomic128/aarch64.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,8 +325,10 @@ unsafe fn atomic_compare_exchange(
325325
old: u128,
326326
new: u128,
327327
success: Ordering,
328-
_failure: Ordering,
328+
failure: Ordering,
329329
) -> Result<u128, u128> {
330+
let success = crate::utils::upgrade_success_ordering(success, failure);
331+
330332
#[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
331333
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
332334
// cfg guarantee that the CPU supports FEAT_LSE.

‎src/imp/atomic128/powerpc64.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,11 @@ unsafe fn atomic_compare_exchange(
153153
old: u128,
154154
new: u128,
155155
success: Ordering,
156-
_failure: Ordering,
156+
failure: Ordering,
157157
) -> Result<u128, u128> {
158+
debug_assert!(dst as usize % 16 == 0);
159+
let success = crate::utils::upgrade_success_ordering(success, failure);
160+
158161
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
159162
let res = unsafe {
160163
let old = U128 { whole: old };

‎src/imp/atomic128/x86_64.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ unsafe fn cmpxchg16b(
154154
#[inline]
155155
unsafe fn _atomic_load_vmovdqa(src: *mut u128, _order: Ordering) -> u128 {
156156
debug_assert!(src as usize % 16 == 0);
157+
157158
// SAFETY: the caller must uphold the safety contract for `_atomic_load_vmovdqa`.
158159
unsafe {
159160
let out: core::arch::x86_64::__m128;
@@ -170,6 +171,7 @@ unsafe fn _atomic_load_vmovdqa(src: *mut u128, _order: Ordering) -> u128 {
170171
#[inline]
171172
unsafe fn _atomic_store_vmovdqa(dst: *mut u128, val: u128, _order: Ordering) {
172173
debug_assert!(dst as usize % 16 == 0);
174+
173175
// SAFETY: the caller must uphold the safety contract for `_atomic_store_vmovdqa`.
174176
unsafe {
175177
let val: core::arch::x86_64::__m128 = core::mem::transmute(val);
@@ -274,6 +276,7 @@ unsafe fn atomic_compare_exchange(
274276
success: Ordering,
275277
failure: Ordering,
276278
) -> Result<u128, u128> {
279+
let success = crate::utils::upgrade_success_ordering(success, failure);
277280
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
278281
let (res, ok) = unsafe { cmpxchg16b(dst, old, new, success, failure) };
279282
if ok {

‎src/imp/core_atomic.rs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
// Wrap the standard library's atomic types in newtype.
2+
//
3+
// This is not a reexport, because we want to backport changes like
4+
// https://github.com/rust-lang/rust/pull/98383 to old compilers.
25

36
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
47
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
@@ -25,6 +28,34 @@ impl AtomicBool {
2528
pub(crate) fn into_inner(self) -> bool {
2629
self.inner.into_inner()
2730
}
31+
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
32+
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
33+
#[inline]
34+
pub(crate) fn compare_exchange(
35+
&self,
36+
current: bool,
37+
new: bool,
38+
success: Ordering,
39+
failure: Ordering,
40+
) -> Result<bool, bool> {
41+
// TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
42+
let success = crate::utils::upgrade_success_ordering(success, failure);
43+
self.inner.compare_exchange(current, new, success, failure)
44+
}
45+
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
46+
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
47+
#[inline]
48+
pub(crate) fn compare_exchange_weak(
49+
&self,
50+
current: bool,
51+
new: bool,
52+
success: Ordering,
53+
failure: Ordering,
54+
) -> Result<bool, bool> {
55+
// TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
56+
let success = crate::utils::upgrade_success_ordering(success, failure);
57+
self.inner.compare_exchange_weak(current, new, success, failure)
58+
}
2859
}
2960
impl core::ops::Deref for AtomicBool {
3061
type Target = core::sync::atomic::AtomicBool;
@@ -61,6 +92,34 @@ impl<T> AtomicPtr<T> {
6192
pub(crate) fn into_inner(self) -> *mut T {
6293
self.inner.into_inner()
6394
}
95+
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
96+
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
97+
#[inline]
98+
pub(crate) fn compare_exchange(
99+
&self,
100+
current: *mut T,
101+
new: *mut T,
102+
success: Ordering,
103+
failure: Ordering,
104+
) -> Result<*mut T, *mut T> {
105+
// TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
106+
let success = crate::utils::upgrade_success_ordering(success, failure);
107+
self.inner.compare_exchange(current, new, success, failure)
108+
}
109+
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
110+
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
111+
#[inline]
112+
pub(crate) fn compare_exchange_weak(
113+
&self,
114+
current: *mut T,
115+
new: *mut T,
116+
success: Ordering,
117+
failure: Ordering,
118+
) -> Result<*mut T, *mut T> {
119+
// TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
120+
let success = crate::utils::upgrade_success_ordering(success, failure);
121+
self.inner.compare_exchange_weak(current, new, success, failure)
122+
}
64123
}
65124
impl<T> core::ops::Deref for AtomicPtr<T> {
66125
type Target = core::sync::atomic::AtomicPtr<T>;
@@ -99,6 +158,46 @@ macro_rules! atomic_int {
99158
pub(crate) fn into_inner(self) -> $int_type {
100159
self.inner.into_inner()
101160
}
161+
#[cfg_attr(
162+
portable_atomic_no_cfg_target_has_atomic,
163+
cfg(not(portable_atomic_no_atomic_cas))
164+
)]
165+
#[cfg_attr(
166+
not(portable_atomic_no_cfg_target_has_atomic),
167+
cfg(target_has_atomic = "ptr")
168+
)]
169+
#[inline]
170+
pub(crate) fn compare_exchange(
171+
&self,
172+
current: $int_type,
173+
new: $int_type,
174+
success: Ordering,
175+
failure: Ordering,
176+
) -> Result<$int_type, $int_type> {
177+
// TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
178+
let success = crate::utils::upgrade_success_ordering(success, failure);
179+
self.inner.compare_exchange(current, new, success, failure)
180+
}
181+
#[cfg_attr(
182+
portable_atomic_no_cfg_target_has_atomic,
183+
cfg(not(portable_atomic_no_atomic_cas))
184+
)]
185+
#[cfg_attr(
186+
not(portable_atomic_no_cfg_target_has_atomic),
187+
cfg(target_has_atomic = "ptr")
188+
)]
189+
#[inline]
190+
pub(crate) fn compare_exchange_weak(
191+
&self,
192+
current: $int_type,
193+
new: $int_type,
194+
success: Ordering,
195+
failure: Ordering,
196+
) -> Result<$int_type, $int_type> {
197+
// TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
198+
let success = crate::utils::upgrade_success_ordering(success, failure);
199+
self.inner.compare_exchange_weak(current, new, success, failure)
200+
}
102201
#[cfg(portable_atomic_no_atomic_min_max)]
103202
#[cfg_attr(
104203
portable_atomic_no_cfg_target_has_atomic,

‎src/lib.rs‎

Lines changed: 12 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -419,8 +419,7 @@ impl AtomicBool {
419419
/// `failure` describes the required ordering for the load operation that takes place when
420420
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
421421
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
422-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
423-
/// and must be equivalent to or weaker than the success ordering.
422+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
424423
#[cfg_attr(
425424
portable_atomic_no_cfg_target_has_atomic,
426425
cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -454,8 +453,7 @@ impl AtomicBool {
454453
/// `failure` describes the required ordering for the load operation that takes place when
455454
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
456455
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
457-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
458-
/// and must be equivalent to or weaker than the success ordering.
456+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
459457
#[cfg_attr(
460458
portable_atomic_no_cfg_target_has_atomic,
461459
cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -593,8 +591,7 @@ impl AtomicBool {
593591
/// Using [`Acquire`] as success ordering makes the store part of this
594592
/// operation [`Relaxed`], and using [`Release`] makes the final successful
595593
/// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],
596-
/// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the
597-
/// success ordering.
594+
/// [`Acquire`] or [`Relaxed`].
598595
#[cfg_attr(
599596
portable_atomic_no_cfg_target_has_atomic,
600597
cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -792,8 +789,7 @@ impl<T> AtomicPtr<T> {
792789
/// `failure` describes the required ordering for the load operation that takes place when
793790
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
794791
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
795-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
796-
/// and must be equivalent to or weaker than the success ordering.
792+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
797793
#[cfg_attr(
798794
portable_atomic_no_cfg_target_has_atomic,
799795
cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -827,8 +823,7 @@ impl<T> AtomicPtr<T> {
827823
/// `failure` describes the required ordering for the load operation that takes place when
828824
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
829825
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
830-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
831-
/// and must be equivalent to or weaker than the success ordering.
826+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
832827
#[cfg_attr(
833828
portable_atomic_no_cfg_target_has_atomic,
834829
cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -867,8 +862,7 @@ impl<T> AtomicPtr<T> {
867862
/// Using [`Acquire`] as success ordering makes the store part of this
868863
/// operation [`Relaxed`], and using [`Release`] makes the final successful
869864
/// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],
870-
/// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the
871-
/// success ordering.
865+
/// [`Acquire`] or [`Relaxed`].
872866
#[cfg_attr(
873867
portable_atomic_no_cfg_target_has_atomic,
874868
cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -1087,8 +1081,7 @@ atomic instructions or locks will be used.
10871081
/// `failure` describes the required ordering for the load operation that takes place when
10881082
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
10891083
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
1090-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
1091-
/// and must be equivalent to or weaker than the success ordering.
1084+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
10921085
#[cfg_attr(
10931086
portable_atomic_no_cfg_target_has_atomic,
10941087
cfg(any(
@@ -1126,8 +1119,7 @@ atomic instructions or locks will be used.
11261119
/// `failure` describes the required ordering for the load operation that takes place when
11271120
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
11281121
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
1129-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
1130-
/// and must be equivalent to or weaker than the success ordering.
1122+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
11311123
#[cfg_attr(
11321124
portable_atomic_no_cfg_target_has_atomic,
11331125
cfg(any(
@@ -1322,8 +1314,7 @@ atomic instructions or locks will be used.
13221314
///
13231315
/// Using [`Acquire`] as success ordering makes the store part
13241316
/// of this operation [`Relaxed`], and using [`Release`] makes the final successful load
1325-
/// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
1326-
/// and must be equivalent to or weaker than the success ordering.
1317+
/// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
13271318
#[cfg_attr(
13281319
portable_atomic_no_cfg_target_has_atomic,
13291320
cfg(any(
@@ -1585,8 +1576,7 @@ This type has the same in-memory representation as the underlying floating point
15851576
/// `failure` describes the required ordering for the load operation that takes place when
15861577
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
15871578
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
1588-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
1589-
/// and must be equivalent to or weaker than the success ordering.
1579+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
15901580
#[cfg_attr(
15911581
portable_atomic_no_cfg_target_has_atomic,
15921582
cfg(any(
@@ -1632,8 +1622,7 @@ This type has the same in-memory representation as the underlying floating point
16321622
/// `failure` describes the required ordering for the load operation that takes place when
16331623
/// the comparison fails. Using [`Acquire`] as success ordering makes the store part
16341624
/// of this operation [`Relaxed`], and using [`Release`] makes the successful load
1635-
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
1636-
/// and must be equivalent to or weaker than the success ordering.
1625+
/// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
16371626
#[cfg_attr(
16381627
portable_atomic_no_cfg_target_has_atomic,
16391628
cfg(any(
@@ -1734,8 +1723,7 @@ This type has the same in-memory representation as the underlying floating point
17341723
///
17351724
/// Using [`Acquire`] as success ordering makes the store part
17361725
/// of this operation [`Relaxed`], and using [`Release`] makes the final successful load
1737-
/// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
1738-
/// and must be equivalent to or weaker than the success ordering.
1726+
/// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
17391727
#[cfg_attr(
17401728
portable_atomic_no_cfg_target_has_atomic,
17411729
cfg(any(

0 commit comments

Comments
 (0)