1 #[cfg(not(crossbeam_no_atomic))]
2 use core::sync::atomic::Ordering;
3 
4 /// Trait which allows reading from primitive atomic types with "consume" ordering.
5 pub trait AtomicConsume {
6     /// Type returned by `load_consume`.
7     type Val;
8 
9     /// Loads a value from the atomic using a "consume" memory ordering.
10     ///
11     /// This is similar to the "acquire" ordering, except that an ordering is
12     /// only guaranteed with operations that "depend on" the result of the load.
13     /// However consume loads are usually much faster than acquire loads on
14     /// architectures with a weak memory model since they don't require memory
15     /// fence instructions.
16     ///
17     /// The exact definition of "depend on" is a bit vague, but it works as you
18     /// would expect in practice since a lot of software, especially the Linux
19     /// kernel, rely on this behavior.
20     ///
21     /// This is currently only implemented on ARM and AArch64, where a fence
22     /// can be avoided. On other architectures this will fall back to a simple
23     /// `load(Ordering::Acquire)`.
load_consume(&self) -> Self::Val24     fn load_consume(&self) -> Self::Val;
25 }
26 
27 #[cfg(not(crossbeam_no_atomic))]
28 // Miri and Loom don't support "consume" ordering and ThreadSanitizer doesn't treat
29 // load(Relaxed) + compiler_fence(Acquire) as "consume" load.
30 // LLVM generates machine code equivalent to fence(Acquire) in compiler_fence(Acquire)
31 // on PowerPC, MIPS, etc. (https://godbolt.org/z/hffvjvW7h), so for now the fence
32 // can be actually avoided here only on ARM and AArch64. See also
33 // https://github.com/rust-lang/rust/issues/62256.
34 #[cfg(all(
35     any(target_arch = "arm", target_arch = "aarch64"),
36     not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)),
37 ))]
38 macro_rules! impl_consume {
39     () => {
40         #[inline]
41         fn load_consume(&self) -> Self::Val {
42             use crate::primitive::sync::atomic::compiler_fence;
43             let result = self.load(Ordering::Relaxed);
44             compiler_fence(Ordering::Acquire);
45             result
46         }
47     };
48 }
49 
50 #[cfg(not(crossbeam_no_atomic))]
51 #[cfg(not(all(
52     any(target_arch = "arm", target_arch = "aarch64"),
53     not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)),
54 )))]
55 macro_rules! impl_consume {
56     () => {
57         #[inline]
58         fn load_consume(&self) -> Self::Val {
59             self.load(Ordering::Acquire)
60         }
61     };
62 }
63 
64 macro_rules! impl_atomic {
65     ($atomic:ident, $val:ty) => {
66         #[cfg(not(crossbeam_no_atomic))]
67         impl AtomicConsume for core::sync::atomic::$atomic {
68             type Val = $val;
69             impl_consume!();
70         }
71         #[cfg(crossbeam_loom)]
72         impl AtomicConsume for loom::sync::atomic::$atomic {
73             type Val = $val;
74             impl_consume!();
75         }
76     };
77 }
78 
79 impl_atomic!(AtomicBool, bool);
80 impl_atomic!(AtomicUsize, usize);
81 impl_atomic!(AtomicIsize, isize);
82 impl_atomic!(AtomicU8, u8);
83 impl_atomic!(AtomicI8, i8);
84 impl_atomic!(AtomicU16, u16);
85 impl_atomic!(AtomicI16, i16);
86 #[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))]
87 impl_atomic!(AtomicU32, u32);
88 #[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))]
89 impl_atomic!(AtomicI32, i32);
90 #[cfg(any(
91     target_has_atomic = "64",
92     not(any(target_pointer_width = "16", target_pointer_width = "32")),
93 ))]
94 impl_atomic!(AtomicU64, u64);
95 #[cfg(any(
96     target_has_atomic = "64",
97     not(any(target_pointer_width = "16", target_pointer_width = "32")),
98 ))]
99 impl_atomic!(AtomicI64, i64);
100 
101 #[cfg(not(crossbeam_no_atomic))]
102 impl<T> AtomicConsume for core::sync::atomic::AtomicPtr<T> {
103     type Val = *mut T;
104     impl_consume!();
105 }
106 
107 #[cfg(crossbeam_loom)]
108 impl<T> AtomicConsume for loom::sync::atomic::AtomicPtr<T> {
109     type Val = *mut T;
110     impl_consume!();
111 }
112