1 // Copyright 2016 Amanieu d'Antras
2 //
3 // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 // http://opensource.org/licenses/MIT>, at your option. This file may not be
6 // copied, modified, or distributed except according to those terms.
7
8 use std::sync::atomic::AtomicUsize;
9
10 // Extension trait to add lock elision primitives to atomic types
11 pub trait AtomicElisionExt {
12 type IntType;
13
14 // Perform a compare_exchange and start a transaction
elision_compare_exchange_acquire( &self, current: Self::IntType, new: Self::IntType, ) -> Result<Self::IntType, Self::IntType>15 fn elision_compare_exchange_acquire(
16 &self,
17 current: Self::IntType,
18 new: Self::IntType,
19 ) -> Result<Self::IntType, Self::IntType>;
20
21 // Perform a fetch_sub and end a transaction
elision_fetch_sub_release(&self, val: Self::IntType) -> Self::IntType22 fn elision_fetch_sub_release(&self, val: Self::IntType) -> Self::IntType;
23 }
24
25 // Indicates whether the target architecture supports lock elision
26 #[inline]
have_elision() -> bool27 pub fn have_elision() -> bool {
28 cfg!(all(
29 feature = "nightly",
30 any(target_arch = "x86", target_arch = "x86_64"),
31 ))
32 }
33
34 // This implementation is never actually called because it is guarded by
35 // have_elision().
36 #[cfg(not(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64"))))]
37 impl AtomicElisionExt for AtomicUsize {
38 type IntType = usize;
39
40 #[inline]
elision_compare_exchange_acquire(&self, _: usize, _: usize) -> Result<usize, usize>41 fn elision_compare_exchange_acquire(&self, _: usize, _: usize) -> Result<usize, usize> {
42 unreachable!();
43 }
44
45 #[inline]
elision_fetch_sub_release(&self, _: usize) -> usize46 fn elision_fetch_sub_release(&self, _: usize) -> usize {
47 unreachable!();
48 }
49 }
50
51 #[cfg(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64")))]
52 impl AtomicElisionExt for AtomicUsize {
53 type IntType = usize;
54
55 #[cfg(target_pointer_width = "32")]
56 #[inline]
elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize>57 fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
58 unsafe {
59 let prev: usize;
60 llvm_asm!("xacquire; lock; cmpxchgl $2, $1"
61 : "={eax}" (prev), "+*m" (self)
62 : "r" (new), "{eax}" (current)
63 : "memory"
64 : "volatile");
65 if prev == current {
66 Ok(prev)
67 } else {
68 Err(prev)
69 }
70 }
71 }
72 #[cfg(target_pointer_width = "64")]
73 #[inline]
elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize>74 fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
75 unsafe {
76 let prev: usize;
77 llvm_asm!("xacquire; lock; cmpxchgq $2, $1"
78 : "={rax}" (prev), "+*m" (self)
79 : "r" (new), "{rax}" (current)
80 : "memory"
81 : "volatile");
82 if prev == current {
83 Ok(prev)
84 } else {
85 Err(prev)
86 }
87 }
88 }
89
90 #[cfg(target_pointer_width = "32")]
91 #[inline]
elision_fetch_sub_release(&self, val: usize) -> usize92 fn elision_fetch_sub_release(&self, val: usize) -> usize {
93 unsafe {
94 let prev: usize;
95 llvm_asm!("xrelease; lock; xaddl $2, $1"
96 : "=r" (prev), "+*m" (self)
97 : "0" (val.wrapping_neg())
98 : "memory"
99 : "volatile");
100 prev
101 }
102 }
103 #[cfg(target_pointer_width = "64")]
104 #[inline]
elision_fetch_sub_release(&self, val: usize) -> usize105 fn elision_fetch_sub_release(&self, val: usize) -> usize {
106 unsafe {
107 let prev: usize;
108 llvm_asm!("xrelease; lock; xaddq $2, $1"
109 : "=r" (prev), "+*m" (self)
110 : "0" (val.wrapping_neg())
111 : "memory"
112 : "volatile");
113 prev
114 }
115 }
116 }
117