1 // Copyright 2016 Amanieu d'Antras
2 //
3 // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 // http://opensource.org/licenses/MIT>, at your option. This file may not be
6 // copied, modified, or distributed except according to those terms.
7
8 use std::sync::atomic::AtomicUsize;
9
10 // Extension trait to add lock elision primitives to atomic types
11 pub trait AtomicElisionExt {
12 type IntType;
13
14 // Perform a compare_exchange and start a transaction
elision_acquire( &self, current: Self::IntType, new: Self::IntType, ) -> Result<Self::IntType, Self::IntType>15 fn elision_acquire(
16 &self,
17 current: Self::IntType,
18 new: Self::IntType,
19 ) -> Result<Self::IntType, Self::IntType>;
20 // Perform a compare_exchange and end a transaction
elision_release( &self, current: Self::IntType, new: Self::IntType, ) -> Result<Self::IntType, Self::IntType>21 fn elision_release(
22 &self,
23 current: Self::IntType,
24 new: Self::IntType,
25 ) -> Result<Self::IntType, Self::IntType>;
26 }
27
28 // Indicates whether the target architecture supports lock elision
29 #[inline]
have_elision() -> bool30 pub fn have_elision() -> bool {
31 cfg!(all(
32 feature = "nightly",
33 any(target_arch = "x86", target_arch = "x86_64"),
34 ))
35 }
36
37 // This implementation is never actually called because it is guarded by
38 // have_elision().
39 #[cfg(not(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64"))))]
40 impl AtomicElisionExt for AtomicUsize {
41 type IntType = usize;
42
43 #[inline]
elision_acquire(&self, _: usize, _: usize) -> Result<usize, usize>44 fn elision_acquire(&self, _: usize, _: usize) -> Result<usize, usize> {
45 unreachable!();
46 }
47
48 #[inline]
elision_release(&self, _: usize, _: usize) -> Result<usize, usize>49 fn elision_release(&self, _: usize, _: usize) -> Result<usize, usize> {
50 unreachable!();
51 }
52 }
53
54 #[cfg(all(feature = "nightly", target_arch = "x86"))]
55 impl AtomicElisionExt for AtomicUsize {
56 type IntType = usize;
57
58 #[inline]
elision_acquire(&self, current: usize, new: usize) -> Result<usize, usize>59 fn elision_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
60 unsafe {
61 let prev: usize;
62 asm!("xacquire; lock; cmpxchgl $2, $1"
63 : "={eax}" (prev), "+*m" (self)
64 : "r" (new), "{eax}" (current)
65 : "memory"
66 : "volatile");
67 if prev == current {
68 Ok(prev)
69 } else {
70 Err(prev)
71 }
72 }
73 }
74
75 #[inline]
elision_release(&self, current: usize, new: usize) -> Result<usize, usize>76 fn elision_release(&self, current: usize, new: usize) -> Result<usize, usize> {
77 unsafe {
78 let prev: usize;
79 asm!("xrelease; lock; cmpxchgl $2, $1"
80 : "={eax}" (prev), "+*m" (self)
81 : "r" (new), "{eax}" (current)
82 : "memory"
83 : "volatile");
84 if prev == current {
85 Ok(prev)
86 } else {
87 Err(prev)
88 }
89 }
90 }
91 }
92
93 #[cfg(all(
94 feature = "nightly",
95 target_arch = "x86_64",
96 target_pointer_width = "32"
97 ))]
98 impl AtomicElisionExt for AtomicUsize {
99 type IntType = usize;
100
101 #[inline]
elision_acquire(&self, current: usize, new: usize) -> Result<usize, usize>102 fn elision_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
103 unsafe {
104 let prev: usize;
105 asm!("xacquire; lock; cmpxchgl $2, $1"
106 : "={rax}" (prev), "+*m" (self)
107 : "r" (new), "{rax}" (current)
108 : "memory"
109 : "volatile");
110 if prev == current {
111 Ok(prev)
112 } else {
113 Err(prev)
114 }
115 }
116 }
117
118 #[inline]
elision_release(&self, current: usize, new: usize) -> Result<usize, usize>119 fn elision_release(&self, current: usize, new: usize) -> Result<usize, usize> {
120 unsafe {
121 let prev: usize;
122 asm!("xrelease; lock; cmpxchgl $2, $1"
123 : "={rax}" (prev), "+*m" (self)
124 : "r" (new), "{rax}" (current)
125 : "memory"
126 : "volatile");
127 if prev == current {
128 Ok(prev)
129 } else {
130 Err(prev)
131 }
132 }
133 }
134 }
135
136 #[cfg(all(
137 feature = "nightly",
138 target_arch = "x86_64",
139 target_pointer_width = "64"
140 ))]
141 impl AtomicElisionExt for AtomicUsize {
142 type IntType = usize;
143
144 #[inline]
elision_acquire(&self, current: usize, new: usize) -> Result<usize, usize>145 fn elision_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
146 unsafe {
147 let prev: usize;
148 asm!("xacquire; lock; cmpxchgq $2, $1"
149 : "={rax}" (prev), "+*m" (self)
150 : "r" (new), "{rax}" (current)
151 : "memory"
152 : "volatile");
153 if prev == current {
154 Ok(prev)
155 } else {
156 Err(prev)
157 }
158 }
159 }
160
161 #[inline]
elision_release(&self, current: usize, new: usize) -> Result<usize, usize>162 fn elision_release(&self, current: usize, new: usize) -> Result<usize, usize> {
163 unsafe {
164 let prev: usize;
165 asm!("xrelease; lock; cmpxchgq $2, $1"
166 : "={rax}" (prev), "+*m" (self)
167 : "r" (new), "{rax}" (current)
168 : "memory"
169 : "volatile");
170 if prev == current {
171 Ok(prev)
172 } else {
173 Err(prev)
174 }
175 }
176 }
177 }
178