1 /*
2 * Copyright (c) 2003 by Hewlett-Packard Company. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 * SOFTWARE.
21 */
22
23 /*
24 * This file specifies Itanimum primitives for use with the Intel (ecc)
25 * compiler. We use intrinsics instead of the inline assembly code in the
26 * gcc file.
27 */
28
29 #include "../all_atomic_load_store.h"
30
31 #include "../test_and_set_t_is_char.h"
32
33 #include <ia64intrin.h>
34
35 /* The acquire release semantics of volatile can be turned off. And volatile */
36 /* operations in icc9 don't imply ordering with respect to other nonvolatile */
37 /* operations. */
38
39 #define AO_INTEL_PTR_t void *
40
41 AO_INLINE AO_t
AO_load_acquire(const volatile AO_t * p)42 AO_load_acquire(const volatile AO_t *p)
43 {
44 return (AO_t)(__ld8_acq((AO_INTEL_PTR_t)p));
45 }
46 #define AO_HAVE_load_acquire
47
48 AO_INLINE void
AO_store_release(volatile AO_t * p,AO_t val)49 AO_store_release(volatile AO_t *p, AO_t val)
50 {
51 __st8_rel((AO_INTEL_PTR_t)p, (__int64)val);
52 }
53 #define AO_HAVE_store_release
54
55 AO_INLINE unsigned char
AO_char_load_acquire(const volatile unsigned char * p)56 AO_char_load_acquire(const volatile unsigned char *p)
57 {
58 /* A normal volatile load generates an ld.acq */
59 return (__ld1_acq((AO_INTEL_PTR_t)p));
60 }
61 #define AO_HAVE_char_load_acquire
62
63 AO_INLINE void
AO_char_store_release(volatile unsigned char * p,unsigned char val)64 AO_char_store_release(volatile unsigned char *p, unsigned char val)
65 {
66 __st1_rel((AO_INTEL_PTR_t)p, val);
67 }
68 #define AO_HAVE_char_store_release
69
70 AO_INLINE unsigned short
AO_short_load_acquire(const volatile unsigned short * p)71 AO_short_load_acquire(const volatile unsigned short *p)
72 {
73 /* A normal volatile load generates an ld.acq */
74 return (__ld2_acq((AO_INTEL_PTR_t)p));
75 }
76 #define AO_HAVE_short_load_acquire
77
78 AO_INLINE void
AO_short_store_release(volatile unsigned short * p,unsigned short val)79 AO_short_store_release(volatile unsigned short *p, unsigned short val)
80 {
81 __st2_rel((AO_INTEL_PTR_t)p, val);
82 }
83 #define AO_HAVE_short_store_release
84
85 AO_INLINE unsigned int
AO_int_load_acquire(const volatile unsigned int * p)86 AO_int_load_acquire(const volatile unsigned int *p)
87 {
88 /* A normal volatile load generates an ld.acq */
89 return (__ld4_acq((AO_INTEL_PTR_t)p));
90 }
91 #define AO_HAVE_int_load_acquire
92
93 AO_INLINE void
AO_int_store_release(volatile unsigned int * p,unsigned int val)94 AO_int_store_release(volatile unsigned int *p, unsigned int val)
95 {
96 __st4_rel((AO_INTEL_PTR_t)p, val);
97 }
98 #define AO_HAVE_int_store_release
99
100 AO_INLINE void
AO_nop_full(void)101 AO_nop_full(void)
102 {
103 __mf();
104 }
105 #define AO_HAVE_nop_full
106
107 #ifndef AO_PREFER_GENERALIZED
108 AO_INLINE AO_t
AO_fetch_and_add1_acquire(volatile AO_t * p)109 AO_fetch_and_add1_acquire(volatile AO_t *p)
110 {
111 return __fetchadd8_acq((unsigned __int64 *)p, 1);
112 }
113 #define AO_HAVE_fetch_and_add1_acquire
114
115 AO_INLINE AO_t
AO_fetch_and_add1_release(volatile AO_t * p)116 AO_fetch_and_add1_release(volatile AO_t *p)
117 {
118 return __fetchadd8_rel((unsigned __int64 *)p, 1);
119 }
120 #define AO_HAVE_fetch_and_add1_release
121
122 AO_INLINE AO_t
AO_fetch_and_sub1_acquire(volatile AO_t * p)123 AO_fetch_and_sub1_acquire(volatile AO_t *p)
124 {
125 return __fetchadd8_acq((unsigned __int64 *)p, -1);
126 }
127 #define AO_HAVE_fetch_and_sub1_acquire
128
129 AO_INLINE AO_t
AO_fetch_and_sub1_release(volatile AO_t * p)130 AO_fetch_and_sub1_release(volatile AO_t *p)
131 {
132 return __fetchadd8_rel((unsigned __int64 *)p, -1);
133 }
134 #define AO_HAVE_fetch_and_sub1_release
135 #endif /* !AO_PREFER_GENERALIZED */
136
137 AO_INLINE AO_t
AO_fetch_compare_and_swap_acquire(volatile AO_t * addr,AO_t old_val,AO_t new_val)138 AO_fetch_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
139 AO_t new_val)
140 {
141 return _InterlockedCompareExchange64_acq(addr, new_val, old_val);
142 }
143 #define AO_HAVE_fetch_compare_and_swap_acquire
144
145 AO_INLINE AO_t
AO_fetch_compare_and_swap_release(volatile AO_t * addr,AO_t old_val,AO_t new_val)146 AO_fetch_compare_and_swap_release(volatile AO_t *addr, AO_t old_val,
147 AO_t new_val)
148 {
149 return _InterlockedCompareExchange64_rel(addr, new_val, old_val);
150 }
151 #define AO_HAVE_fetch_compare_and_swap_release
152
153 AO_INLINE unsigned char
AO_char_fetch_compare_and_swap_acquire(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)154 AO_char_fetch_compare_and_swap_acquire(volatile unsigned char *addr,
155 unsigned char old_val,
156 unsigned char new_val)
157 {
158 return _InterlockedCompareExchange8_acq(addr, new_val, old_val);
159 }
160 #define AO_HAVE_char_fetch_compare_and_swap_acquire
161
162 AO_INLINE unsigned char
AO_char_fetch_compare_and_swap_release(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)163 AO_char_fetch_compare_and_swap_release(volatile unsigned char *addr,
164 unsigned char old_val,
165 unsigned char new_val)
166 {
167 return _InterlockedCompareExchange8_rel(addr, new_val, old_val);
168 }
169 #define AO_HAVE_char_fetch_compare_and_swap_release
170
171 AO_INLINE unsigned short
AO_short_fetch_compare_and_swap_acquire(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)172 AO_short_fetch_compare_and_swap_acquire(volatile unsigned short *addr,
173 unsigned short old_val,
174 unsigned short new_val)
175 {
176 return _InterlockedCompareExchange16_acq(addr, new_val, old_val);
177 }
178 #define AO_HAVE_short_fetch_compare_and_swap_acquire
179
180 AO_INLINE unsigned short
AO_short_fetch_compare_and_swap_release(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)181 AO_short_fetch_compare_and_swap_release(volatile unsigned short *addr,
182 unsigned short old_val,
183 unsigned short new_val)
184 {
185 return _InterlockedCompareExchange16_rel(addr, new_val, old_val);
186 }
187 #define AO_HAVE_short_fetch_compare_and_swap_release
188
189 AO_INLINE unsigned int
AO_int_fetch_compare_and_swap_acquire(volatile unsigned int * addr,unsigned int old_val,unsigned int new_val)190 AO_int_fetch_compare_and_swap_acquire(volatile unsigned int *addr,
191 unsigned int old_val,
192 unsigned int new_val)
193 {
194 return _InterlockedCompareExchange_acq(addr, new_val, old_val);
195 }
196 #define AO_HAVE_int_fetch_compare_and_swap_acquire
197
198 AO_INLINE unsigned int
AO_int_fetch_compare_and_swap_release(volatile unsigned int * addr,unsigned int old_val,unsigned int new_val)199 AO_int_fetch_compare_and_swap_release(volatile unsigned int *addr,
200 unsigned int old_val,
201 unsigned int new_val)
202 {
203 return _InterlockedCompareExchange_rel(addr, new_val, old_val);
204 }
205 #define AO_HAVE_int_fetch_compare_and_swap_release
206