1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2010-2016. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /*
22 * Description: Native atomics ethread support when using VC++
23 * Author: Rickard Green
24 */
25
26 #undef ETHR_INCLUDE_ATOMIC_IMPL__
27 #if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
28 # define ETHR_WIN_ATOMIC32_H__
29 # if (defined(ETHR_MEMBAR) \
30 && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \
31 && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE))
32 # define ETHR_INCLUDE_ATOMIC_IMPL__ 4
33 # endif
34 # undef ETHR_ATOMIC_WANT_32BIT_IMPL__
35 #elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
36 # define ETHR_WIN_ATOMIC64_H__
37 # if (defined(ETHR_MEMBAR) \
38 && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \
39 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \
40 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL)))
41 # define ETHR_INCLUDE_ATOMIC_IMPL__ 8
42 # endif
43 # undef ETHR_ATOMIC_WANT_64BIT_IMPL__
44 #endif
45
46 #if !defined(_MSC_VER) || _MSC_VER < 1400
47 # undef ETHR_INCLUDE_ATOMIC_IMPL__
48 #endif
49
50 #ifdef ETHR_INCLUDE_ATOMIC_IMPL__
51
52 # ifndef ETHR_WIN_ATOMIC_COMMON__
53 # define ETHR_WIN_ATOMIC_COMMON__
54
55 # if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
56 # define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
57 # else
58 # define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
59 # endif
60
61 # endif /* ETHR_WIN_ATOMIC_COMMON__ */
62
63 # if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
64
65 # define ETHR_HAVE_NATIVE_ATOMIC32 1
66 # define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked"
67
68 # ifdef ETHR_HAVE__INTERLOCKEDDECREMENT
69 # define ETHR_WIN_HAVE_DEC
70 # pragma intrinsic(_InterlockedDecrement)
71 # endif
72 # ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL
73 # define ETHR_WIN_HAVE_DEC_REL
74 # pragma intrinsic(_InterlockedDecrement_rel)
75 # endif
76 # ifdef ETHR_HAVE__INTERLOCKEDINCREMENT
77 # define ETHR_WIN_HAVE_INC
78 # pragma intrinsic(_InterlockedIncrement)
79 # endif
80 # ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ
81 # define ETHR_WIN_HAVE_INC_ACQ
82 # pragma intrinsic(_InterlockedIncrement_acq)
83 # endif
84 # ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD
85 # define ETHR_WIN_HAVE_XCHG_ADD
86 # pragma intrinsic(_InterlockedExchangeAdd)
87 # endif
88 # ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ
89 # define ETHR_WIN_HAVE_XCHG_ADD_ACQ
90 # pragma intrinsic(_InterlockedExchangeAdd_acq)
91 # endif
92 # ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE
93 # define ETHR_WIN_HAVE_XCHG
94 # pragma intrinsic(_InterlockedExchange)
95 # endif
96 # ifdef ETHR_HAVE__INTERLOCKEDAND
97 # define ETHR_WIN_HAVE_AND
98 # pragma intrinsic(_InterlockedAnd)
99 # endif
100 # ifdef ETHR_HAVE__INTERLOCKEDOR
101 # define ETHR_WIN_HAVE_OR
102 # pragma intrinsic(_InterlockedOr)
103 # endif
104 # ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE
105 # define ETHR_WIN_HAVE_CMPXCHG
106 # pragma intrinsic(_InterlockedCompareExchange)
107 # endif
108 # ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ
109 # define ETHR_WIN_HAVE_CMPXCHG_ACQ
110 # pragma intrinsic(_InterlockedCompareExchange_acq)
111 # endif
112 # ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL
113 # define ETHR_WIN_HAVE_CMPXCHG_REL
114 # pragma intrinsic(_InterlockedCompareExchange_rel)
115 # endif
116
117 # define ETHR_ILCKD__(X) _Interlocked ## X
118 # define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
119 # define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
120
121 # define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
122 # define ETHR_ATMC_T__ ethr_native_atomic32_t
123 # define ETHR_AINT_T__ ethr_sint32_t
124
125 # elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
126
127 # define ETHR_HAVE_NATIVE_ATOMIC64 1
128 # define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked"
129
130 # ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
131 # define ETHR_WIN_HAVE_DEC
132 # pragma intrinsic(_InterlockedDecrement64)
133 # endif
134 # ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL
135 # define ETHR_WIN_HAVE_DEC_REL
136 # pragma intrinsic(_InterlockedDecrement64_rel)
137 # endif
138 # ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ
139 # define ETHR_WIN_HAVE_INC_ACQ
140 # pragma intrinsic(_InterlockedIncrement64_acq)
141 # endif
142 # ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
143 # define ETHR_WIN_HAVE_INC
144 # pragma intrinsic(_InterlockedIncrement64)
145 # endif
146 # ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
147 # define ETHR_WIN_HAVE_XCHG_ADD
148 # pragma intrinsic(_InterlockedExchangeAdd64)
149 # endif
150 # ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ
151 # define ETHR_WIN_HAVE_XCHG_ADD_ACQ
152 # pragma intrinsic(_InterlockedExchangeAdd64_acq)
153 # endif
154 # ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
155 # define ETHR_WIN_HAVE_XCHG
156 # pragma intrinsic(_InterlockedExchange64)
157 # endif
158 # ifdef ETHR_HAVE__INTERLOCKEDAND64
159 # define ETHR_WIN_HAVE_AND
160 # pragma intrinsic(_InterlockedAnd64)
161 # endif
162 # ifdef ETHR_HAVE__INTERLOCKEDOR64
163 # define ETHR_WIN_HAVE_OR
164 # pragma intrinsic(_InterlockedOr64)
165 # endif
166 # ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
167 # define ETHR_WIN_HAVE_CMPXCHG
168 # pragma intrinsic(_InterlockedCompareExchange64)
169 # endif
170 # ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ
171 # define ETHR_WIN_HAVE_CMPXCHG_ACQ
172 # pragma intrinsic(_InterlockedCompareExchange64_acq)
173 # endif
174 # ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL
175 # define ETHR_WIN_HAVE_CMPXCHG_REL
176 # pragma intrinsic(_InterlockedCompareExchange64_rel)
177 # endif
178
179 # define ETHR_ILCKD__(X) _Interlocked ## X ## 64
180 # define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
181 # define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
182
183 # define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
184 # define ETHR_ATMC_T__ ethr_native_atomic64_t
185 # define ETHR_AINT_T__ ethr_sint64_t
186
187 # else
188 # error "Unsupported integer size"
189 # endif
190
191 typedef struct {
192 volatile ETHR_AINT_T__ value;
193 } ETHR_ATMC_T__;
194
195 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
196
197 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
198 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
199 #else
200 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
201 #endif
202
203 static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)204 ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
205 {
206 return (ETHR_AINT_T__ *) &var->value;
207 }
208
209 #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
210
211 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
212 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
213 #else
214 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
215 #endif
216
217 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set)218 ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
219 {
220 var->value = i;
221 }
222
223 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
224 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
225 #else
226 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
227 #endif
228
229 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_relb)230 ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
231 {
232 #if defined(_M_IX86)
233 if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
234 (void) ETHR_ILCKD__(Exchange)(&var->value, i);
235 else
236 #endif /* _M_IX86 */
237 {
238 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
239 var->value = i;
240 }
241 }
242
243 #endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
244
245 #if defined(ETHR_WIN_HAVE_XCHG)
246
247 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
248 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
249 #else
250 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
251 #endif
252
253 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_mb)254 ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
255 {
256 (void) ETHR_ILCKD__(Exchange)(&var->value, i);
257 }
258
259 #endif
260
261 #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
262
263 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
264 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
265 #else
266 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
267 #endif
268
269 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read)270 ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
271 {
272 return var->value;
273 }
274
275 #endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
276
277 #if defined(ETHR_WIN_HAVE_XCHG_ADD)
278
279 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
280 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
281 #else
282 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
283 #endif
284
285 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_mb)286 ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
287 {
288 return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
289 }
290
291 #endif
292
293 #if defined(ETHR_WIN_HAVE_INC)
294
295 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
296 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1
297 #else
298 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1
299 #endif
300
301 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return_mb)302 ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var)
303 {
304 return ETHR_ILCKD__(Increment)(&var->value);
305 }
306
307 #endif
308
309 #if defined(ETHR_WIN_HAVE_INC_ACQ)
310
311 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
312 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
313 #else
314 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
315 #endif
316
317 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return_acqb)318 ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
319 {
320 return ETHR_ILCKD_ACQ__(Increment)(&var->value);
321 }
322
323 #endif
324
325 #if defined(ETHR_WIN_HAVE_DEC)
326
327 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
328 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1
329 #else
330 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1
331 #endif
332
333 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return_mb)334 ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var)
335 {
336 return ETHR_ILCKD__(Decrement)(&var->value);
337 }
338
339 #endif
340
341 #if defined(ETHR_WIN_HAVE_DEC_REL)
342
343 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
344 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
345 #else
346 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
347 #endif
348
349 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return_relb)350 ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
351 {
352 return ETHR_ILCKD_REL__(Decrement)(&var->value);
353 }
354
355 #endif
356
357 #if defined(ETHR_WIN_HAVE_AND)
358
359 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
360 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
361 #else
362 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
363 #endif
364
365 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(and_retold_mb)366 ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
367 {
368 return ETHR_ILCKD__(And)(&var->value, mask);
369 }
370
371 #endif
372
373 #if defined(ETHR_WIN_HAVE_OR)
374
375 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
376 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
377 #else
378 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
379 #endif
380
381 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(or_retold_mb)382 ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
383 {
384 return ETHR_ILCKD__(Or)(&var->value, mask);
385 }
386
387 #endif
388
389 #if defined(ETHR_WIN_HAVE_XCHG)
390
391 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
392 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1
393 #else
394 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1
395 #endif
396
397 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(xchg_mb)398 ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
399 {
400 return ETHR_ILCKD__(Exchange)(&var->value, new);
401 }
402
403 #endif
404
405 #if defined(ETHR_WIN_HAVE_CMPXCHG)
406
407 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
408 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
409 #else
410 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
411 #endif
412
413 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_mb)414 ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
415 ETHR_AINT_T__ new,
416 ETHR_AINT_T__ old)
417 {
418 return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
419 }
420
421 #endif
422
423 #if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ)
424
425 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
426 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
427 #else
428 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
429 #endif
430
431 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_acqb)432 ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
433 ETHR_AINT_T__ new,
434 ETHR_AINT_T__ old)
435 {
436 return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old);
437 }
438
439 #endif
440
441 #if defined(ETHR_WIN_HAVE_CMPXCHG_REL)
442
443 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
444 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
445 #else
446 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
447 #endif
448
449 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_relb)450 ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
451 ETHR_AINT_T__ new,
452 ETHR_AINT_T__ old)
453 {
454 return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old);
455 }
456
457 #endif
458
459 #endif /* ETHR_TRY_INLINE_FUNCS */
460
461 #undef ETHR_ILCKD__
462 #undef ETHR_ILCKD_ACQ__
463 #undef ETHR_ILCKD_REL__
464 #undef ETHR_NATMC_FUNC__
465 #undef ETHR_ATMC_T__
466 #undef ETHR_AINT_T__
467 #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
468 #undef ETHR_WIN_HAVE_CMPXCHG
469 #undef ETHR_WIN_HAVE_DEC
470 #undef ETHR_WIN_HAVE_INC
471 #undef ETHR_WIN_HAVE_XCHG_ADD
472 #undef ETHR_WIN_HAVE_XCHG
473 #undef ETHR_WIN_HAVE_AND
474 #undef ETHR_WIN_HAVE_OR
475 #undef ETHR_WIN_HAVE_XCHG_ADD_ACQ
476 #undef ETHR_WIN_HAVE_INC_ACQ
477 #undef ETHR_WIN_HAVE_DEC_REL
478 #undef ETHR_WIN_HAVE_CMPXCHG_ACQ
479 #undef ETHR_WIN_HAVE_CMPXCHG_REL
480
481 #endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
482