1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10 #ifndef __IMMINTRIN_H
11 #define __IMMINTRIN_H
12
13 #include <x86gprintrin.h>
14
15 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
16 defined(__MMX__)
17 #include <mmintrin.h>
18 #endif
19
20 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
21 defined(__SSE__)
22 #include <xmmintrin.h>
23 #endif
24
25 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
26 defined(__SSE2__)
27 #include <emmintrin.h>
28 #endif
29
30 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
31 defined(__SSE3__)
32 #include <pmmintrin.h>
33 #endif
34
35 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
36 defined(__SSSE3__)
37 #include <tmmintrin.h>
38 #endif
39
40 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
41 (defined(__SSE4_2__) || defined(__SSE4_1__))
42 #include <smmintrin.h>
43 #endif
44
45 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
46 (defined(__AES__) || defined(__PCLMUL__))
47 #include <wmmintrin.h>
48 #endif
49
50 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
51 defined(__CLFLUSHOPT__)
52 #include <clflushoptintrin.h>
53 #endif
54
55 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
56 defined(__CLWB__)
57 #include <clwbintrin.h>
58 #endif
59
60 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
61 defined(__AVX__)
62 #include <avxintrin.h>
63 #endif
64
65 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
66 defined(__AVX2__)
67 #include <avx2intrin.h>
68 #endif
69
70 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
71 defined(__F16C__)
72 #include <f16cintrin.h>
73 #endif
74
75 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
76 defined(__VPCLMULQDQ__)
77 #include <vpclmulqdqintrin.h>
78 #endif
79
80 /* No feature check desired due to internal checks */
81 #include <bmiintrin.h>
82
83 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
84 defined(__BMI2__)
85 #include <bmi2intrin.h>
86 #endif
87
88 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
89 defined(__LZCNT__)
90 #include <lzcntintrin.h>
91 #endif
92
93 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
94 defined(__POPCNT__)
95 #include <popcntintrin.h>
96 #endif
97
98 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
99 defined(__FMA__)
100 #include <fmaintrin.h>
101 #endif
102
103 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
104 defined(__AVX512F__)
105 #include <avx512fintrin.h>
106 #endif
107
108 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
109 defined(__AVX512VL__)
110 #include <avx512vlintrin.h>
111 #endif
112
113 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
114 defined(__AVX512BW__)
115 #include <avx512bwintrin.h>
116 #endif
117
118 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
119 defined(__AVX512BITALG__)
120 #include <avx512bitalgintrin.h>
121 #endif
122
123 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
124 defined(__AVX512CD__)
125 #include <avx512cdintrin.h>
126 #endif
127
128 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
129 defined(__AVX512VPOPCNTDQ__)
130 #include <avx512vpopcntdqintrin.h>
131 #endif
132
133 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
134 (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
135 #include <avx512vpopcntdqvlintrin.h>
136 #endif
137
138 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
139 defined(__AVX512VNNI__)
140 #include <avx512vnniintrin.h>
141 #endif
142
143 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
144 (defined(__AVX512VL__) && defined(__AVX512VNNI__))
145 #include <avx512vlvnniintrin.h>
146 #endif
147
148 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
149 defined(__AVXVNNI__)
150 #include <avxvnniintrin.h>
151 #endif
152
153 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
154 defined(__AVX512DQ__)
155 #include <avx512dqintrin.h>
156 #endif
157
158 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
159 (defined(__AVX512VL__) && defined(__AVX512BITALG__))
160 #include <avx512vlbitalgintrin.h>
161 #endif
162
163 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
164 (defined(__AVX512VL__) && defined(__AVX512BW__))
165 #include <avx512vlbwintrin.h>
166 #endif
167
168 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
169 (defined(__AVX512VL__) && defined(__AVX512CD__))
170 #include <avx512vlcdintrin.h>
171 #endif
172
173 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
174 (defined(__AVX512VL__) && defined(__AVX512DQ__))
175 #include <avx512vldqintrin.h>
176 #endif
177
178 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
179 defined(__AVX512ER__)
180 #include <avx512erintrin.h>
181 #endif
182
183 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
184 defined(__AVX512IFMA__)
185 #include <avx512ifmaintrin.h>
186 #endif
187
188 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
189 (defined(__AVX512IFMA__) && defined(__AVX512VL__))
190 #include <avx512ifmavlintrin.h>
191 #endif
192
193 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
194 defined(__AVX512VBMI__)
195 #include <avx512vbmiintrin.h>
196 #endif
197
198 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
199 (defined(__AVX512VBMI__) && defined(__AVX512VL__))
200 #include <avx512vbmivlintrin.h>
201 #endif
202
203 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
204 defined(__AVX512VBMI2__)
205 #include <avx512vbmi2intrin.h>
206 #endif
207
208 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
209 (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
210 #include <avx512vlvbmi2intrin.h>
211 #endif
212
213 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
214 defined(__AVX512PF__)
215 #include <avx512pfintrin.h>
216 #endif
217
218 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
219 defined(__AVX512BF16__)
220 #include <avx512bf16intrin.h>
221 #endif
222
223 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
224 (defined(__AVX512VL__) && defined(__AVX512BF16__))
225 #include <avx512vlbf16intrin.h>
226 #endif
227
228 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
229 defined(__PKU__)
230 #include <pkuintrin.h>
231 #endif
232
233 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
234 defined(__VAES__)
235 #include <vaesintrin.h>
236 #endif
237
238 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
239 defined(__GFNI__)
240 #include <gfniintrin.h>
241 #endif
242
243 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
244 defined(__RDPID__)
245 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
246 ///
247 /// \headerfile <immintrin.h>
248 ///
249 /// This intrinsic corresponds to the <c> RDPID </c> instruction.
250 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))
_rdpid_u32(void)251 _rdpid_u32(void) {
252 return __builtin_ia32_rdpid();
253 }
254 #endif // __RDPID__
255
256 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
257 defined(__RDRND__)
258 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand16_step(unsigned short * __p)259 _rdrand16_step(unsigned short *__p)
260 {
261 return __builtin_ia32_rdrand16_step(__p);
262 }
263
264 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand32_step(unsigned int * __p)265 _rdrand32_step(unsigned int *__p)
266 {
267 return __builtin_ia32_rdrand32_step(__p);
268 }
269
270 #ifdef __x86_64__
271 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand64_step(unsigned long long * __p)272 _rdrand64_step(unsigned long long *__p)
273 {
274 return __builtin_ia32_rdrand64_step(__p);
275 }
276 #endif
277 #endif /* __RDRND__ */
278
279 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
280 defined(__FSGSBASE__)
281 #ifdef __x86_64__
282 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readfsbase_u32(void)283 _readfsbase_u32(void)
284 {
285 return __builtin_ia32_rdfsbase32();
286 }
287
288 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readfsbase_u64(void)289 _readfsbase_u64(void)
290 {
291 return __builtin_ia32_rdfsbase64();
292 }
293
294 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readgsbase_u32(void)295 _readgsbase_u32(void)
296 {
297 return __builtin_ia32_rdgsbase32();
298 }
299
300 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readgsbase_u64(void)301 _readgsbase_u64(void)
302 {
303 return __builtin_ia32_rdgsbase64();
304 }
305
306 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writefsbase_u32(unsigned int __V)307 _writefsbase_u32(unsigned int __V)
308 {
309 __builtin_ia32_wrfsbase32(__V);
310 }
311
312 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writefsbase_u64(unsigned long long __V)313 _writefsbase_u64(unsigned long long __V)
314 {
315 __builtin_ia32_wrfsbase64(__V);
316 }
317
318 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writegsbase_u32(unsigned int __V)319 _writegsbase_u32(unsigned int __V)
320 {
321 __builtin_ia32_wrgsbase32(__V);
322 }
323
324 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writegsbase_u64(unsigned long long __V)325 _writegsbase_u64(unsigned long long __V)
326 {
327 __builtin_ia32_wrgsbase64(__V);
328 }
329
330 #endif
331 #endif /* __FSGSBASE__ */
332
333 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
334 defined(__MOVBE__)
335
336 /* The structs used below are to force the load/store to be unaligned. This
337 * is accomplished with the __packed__ attribute. The __may_alias__ prevents
338 * tbaa metadata from being generated based on the struct and the type of the
339 * field inside of it.
340 */
341
342 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_loadbe_i16(void const * __P)343 _loadbe_i16(void const * __P) {
344 struct __loadu_i16 {
345 short __v;
346 } __attribute__((__packed__, __may_alias__));
347 return __builtin_bswap16(((const struct __loadu_i16*)__P)->__v);
348 }
349
350 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_storebe_i16(void * __P,short __D)351 _storebe_i16(void * __P, short __D) {
352 struct __storeu_i16 {
353 short __v;
354 } __attribute__((__packed__, __may_alias__));
355 ((struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D);
356 }
357
358 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_loadbe_i32(void const * __P)359 _loadbe_i32(void const * __P) {
360 struct __loadu_i32 {
361 int __v;
362 } __attribute__((__packed__, __may_alias__));
363 return __builtin_bswap32(((const struct __loadu_i32*)__P)->__v);
364 }
365
366 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_storebe_i32(void * __P,int __D)367 _storebe_i32(void * __P, int __D) {
368 struct __storeu_i32 {
369 int __v;
370 } __attribute__((__packed__, __may_alias__));
371 ((struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D);
372 }
373
374 #ifdef __x86_64__
375 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_loadbe_i64(void const * __P)376 _loadbe_i64(void const * __P) {
377 struct __loadu_i64 {
378 long long __v;
379 } __attribute__((__packed__, __may_alias__));
380 return __builtin_bswap64(((const struct __loadu_i64*)__P)->__v);
381 }
382
383 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
_storebe_i64(void * __P,long long __D)384 _storebe_i64(void * __P, long long __D) {
385 struct __storeu_i64 {
386 long long __v;
387 } __attribute__((__packed__, __may_alias__));
388 ((struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D);
389 }
390 #endif
391 #endif /* __MOVBE */
392
393 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
394 defined(__RTM__)
395 #include <rtmintrin.h>
396 #include <xtestintrin.h>
397 #endif
398
399 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
400 defined(__SHA__)
401 #include <shaintrin.h>
402 #endif
403
404 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
405 defined(__FXSR__)
406 #include <fxsrintrin.h>
407 #endif
408
409 /* No feature check desired due to internal MSC_VER checks */
410 #include <xsaveintrin.h>
411
412 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
413 defined(__XSAVEOPT__)
414 #include <xsaveoptintrin.h>
415 #endif
416
417 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
418 defined(__XSAVEC__)
419 #include <xsavecintrin.h>
420 #endif
421
422 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
423 defined(__XSAVES__)
424 #include <xsavesintrin.h>
425 #endif
426
427 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
428 defined(__SHSTK__)
429 #include <cetintrin.h>
430 #endif
431
432 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
433 * whereas others are also available at all times. */
434 #include <adxintrin.h>
435
436 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
437 defined(__RDSEED__)
438 #include <rdseedintrin.h>
439 #endif
440
441 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
442 defined(__WBNOINVD__)
443 #include <wbnoinvdintrin.h>
444 #endif
445
446 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
447 defined(__CLDEMOTE__)
448 #include <cldemoteintrin.h>
449 #endif
450
451 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
452 defined(__WAITPKG__)
453 #include <waitpkgintrin.h>
454 #endif
455
456 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
457 defined(__MOVDIRI__) || defined(__MOVDIR64B__)
458 #include <movdirintrin.h>
459 #endif
460
461 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
462 defined(__PCONFIG__)
463 #include <pconfigintrin.h>
464 #endif
465
466 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
467 defined(__SGX__)
468 #include <sgxintrin.h>
469 #endif
470
471 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
472 defined(__PTWRITE__)
473 #include <ptwriteintrin.h>
474 #endif
475
476 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
477 defined(__INVPCID__)
478 #include <invpcidintrin.h>
479 #endif
480
481 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
482 defined(__KL__) || defined(__WIDEKL__)
483 #include <keylockerintrin.h>
484 #endif
485
486 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
487 defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
488 #include <amxintrin.h>
489 #endif
490
491 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
492 defined(__AVX512VP2INTERSECT__)
493 #include <avx512vp2intersectintrin.h>
494 #endif
495
496 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
497 (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
498 #include <avx512vlvp2intersectintrin.h>
499 #endif
500
501 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
502 defined(__ENQCMD__)
503 #include <enqcmdintrin.h>
504 #endif
505
506 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
507 defined(__SERIALIZE__)
508 #include <serializeintrin.h>
509 #endif
510
511 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
512 defined(__TSXLDTRK__)
513 #include <tsxldtrkintrin.h>
514 #endif
515
516 #if defined(_MSC_VER) && __has_extension(gnu_asm)
517 /* Define the default attributes for these intrinsics */
518 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
519 #ifdef __cplusplus
520 extern "C" {
521 #endif
522 /*----------------------------------------------------------------------------*\
523 |* Interlocked Exchange HLE
524 \*----------------------------------------------------------------------------*/
525 #if defined(__i386__) || defined(__x86_64__)
526 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedExchange_HLEAcquire(long volatile * _Target,long _Value)527 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {
528 __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1"
529 : "+r" (_Value), "+m" (*_Target) :: "memory");
530 return _Value;
531 }
532 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedExchange_HLERelease(long volatile * _Target,long _Value)533 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {
534 __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1"
535 : "+r" (_Value), "+m" (*_Target) :: "memory");
536 return _Value;
537 }
538 #endif
539 #if defined(__x86_64__)
540 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedExchange64_HLEAcquire(__int64 volatile * _Target,__int64 _Value)541 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {
542 __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1"
543 : "+r" (_Value), "+m" (*_Target) :: "memory");
544 return _Value;
545 }
546 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedExchange64_HLERelease(__int64 volatile * _Target,__int64 _Value)547 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {
548 __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1"
549 : "+r" (_Value), "+m" (*_Target) :: "memory");
550 return _Value;
551 }
552 #endif
553 /*----------------------------------------------------------------------------*\
554 |* Interlocked Compare Exchange HLE
555 \*----------------------------------------------------------------------------*/
556 #if defined(__i386__) || defined(__x86_64__)
557 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedCompareExchange_HLEAcquire(long volatile * _Destination,long _Exchange,long _Comparand)558 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,
559 long _Exchange, long _Comparand) {
560 __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1"
561 : "+a" (_Comparand), "+m" (*_Destination)
562 : "r" (_Exchange) : "memory");
563 return _Comparand;
564 }
565 static __inline__ long __DEFAULT_FN_ATTRS
_InterlockedCompareExchange_HLERelease(long volatile * _Destination,long _Exchange,long _Comparand)566 _InterlockedCompareExchange_HLERelease(long volatile *_Destination,
567 long _Exchange, long _Comparand) {
568 __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1"
569 : "+a" (_Comparand), "+m" (*_Destination)
570 : "r" (_Exchange) : "memory");
571 return _Comparand;
572 }
573 #endif
574 #if defined(__x86_64__)
575 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedCompareExchange64_HLEAcquire(__int64 volatile * _Destination,__int64 _Exchange,__int64 _Comparand)576 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,
577 __int64 _Exchange, __int64 _Comparand) {
578 __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1"
579 : "+a" (_Comparand), "+m" (*_Destination)
580 : "r" (_Exchange) : "memory");
581 return _Comparand;
582 }
583 static __inline__ __int64 __DEFAULT_FN_ATTRS
_InterlockedCompareExchange64_HLERelease(__int64 volatile * _Destination,__int64 _Exchange,__int64 _Comparand)584 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
585 __int64 _Exchange, __int64 _Comparand) {
586 __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1"
587 : "+a" (_Comparand), "+m" (*_Destination)
588 : "r" (_Exchange) : "memory");
589 return _Comparand;
590 }
591 #endif
592 #ifdef __cplusplus
593 }
594 #endif
595
596 #undef __DEFAULT_FN_ATTRS
597
598 #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */
599
600 #endif /* __IMMINTRIN_H */
601