1 /* Copyright (C) 2002-2013 Free Software Foundation, Inc.
2
3 This file is part of GCC.
4
5 GCC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3, or (at your option)
8 any later version.
9
10 GCC is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 Under Section 7 of GPL version 3, you are granted additional
16 permissions described in the GCC Runtime Library Exception, version
17 3.1, as published by the Free Software Foundation.
18
19 You should have received a copy of the GNU General Public License and
20 a copy of the GCC Runtime Library Exception along with this program;
21 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
22 <http://www.gnu.org/licenses/>. */
23
24 /* Implemented from the specification included in the Intel C++ Compiler
25 User Guide and Reference, version 9.0. */
26
27 #ifndef _XMMINTRIN_H_INCLUDED
28 #define _XMMINTRIN_H_INCLUDED
29
30 #ifndef __SSE__
31 # error "SSE instruction set not enabled"
32 #else
33
34 /* We need type definitions from the MMX header file. */
35 #include <mmintrin.h>
36
37 /* Get _mm_malloc () and _mm_free (). */
38 #include <mm_malloc.h>
39
40 /* The Intel API is flexible enough that we must allow aliasing with other
41 vector types, and their scalar components. */
42 typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__));
43
44 /* Internal data types for implementing the intrinsics. */
45 typedef float __v4sf __attribute__ ((__vector_size__ (16)));
46
47 /* Create a selector for use with the SHUFPS instruction. */
48 #define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \
49 (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0))
50
51 /* Constants for use with _mm_prefetch. */
52 enum _mm_hint
53 {
54 _MM_HINT_T0 = 3,
55 _MM_HINT_T1 = 2,
56 _MM_HINT_T2 = 1,
57 _MM_HINT_NTA = 0
58 };
59
60 /* Bits in the MXCSR. */
61 #define _MM_EXCEPT_MASK 0x003f
62 #define _MM_EXCEPT_INVALID 0x0001
63 #define _MM_EXCEPT_DENORM 0x0002
64 #define _MM_EXCEPT_DIV_ZERO 0x0004
65 #define _MM_EXCEPT_OVERFLOW 0x0008
66 #define _MM_EXCEPT_UNDERFLOW 0x0010
67 #define _MM_EXCEPT_INEXACT 0x0020
68
69 #define _MM_MASK_MASK 0x1f80
70 #define _MM_MASK_INVALID 0x0080
71 #define _MM_MASK_DENORM 0x0100
72 #define _MM_MASK_DIV_ZERO 0x0200
73 #define _MM_MASK_OVERFLOW 0x0400
74 #define _MM_MASK_UNDERFLOW 0x0800
75 #define _MM_MASK_INEXACT 0x1000
76
77 #define _MM_ROUND_MASK 0x6000
78 #define _MM_ROUND_NEAREST 0x0000
79 #define _MM_ROUND_DOWN 0x2000
80 #define _MM_ROUND_UP 0x4000
81 #define _MM_ROUND_TOWARD_ZERO 0x6000
82
83 #define _MM_FLUSH_ZERO_MASK 0x8000
84 #define _MM_FLUSH_ZERO_ON 0x8000
85 #define _MM_FLUSH_ZERO_OFF 0x0000
86
87 /* Create a vector of zeros. */
88 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setzero_ps(void)89 _mm_setzero_ps (void)
90 {
91 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };
92 }
93
94 /* Perform the respective operation on the lower SPFP (single-precision
95 floating-point) values of A and B; the upper three SPFP values are
96 passed through from A. */
97
98 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_ss(__m128 __A,__m128 __B)99 _mm_add_ss (__m128 __A, __m128 __B)
100 {
101 return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B);
102 }
103
104 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_ss(__m128 __A,__m128 __B)105 _mm_sub_ss (__m128 __A, __m128 __B)
106 {
107 return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B);
108 }
109
110 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_ss(__m128 __A,__m128 __B)111 _mm_mul_ss (__m128 __A, __m128 __B)
112 {
113 return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B);
114 }
115
116 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_div_ss(__m128 __A,__m128 __B)117 _mm_div_ss (__m128 __A, __m128 __B)
118 {
119 return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B);
120 }
121
122 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sqrt_ss(__m128 __A)123 _mm_sqrt_ss (__m128 __A)
124 {
125 return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A);
126 }
127
128 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rcp_ss(__m128 __A)129 _mm_rcp_ss (__m128 __A)
130 {
131 return (__m128) __builtin_ia32_rcpss ((__v4sf)__A);
132 }
133
134 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rsqrt_ss(__m128 __A)135 _mm_rsqrt_ss (__m128 __A)
136 {
137 return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A);
138 }
139
140 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_ss(__m128 __A,__m128 __B)141 _mm_min_ss (__m128 __A, __m128 __B)
142 {
143 return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B);
144 }
145
146 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_ss(__m128 __A,__m128 __B)147 _mm_max_ss (__m128 __A, __m128 __B)
148 {
149 return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B);
150 }
151
152 /* Perform the respective operation on the four SPFP values in A and B. */
153
154 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_ps(__m128 __A,__m128 __B)155 _mm_add_ps (__m128 __A, __m128 __B)
156 {
157 return (__m128) __builtin_ia32_addps ((__v4sf)__A, (__v4sf)__B);
158 }
159
160 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_ps(__m128 __A,__m128 __B)161 _mm_sub_ps (__m128 __A, __m128 __B)
162 {
163 return (__m128) __builtin_ia32_subps ((__v4sf)__A, (__v4sf)__B);
164 }
165
166 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_ps(__m128 __A,__m128 __B)167 _mm_mul_ps (__m128 __A, __m128 __B)
168 {
169 return (__m128) __builtin_ia32_mulps ((__v4sf)__A, (__v4sf)__B);
170 }
171
172 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_div_ps(__m128 __A,__m128 __B)173 _mm_div_ps (__m128 __A, __m128 __B)
174 {
175 return (__m128) __builtin_ia32_divps ((__v4sf)__A, (__v4sf)__B);
176 }
177
178 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sqrt_ps(__m128 __A)179 _mm_sqrt_ps (__m128 __A)
180 {
181 return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A);
182 }
183
184 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rcp_ps(__m128 __A)185 _mm_rcp_ps (__m128 __A)
186 {
187 return (__m128) __builtin_ia32_rcpps ((__v4sf)__A);
188 }
189
190 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rsqrt_ps(__m128 __A)191 _mm_rsqrt_ps (__m128 __A)
192 {
193 return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A);
194 }
195
196 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_ps(__m128 __A,__m128 __B)197 _mm_min_ps (__m128 __A, __m128 __B)
198 {
199 return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B);
200 }
201
202 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_ps(__m128 __A,__m128 __B)203 _mm_max_ps (__m128 __A, __m128 __B)
204 {
205 return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B);
206 }
207
208 /* Perform logical bit-wise operations on 128-bit values. */
209
210 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_and_ps(__m128 __A,__m128 __B)211 _mm_and_ps (__m128 __A, __m128 __B)
212 {
213 return __builtin_ia32_andps (__A, __B);
214 }
215
216 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_andnot_ps(__m128 __A,__m128 __B)217 _mm_andnot_ps (__m128 __A, __m128 __B)
218 {
219 return __builtin_ia32_andnps (__A, __B);
220 }
221
222 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_or_ps(__m128 __A,__m128 __B)223 _mm_or_ps (__m128 __A, __m128 __B)
224 {
225 return __builtin_ia32_orps (__A, __B);
226 }
227
228 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_xor_ps(__m128 __A,__m128 __B)229 _mm_xor_ps (__m128 __A, __m128 __B)
230 {
231 return __builtin_ia32_xorps (__A, __B);
232 }
233
234 /* Perform a comparison on the lower SPFP values of A and B. If the
235 comparison is true, place a mask of all ones in the result, otherwise a
236 mask of zeros. The upper three SPFP values are passed through from A. */
237
238 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_ss(__m128 __A,__m128 __B)239 _mm_cmpeq_ss (__m128 __A, __m128 __B)
240 {
241 return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B);
242 }
243
244 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_ss(__m128 __A,__m128 __B)245 _mm_cmplt_ss (__m128 __A, __m128 __B)
246 {
247 return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B);
248 }
249
250 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmple_ss(__m128 __A,__m128 __B)251 _mm_cmple_ss (__m128 __A, __m128 __B)
252 {
253 return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B);
254 }
255
256 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_ss(__m128 __A,__m128 __B)257 _mm_cmpgt_ss (__m128 __A, __m128 __B)
258 {
259 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
260 (__v4sf)
261 __builtin_ia32_cmpltss ((__v4sf) __B,
262 (__v4sf)
263 __A));
264 }
265
266 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpge_ss(__m128 __A,__m128 __B)267 _mm_cmpge_ss (__m128 __A, __m128 __B)
268 {
269 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
270 (__v4sf)
271 __builtin_ia32_cmpless ((__v4sf) __B,
272 (__v4sf)
273 __A));
274 }
275
276 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpneq_ss(__m128 __A,__m128 __B)277 _mm_cmpneq_ss (__m128 __A, __m128 __B)
278 {
279 return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B);
280 }
281
282 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnlt_ss(__m128 __A,__m128 __B)283 _mm_cmpnlt_ss (__m128 __A, __m128 __B)
284 {
285 return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B);
286 }
287
288 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnle_ss(__m128 __A,__m128 __B)289 _mm_cmpnle_ss (__m128 __A, __m128 __B)
290 {
291 return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B);
292 }
293
294 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpngt_ss(__m128 __A,__m128 __B)295 _mm_cmpngt_ss (__m128 __A, __m128 __B)
296 {
297 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
298 (__v4sf)
299 __builtin_ia32_cmpnltss ((__v4sf) __B,
300 (__v4sf)
301 __A));
302 }
303
304 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnge_ss(__m128 __A,__m128 __B)305 _mm_cmpnge_ss (__m128 __A, __m128 __B)
306 {
307 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
308 (__v4sf)
309 __builtin_ia32_cmpnless ((__v4sf) __B,
310 (__v4sf)
311 __A));
312 }
313
314 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpord_ss(__m128 __A,__m128 __B)315 _mm_cmpord_ss (__m128 __A, __m128 __B)
316 {
317 return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B);
318 }
319
320 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpunord_ss(__m128 __A,__m128 __B)321 _mm_cmpunord_ss (__m128 __A, __m128 __B)
322 {
323 return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B);
324 }
325
326 /* Perform a comparison on the four SPFP values of A and B. For each
327 element, if the comparison is true, place a mask of all ones in the
328 result, otherwise a mask of zeros. */
329
330 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_ps(__m128 __A,__m128 __B)331 _mm_cmpeq_ps (__m128 __A, __m128 __B)
332 {
333 return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B);
334 }
335
336 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_ps(__m128 __A,__m128 __B)337 _mm_cmplt_ps (__m128 __A, __m128 __B)
338 {
339 return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B);
340 }
341
342 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmple_ps(__m128 __A,__m128 __B)343 _mm_cmple_ps (__m128 __A, __m128 __B)
344 {
345 return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B);
346 }
347
348 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_ps(__m128 __A,__m128 __B)349 _mm_cmpgt_ps (__m128 __A, __m128 __B)
350 {
351 return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B);
352 }
353
354 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpge_ps(__m128 __A,__m128 __B)355 _mm_cmpge_ps (__m128 __A, __m128 __B)
356 {
357 return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B);
358 }
359
360 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpneq_ps(__m128 __A,__m128 __B)361 _mm_cmpneq_ps (__m128 __A, __m128 __B)
362 {
363 return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B);
364 }
365
366 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnlt_ps(__m128 __A,__m128 __B)367 _mm_cmpnlt_ps (__m128 __A, __m128 __B)
368 {
369 return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B);
370 }
371
372 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnle_ps(__m128 __A,__m128 __B)373 _mm_cmpnle_ps (__m128 __A, __m128 __B)
374 {
375 return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B);
376 }
377
378 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpngt_ps(__m128 __A,__m128 __B)379 _mm_cmpngt_ps (__m128 __A, __m128 __B)
380 {
381 return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B);
382 }
383
384 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnge_ps(__m128 __A,__m128 __B)385 _mm_cmpnge_ps (__m128 __A, __m128 __B)
386 {
387 return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B);
388 }
389
390 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpord_ps(__m128 __A,__m128 __B)391 _mm_cmpord_ps (__m128 __A, __m128 __B)
392 {
393 return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B);
394 }
395
396 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpunord_ps(__m128 __A,__m128 __B)397 _mm_cmpunord_ps (__m128 __A, __m128 __B)
398 {
399 return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B);
400 }
401
402 /* Compare the lower SPFP values of A and B and return 1 if true
403 and 0 if false. */
404
405 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comieq_ss(__m128 __A,__m128 __B)406 _mm_comieq_ss (__m128 __A, __m128 __B)
407 {
408 return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B);
409 }
410
411 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comilt_ss(__m128 __A,__m128 __B)412 _mm_comilt_ss (__m128 __A, __m128 __B)
413 {
414 return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B);
415 }
416
417 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comile_ss(__m128 __A,__m128 __B)418 _mm_comile_ss (__m128 __A, __m128 __B)
419 {
420 return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B);
421 }
422
423 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comigt_ss(__m128 __A,__m128 __B)424 _mm_comigt_ss (__m128 __A, __m128 __B)
425 {
426 return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B);
427 }
428
429 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comige_ss(__m128 __A,__m128 __B)430 _mm_comige_ss (__m128 __A, __m128 __B)
431 {
432 return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B);
433 }
434
435 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comineq_ss(__m128 __A,__m128 __B)436 _mm_comineq_ss (__m128 __A, __m128 __B)
437 {
438 return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B);
439 }
440
441 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomieq_ss(__m128 __A,__m128 __B)442 _mm_ucomieq_ss (__m128 __A, __m128 __B)
443 {
444 return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B);
445 }
446
447 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomilt_ss(__m128 __A,__m128 __B)448 _mm_ucomilt_ss (__m128 __A, __m128 __B)
449 {
450 return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B);
451 }
452
453 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomile_ss(__m128 __A,__m128 __B)454 _mm_ucomile_ss (__m128 __A, __m128 __B)
455 {
456 return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B);
457 }
458
459 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomigt_ss(__m128 __A,__m128 __B)460 _mm_ucomigt_ss (__m128 __A, __m128 __B)
461 {
462 return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B);
463 }
464
465 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomige_ss(__m128 __A,__m128 __B)466 _mm_ucomige_ss (__m128 __A, __m128 __B)
467 {
468 return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B);
469 }
470
471 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomineq_ss(__m128 __A,__m128 __B)472 _mm_ucomineq_ss (__m128 __A, __m128 __B)
473 {
474 return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B);
475 }
476
477 /* Convert the lower SPFP value to a 32-bit integer according to the current
478 rounding mode. */
479 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_si32(__m128 __A)480 _mm_cvtss_si32 (__m128 __A)
481 {
482 return __builtin_ia32_cvtss2si ((__v4sf) __A);
483 }
484
485 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_ss2si(__m128 __A)486 _mm_cvt_ss2si (__m128 __A)
487 {
488 return _mm_cvtss_si32 (__A);
489 }
490
491 #ifdef __x86_64__
492 /* Convert the lower SPFP value to a 32-bit integer according to the
493 current rounding mode. */
494
495 /* Intel intrinsic. */
496 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_si64(__m128 __A)497 _mm_cvtss_si64 (__m128 __A)
498 {
499 return __builtin_ia32_cvtss2si64 ((__v4sf) __A);
500 }
501
502 /* Microsoft intrinsic. */
503 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_si64x(__m128 __A)504 _mm_cvtss_si64x (__m128 __A)
505 {
506 return __builtin_ia32_cvtss2si64 ((__v4sf) __A);
507 }
508 #endif
509
510 /* Convert the two lower SPFP values to 32-bit integers according to the
511 current rounding mode. Return the integers in packed form. */
512 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pi32(__m128 __A)513 _mm_cvtps_pi32 (__m128 __A)
514 {
515 return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A);
516 }
517
518 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_ps2pi(__m128 __A)519 _mm_cvt_ps2pi (__m128 __A)
520 {
521 return _mm_cvtps_pi32 (__A);
522 }
523
524 /* Truncate the lower SPFP value to a 32-bit integer. */
525 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttss_si32(__m128 __A)526 _mm_cvttss_si32 (__m128 __A)
527 {
528 return __builtin_ia32_cvttss2si ((__v4sf) __A);
529 }
530
531 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtt_ss2si(__m128 __A)532 _mm_cvtt_ss2si (__m128 __A)
533 {
534 return _mm_cvttss_si32 (__A);
535 }
536
537 #ifdef __x86_64__
538 /* Truncate the lower SPFP value to a 32-bit integer. */
539
540 /* Intel intrinsic. */
541 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttss_si64(__m128 __A)542 _mm_cvttss_si64 (__m128 __A)
543 {
544 return __builtin_ia32_cvttss2si64 ((__v4sf) __A);
545 }
546
547 /* Microsoft intrinsic. */
548 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttss_si64x(__m128 __A)549 _mm_cvttss_si64x (__m128 __A)
550 {
551 return __builtin_ia32_cvttss2si64 ((__v4sf) __A);
552 }
553 #endif
554
555 /* Truncate the two lower SPFP values to 32-bit integers. Return the
556 integers in packed form. */
557 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttps_pi32(__m128 __A)558 _mm_cvttps_pi32 (__m128 __A)
559 {
560 return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A);
561 }
562
563 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtt_ps2pi(__m128 __A)564 _mm_cvtt_ps2pi (__m128 __A)
565 {
566 return _mm_cvttps_pi32 (__A);
567 }
568
569 /* Convert B to a SPFP value and insert it as element zero in A. */
570 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi32_ss(__m128 __A,int __B)571 _mm_cvtsi32_ss (__m128 __A, int __B)
572 {
573 return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B);
574 }
575
576 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_si2ss(__m128 __A,int __B)577 _mm_cvt_si2ss (__m128 __A, int __B)
578 {
579 return _mm_cvtsi32_ss (__A, __B);
580 }
581
582 #ifdef __x86_64__
583 /* Convert B to a SPFP value and insert it as element zero in A. */
584
585 /* Intel intrinsic. */
586 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi64_ss(__m128 __A,long long __B)587 _mm_cvtsi64_ss (__m128 __A, long long __B)
588 {
589 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B);
590 }
591
592 /* Microsoft intrinsic. */
593 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi64x_ss(__m128 __A,long long __B)594 _mm_cvtsi64x_ss (__m128 __A, long long __B)
595 {
596 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B);
597 }
598 #endif
599
600 /* Convert the two 32-bit values in B to SPFP form and insert them
601 as the two lower elements in A. */
602 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi32_ps(__m128 __A,__m64 __B)603 _mm_cvtpi32_ps (__m128 __A, __m64 __B)
604 {
605 return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B);
606 }
607
608 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_pi2ps(__m128 __A,__m64 __B)609 _mm_cvt_pi2ps (__m128 __A, __m64 __B)
610 {
611 return _mm_cvtpi32_ps (__A, __B);
612 }
613
614 /* Convert the four signed 16-bit values in A to SPFP form. */
615 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi16_ps(__m64 __A)616 _mm_cvtpi16_ps (__m64 __A)
617 {
618 __v4hi __sign;
619 __v2si __hisi, __losi;
620 __v4sf __zero, __ra, __rb;
621
622 /* This comparison against zero gives us a mask that can be used to
623 fill in the missing sign bits in the unpack operations below, so
624 that we get signed values after unpacking. */
625 __sign = __builtin_ia32_pcmpgtw ((__v4hi)0LL, (__v4hi)__A);
626
627 /* Convert the four words to doublewords. */
628 __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign);
629 __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign);
630
631 /* Convert the doublewords to floating point two at a time. */
632 __zero = (__v4sf) _mm_setzero_ps ();
633 __ra = __builtin_ia32_cvtpi2ps (__zero, __losi);
634 __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi);
635
636 return (__m128) __builtin_ia32_movlhps (__ra, __rb);
637 }
638
639 /* Convert the four unsigned 16-bit values in A to SPFP form. */
640 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpu16_ps(__m64 __A)641 _mm_cvtpu16_ps (__m64 __A)
642 {
643 __v2si __hisi, __losi;
644 __v4sf __zero, __ra, __rb;
645
646 /* Convert the four words to doublewords. */
647 __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, (__v4hi)0LL);
648 __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, (__v4hi)0LL);
649
650 /* Convert the doublewords to floating point two at a time. */
651 __zero = (__v4sf) _mm_setzero_ps ();
652 __ra = __builtin_ia32_cvtpi2ps (__zero, __losi);
653 __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi);
654
655 return (__m128) __builtin_ia32_movlhps (__ra, __rb);
656 }
657
658 /* Convert the low four signed 8-bit values in A to SPFP form. */
659 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi8_ps(__m64 __A)660 _mm_cvtpi8_ps (__m64 __A)
661 {
662 __v8qi __sign;
663
664 /* This comparison against zero gives us a mask that can be used to
665 fill in the missing sign bits in the unpack operations below, so
666 that we get signed values after unpacking. */
667 __sign = __builtin_ia32_pcmpgtb ((__v8qi)0LL, (__v8qi)__A);
668
669 /* Convert the four low bytes to words. */
670 __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign);
671
672 return _mm_cvtpi16_ps(__A);
673 }
674
675 /* Convert the low four unsigned 8-bit values in A to SPFP form. */
676 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpu8_ps(__m64 __A)677 _mm_cvtpu8_ps(__m64 __A)
678 {
679 __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, (__v8qi)0LL);
680 return _mm_cvtpu16_ps(__A);
681 }
682
683 /* Convert the four signed 32-bit values in A and B to SPFP form. */
684 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi32x2_ps(__m64 __A,__m64 __B)685 _mm_cvtpi32x2_ps(__m64 __A, __m64 __B)
686 {
687 __v4sf __zero = (__v4sf) _mm_setzero_ps ();
688 __v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A);
689 __v4sf __sfb = __builtin_ia32_cvtpi2ps (__sfa, (__v2si)__B);
690 return (__m128) __builtin_ia32_movlhps (__sfa, __sfb);
691 }
692
693 /* Convert the four SPFP values in A to four signed 16-bit integers. */
694 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pi16(__m128 __A)695 _mm_cvtps_pi16(__m128 __A)
696 {
697 __v4sf __hisf = (__v4sf)__A;
698 __v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf);
699 __v2si __hisi = __builtin_ia32_cvtps2pi (__hisf);
700 __v2si __losi = __builtin_ia32_cvtps2pi (__losf);
701 return (__m64) __builtin_ia32_packssdw (__hisi, __losi);
702 }
703
704 /* Convert the four SPFP values in A to four signed 8-bit integers. */
705 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pi8(__m128 __A)706 _mm_cvtps_pi8(__m128 __A)
707 {
708 __v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A);
709 return (__m64) __builtin_ia32_packsswb (__tmp, (__v4hi)0LL);
710 }
711
712 /* Selects four specific SPFP values from A and B based on MASK. */
713 #ifdef __OPTIMIZE__
714 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_ps(__m128 __A,__m128 __B,int const __mask)715 _mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask)
716 {
717 return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask);
718 }
719 #else
720 #define _mm_shuffle_ps(A, B, MASK) \
721 ((__m128) __builtin_ia32_shufps ((__v4sf)(__m128)(A), \
722 (__v4sf)(__m128)(B), (int)(MASK)))
723 #endif
724
725 /* Selects and interleaves the upper two SPFP values from A and B. */
726 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_ps(__m128 __A,__m128 __B)727 _mm_unpackhi_ps (__m128 __A, __m128 __B)
728 {
729 return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B);
730 }
731
732 /* Selects and interleaves the lower two SPFP values from A and B. */
733 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_ps(__m128 __A,__m128 __B)734 _mm_unpacklo_ps (__m128 __A, __m128 __B)
735 {
736 return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B);
737 }
738
739 /* Sets the upper two SPFP values with 64-bits of data loaded from P;
740 the lower two values are passed through from A. */
741 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadh_pi(__m128 __A,__m64 const * __P)742 _mm_loadh_pi (__m128 __A, __m64 const *__P)
743 {
744 return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (const __v2sf *)__P);
745 }
746
747 /* Stores the upper two SPFP values of A into P. */
748 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeh_pi(__m64 * __P,__m128 __A)749 _mm_storeh_pi (__m64 *__P, __m128 __A)
750 {
751 __builtin_ia32_storehps ((__v2sf *)__P, (__v4sf)__A);
752 }
753
754 /* Moves the upper two values of B into the lower two values of A. */
755 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movehl_ps(__m128 __A,__m128 __B)756 _mm_movehl_ps (__m128 __A, __m128 __B)
757 {
758 return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B);
759 }
760
761 /* Moves the lower two values of B into the upper two values of A. */
762 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movelh_ps(__m128 __A,__m128 __B)763 _mm_movelh_ps (__m128 __A, __m128 __B)
764 {
765 return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B);
766 }
767
768 /* Sets the lower two SPFP values with 64-bits of data loaded from P;
769 the upper two values are passed through from A. */
770 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadl_pi(__m128 __A,__m64 const * __P)771 _mm_loadl_pi (__m128 __A, __m64 const *__P)
772 {
773 return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (const __v2sf *)__P);
774 }
775
776 /* Stores the lower two SPFP values of A into P. */
777 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storel_pi(__m64 * __P,__m128 __A)778 _mm_storel_pi (__m64 *__P, __m128 __A)
779 {
780 __builtin_ia32_storelps ((__v2sf *)__P, (__v4sf)__A);
781 }
782
783 /* Creates a 4-bit mask from the most significant bits of the SPFP values. */
784 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movemask_ps(__m128 __A)785 _mm_movemask_ps (__m128 __A)
786 {
787 return __builtin_ia32_movmskps ((__v4sf)__A);
788 }
789
790 /* Return the contents of the control register. */
791 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_getcsr(void)792 _mm_getcsr (void)
793 {
794 return __builtin_ia32_stmxcsr ();
795 }
796
797 /* Read exception bits from the control register. */
798 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_EXCEPTION_STATE(void)799 _MM_GET_EXCEPTION_STATE (void)
800 {
801 return _mm_getcsr() & _MM_EXCEPT_MASK;
802 }
803
804 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_EXCEPTION_MASK(void)805 _MM_GET_EXCEPTION_MASK (void)
806 {
807 return _mm_getcsr() & _MM_MASK_MASK;
808 }
809
810 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_ROUNDING_MODE(void)811 _MM_GET_ROUNDING_MODE (void)
812 {
813 return _mm_getcsr() & _MM_ROUND_MASK;
814 }
815
816 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_FLUSH_ZERO_MODE(void)817 _MM_GET_FLUSH_ZERO_MODE (void)
818 {
819 return _mm_getcsr() & _MM_FLUSH_ZERO_MASK;
820 }
821
822 /* Set the control register to I. */
823 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setcsr(unsigned int __I)824 _mm_setcsr (unsigned int __I)
825 {
826 __builtin_ia32_ldmxcsr (__I);
827 }
828
829 /* Set exception bits in the control register. */
830 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_EXCEPTION_STATE(unsigned int __mask)831 _MM_SET_EXCEPTION_STATE(unsigned int __mask)
832 {
833 _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | __mask);
834 }
835
836 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_EXCEPTION_MASK(unsigned int __mask)837 _MM_SET_EXCEPTION_MASK (unsigned int __mask)
838 {
839 _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | __mask);
840 }
841
842 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_ROUNDING_MODE(unsigned int __mode)843 _MM_SET_ROUNDING_MODE (unsigned int __mode)
844 {
845 _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | __mode);
846 }
847
848 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_FLUSH_ZERO_MODE(unsigned int __mode)849 _MM_SET_FLUSH_ZERO_MODE (unsigned int __mode)
850 {
851 _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | __mode);
852 }
853
854 /* Create a vector with element 0 as F and the rest zero. */
855 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_ss(float __F)856 _mm_set_ss (float __F)
857 {
858 return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f };
859 }
860
861 /* Create a vector with all four elements equal to F. */
862 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_ps(float __F)863 _mm_set1_ps (float __F)
864 {
865 return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F };
866 }
867
868 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_ps1(float __F)869 _mm_set_ps1 (float __F)
870 {
871 return _mm_set1_ps (__F);
872 }
873
874 /* Create a vector with element 0 as *P and the rest zero. */
875 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_ss(float const * __P)876 _mm_load_ss (float const *__P)
877 {
878 return _mm_set_ss (*__P);
879 }
880
881 /* Create a vector with all four elements equal to *P. */
882 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load1_ps(float const * __P)883 _mm_load1_ps (float const *__P)
884 {
885 return _mm_set1_ps (*__P);
886 }
887
888 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_ps1(float const * __P)889 _mm_load_ps1 (float const *__P)
890 {
891 return _mm_load1_ps (__P);
892 }
893
894 /* Load four SPFP values from P. The address must be 16-byte aligned. */
895 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_ps(float const * __P)896 _mm_load_ps (float const *__P)
897 {
898 return (__m128) *(__v4sf *)__P;
899 }
900
901 /* Load four SPFP values from P. The address need not be 16-byte aligned. */
902 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_ps(float const * __P)903 _mm_loadu_ps (float const *__P)
904 {
905 return (__m128) __builtin_ia32_loadups (__P);
906 }
907
908 /* Load four SPFP values in reverse order. The address must be aligned. */
909 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadr_ps(float const * __P)910 _mm_loadr_ps (float const *__P)
911 {
912 __v4sf __tmp = *(__v4sf *)__P;
913 return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,1,2,3));
914 }
915
916 /* Create the vector [Z Y X W]. */
917 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_ps(const float __Z,const float __Y,const float __X,const float __W)918 _mm_set_ps (const float __Z, const float __Y, const float __X, const float __W)
919 {
920 return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z };
921 }
922
923 /* Create the vector [W X Y Z]. */
924 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_ps(float __Z,float __Y,float __X,float __W)925 _mm_setr_ps (float __Z, float __Y, float __X, float __W)
926 {
927 return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W };
928 }
929
930 /* Stores the lower SPFP value. */
931 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_ss(float * __P,__m128 __A)932 _mm_store_ss (float *__P, __m128 __A)
933 {
934 *__P = __builtin_ia32_vec_ext_v4sf ((__v4sf)__A, 0);
935 }
936
937 extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_f32(__m128 __A)938 _mm_cvtss_f32 (__m128 __A)
939 {
940 return __builtin_ia32_vec_ext_v4sf ((__v4sf)__A, 0);
941 }
942
943 /* Store four SPFP values. The address must be 16-byte aligned. */
944 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_ps(float * __P,__m128 __A)945 _mm_store_ps (float *__P, __m128 __A)
946 {
947 *(__v4sf *)__P = (__v4sf)__A;
948 }
949
950 /* Store four SPFP values. The address need not be 16-byte aligned. */
951 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_ps(float * __P,__m128 __A)952 _mm_storeu_ps (float *__P, __m128 __A)
953 {
954 __builtin_ia32_storeups (__P, (__v4sf)__A);
955 }
956
957 /* Store the lower SPFP value across four words. */
958 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store1_ps(float * __P,__m128 __A)959 _mm_store1_ps (float *__P, __m128 __A)
960 {
961 __v4sf __va = (__v4sf)__A;
962 __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,0,0,0));
963 _mm_storeu_ps (__P, __tmp);
964 }
965
966 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_ps1(float * __P,__m128 __A)967 _mm_store_ps1 (float *__P, __m128 __A)
968 {
969 _mm_store1_ps (__P, __A);
970 }
971
972 /* Store four SPFP values in reverse order. The address must be aligned. */
973 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storer_ps(float * __P,__m128 __A)974 _mm_storer_ps (float *__P, __m128 __A)
975 {
976 __v4sf __va = (__v4sf)__A;
977 __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,1,2,3));
978 _mm_store_ps (__P, __tmp);
979 }
980
981 /* Sets the low SPFP value of A from the low value of B. */
982 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_move_ss(__m128 __A,__m128 __B)983 _mm_move_ss (__m128 __A, __m128 __B)
984 {
985 return (__m128) __builtin_ia32_movss ((__v4sf)__A, (__v4sf)__B);
986 }
987
988 /* Extracts one of the four words of A. The selector N must be immediate. */
989 #ifdef __OPTIMIZE__
990 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_extract_pi16(__m64 const __A,int const __N)991 _mm_extract_pi16 (__m64 const __A, int const __N)
992 {
993 return __builtin_ia32_vec_ext_v4hi ((__v4hi)__A, __N);
994 }
995
996 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pextrw(__m64 const __A,int const __N)997 _m_pextrw (__m64 const __A, int const __N)
998 {
999 return _mm_extract_pi16 (__A, __N);
1000 }
1001 #else
1002 #define _mm_extract_pi16(A, N) \
1003 ((int) __builtin_ia32_vec_ext_v4hi ((__v4hi)(__m64)(A), (int)(N)))
1004
1005 #define _m_pextrw(A, N) _mm_extract_pi16(A, N)
1006 #endif
1007
1008 /* Inserts word D into one of four words of A. The selector N must be
1009 immediate. */
1010 #ifdef __OPTIMIZE__
1011 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_insert_pi16(__m64 const __A,int const __D,int const __N)1012 _mm_insert_pi16 (__m64 const __A, int const __D, int const __N)
1013 {
1014 return (__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)__A, __D, __N);
1015 }
1016
1017 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pinsrw(__m64 const __A,int const __D,int const __N)1018 _m_pinsrw (__m64 const __A, int const __D, int const __N)
1019 {
1020 return _mm_insert_pi16 (__A, __D, __N);
1021 }
1022 #else
1023 #define _mm_insert_pi16(A, D, N) \
1024 ((__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)(__m64)(A), \
1025 (int)(D), (int)(N)))
1026
1027 #define _m_pinsrw(A, D, N) _mm_insert_pi16(A, D, N)
1028 #endif
1029
1030 /* Compute the element-wise maximum of signed 16-bit values. */
1031 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_pi16(__m64 __A,__m64 __B)1032 _mm_max_pi16 (__m64 __A, __m64 __B)
1033 {
1034 return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B);
1035 }
1036
1037 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmaxsw(__m64 __A,__m64 __B)1038 _m_pmaxsw (__m64 __A, __m64 __B)
1039 {
1040 return _mm_max_pi16 (__A, __B);
1041 }
1042
1043 /* Compute the element-wise maximum of unsigned 8-bit values. */
1044 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_pu8(__m64 __A,__m64 __B)1045 _mm_max_pu8 (__m64 __A, __m64 __B)
1046 {
1047 return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B);
1048 }
1049
1050 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmaxub(__m64 __A,__m64 __B)1051 _m_pmaxub (__m64 __A, __m64 __B)
1052 {
1053 return _mm_max_pu8 (__A, __B);
1054 }
1055
1056 /* Compute the element-wise minimum of signed 16-bit values. */
1057 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_pi16(__m64 __A,__m64 __B)1058 _mm_min_pi16 (__m64 __A, __m64 __B)
1059 {
1060 return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B);
1061 }
1062
1063 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pminsw(__m64 __A,__m64 __B)1064 _m_pminsw (__m64 __A, __m64 __B)
1065 {
1066 return _mm_min_pi16 (__A, __B);
1067 }
1068
1069 /* Compute the element-wise minimum of unsigned 8-bit values. */
1070 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_pu8(__m64 __A,__m64 __B)1071 _mm_min_pu8 (__m64 __A, __m64 __B)
1072 {
1073 return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B);
1074 }
1075
1076 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pminub(__m64 __A,__m64 __B)1077 _m_pminub (__m64 __A, __m64 __B)
1078 {
1079 return _mm_min_pu8 (__A, __B);
1080 }
1081
1082 /* Create an 8-bit mask of the signs of 8-bit values. */
1083 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movemask_pi8(__m64 __A)1084 _mm_movemask_pi8 (__m64 __A)
1085 {
1086 return __builtin_ia32_pmovmskb ((__v8qi)__A);
1087 }
1088
1089 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmovmskb(__m64 __A)1090 _m_pmovmskb (__m64 __A)
1091 {
1092 return _mm_movemask_pi8 (__A);
1093 }
1094
1095 /* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
1096 in B and produce the high 16 bits of the 32-bit results. */
1097 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhi_pu16(__m64 __A,__m64 __B)1098 _mm_mulhi_pu16 (__m64 __A, __m64 __B)
1099 {
1100 return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B);
1101 }
1102
1103 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmulhuw(__m64 __A,__m64 __B)1104 _m_pmulhuw (__m64 __A, __m64 __B)
1105 {
1106 return _mm_mulhi_pu16 (__A, __B);
1107 }
1108
1109 /* Return a combination of the four 16-bit values in A. The selector
1110 must be an immediate. */
1111 #ifdef __OPTIMIZE__
1112 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_pi16(__m64 __A,int const __N)1113 _mm_shuffle_pi16 (__m64 __A, int const __N)
1114 {
1115 return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N);
1116 }
1117
1118 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pshufw(__m64 __A,int const __N)1119 _m_pshufw (__m64 __A, int const __N)
1120 {
1121 return _mm_shuffle_pi16 (__A, __N);
1122 }
1123 #else
1124 #define _mm_shuffle_pi16(A, N) \
1125 ((__m64) __builtin_ia32_pshufw ((__v4hi)(__m64)(A), (int)(N)))
1126
1127 #define _m_pshufw(A, N) _mm_shuffle_pi16 (A, N)
1128 #endif
1129
1130 /* Conditionally store byte elements of A into P. The high bit of each
1131 byte in the selector N determines whether the corresponding byte from
1132 A is stored. */
1133 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_maskmove_si64(__m64 __A,__m64 __N,char * __P)1134 _mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P)
1135 {
1136 __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P);
1137 }
1138
1139 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_maskmovq(__m64 __A,__m64 __N,char * __P)1140 _m_maskmovq (__m64 __A, __m64 __N, char *__P)
1141 {
1142 _mm_maskmove_si64 (__A, __N, __P);
1143 }
1144
1145 /* Compute the rounded averages of the unsigned 8-bit values in A and B. */
1146 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_avg_pu8(__m64 __A,__m64 __B)1147 _mm_avg_pu8 (__m64 __A, __m64 __B)
1148 {
1149 return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B);
1150 }
1151
1152 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pavgb(__m64 __A,__m64 __B)1153 _m_pavgb (__m64 __A, __m64 __B)
1154 {
1155 return _mm_avg_pu8 (__A, __B);
1156 }
1157
1158 /* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1159 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_avg_pu16(__m64 __A,__m64 __B)1160 _mm_avg_pu16 (__m64 __A, __m64 __B)
1161 {
1162 return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B);
1163 }
1164
1165 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pavgw(__m64 __A,__m64 __B)1166 _m_pavgw (__m64 __A, __m64 __B)
1167 {
1168 return _mm_avg_pu16 (__A, __B);
1169 }
1170
1171 /* Compute the sum of the absolute differences of the unsigned 8-bit
1172 values in A and B. Return the value in the lower 16-bit word; the
1173 upper words are cleared. */
1174 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sad_pu8(__m64 __A,__m64 __B)1175 _mm_sad_pu8 (__m64 __A, __m64 __B)
1176 {
1177 return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B);
1178 }
1179
1180 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psadbw(__m64 __A,__m64 __B)1181 _m_psadbw (__m64 __A, __m64 __B)
1182 {
1183 return _mm_sad_pu8 (__A, __B);
1184 }
1185
1186 /* Loads one cache line from address P to a location "closer" to the
1187 processor. The selector I specifies the type of prefetch operation. */
1188 #ifdef __OPTIMIZE__
1189 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_prefetch(const void * __P,enum _mm_hint __I)1190 _mm_prefetch (const void *__P, enum _mm_hint __I)
1191 {
1192 __builtin_prefetch (__P, 0, __I);
1193 }
1194 #else
1195 #define _mm_prefetch(P, I) \
1196 __builtin_prefetch ((P), 0, (I))
1197 #endif
1198
1199 /* Stores the data in A to the address P without polluting the caches. */
1200 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_pi(__m64 * __P,__m64 __A)1201 _mm_stream_pi (__m64 *__P, __m64 __A)
1202 {
1203 __builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A);
1204 }
1205
1206 /* Likewise. The address must be 16-byte aligned. */
1207 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_ps(float * __P,__m128 __A)1208 _mm_stream_ps (float *__P, __m128 __A)
1209 {
1210 __builtin_ia32_movntps (__P, (__v4sf)__A);
1211 }
1212
1213 /* Guarantees that every preceding store is globally visible before
1214 any subsequent store. */
1215 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sfence(void)1216 _mm_sfence (void)
1217 {
1218 __builtin_ia32_sfence ();
1219 }
1220
1221 /* The execution of the next instruction is delayed by an implementation
1222 specific amount of time. The instruction does not modify the
1223 architectural state. */
1224 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_pause(void)1225 _mm_pause (void)
1226 {
1227 __builtin_ia32_pause ();
1228 }
1229
1230 /* Transpose the 4x4 matrix composed of row[0-3]. */
1231 #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
1232 do { \
1233 __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \
1234 __v4sf __t0 = __builtin_ia32_unpcklps (__r0, __r1); \
1235 __v4sf __t1 = __builtin_ia32_unpcklps (__r2, __r3); \
1236 __v4sf __t2 = __builtin_ia32_unpckhps (__r0, __r1); \
1237 __v4sf __t3 = __builtin_ia32_unpckhps (__r2, __r3); \
1238 (row0) = __builtin_ia32_movlhps (__t0, __t1); \
1239 (row1) = __builtin_ia32_movhlps (__t1, __t0); \
1240 (row2) = __builtin_ia32_movlhps (__t2, __t3); \
1241 (row3) = __builtin_ia32_movhlps (__t3, __t2); \
1242 } while (0)
1243
1244 /* For backward source compatibility. */
1245 #ifdef __SSE2__
1246 # include <emmintrin.h>
1247 #endif
1248
1249 #endif /* __SSE__ */
1250 #endif /* _XMMINTRIN_H_INCLUDED */
1251