1 /*
2 * This file is part of the GROMACS molecular simulation package.
3 *
4 * Copyright (c) 2014,2015,2016,2017,2019,2020, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
8 *
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
13 *
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 *
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
31 *
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
34 */
35 #ifndef GMX_SIMD_IMPL_X86_SSE2_SIMD_DOUBLE_H
36 #define GMX_SIMD_IMPL_X86_SSE2_SIMD_DOUBLE_H
37
38 #include "config.h"
39
40 #include <cassert>
41 #include <cstddef>
42 #include <cstdint>
43
44 #include <emmintrin.h>
45
46 #include "gromacs/math/utilities.h"
47
48 #include "impl_x86_sse2_simd_float.h"
49
50 namespace gmx
51 {
52
53 class SimdDouble
54 {
55 public:
SimdDouble()56 SimdDouble() {}
57
SimdDouble(double d)58 SimdDouble(double d) : simdInternal_(_mm_set1_pd(d)) {}
59
60 // Internal utility constructor to simplify return statements
SimdDouble(__m128d simd)61 SimdDouble(__m128d simd) : simdInternal_(simd) {}
62
63 __m128d simdInternal_;
64 };
65
66 class SimdDInt32
67 {
68 public:
SimdDInt32()69 SimdDInt32() {}
70
SimdDInt32(std::int32_t i)71 SimdDInt32(std::int32_t i) : simdInternal_(_mm_set1_epi32(i)) {}
72
73 // Internal utility constructor to simplify return statements
SimdDInt32(__m128i simd)74 SimdDInt32(__m128i simd) : simdInternal_(simd) {}
75
76 __m128i simdInternal_;
77 };
78
79 class SimdDBool
80 {
81 public:
SimdDBool()82 SimdDBool() {}
83
SimdDBool(bool b)84 SimdDBool(bool b) : simdInternal_(_mm_castsi128_pd(_mm_set1_epi32(b ? 0xFFFFFFFF : 0))) {}
85
86 // Internal utility constructor to simplify return statements
SimdDBool(__m128d simd)87 SimdDBool(__m128d simd) : simdInternal_(simd) {}
88
89 __m128d simdInternal_;
90 };
91
92 class SimdDIBool
93 {
94 public:
SimdDIBool()95 SimdDIBool() {}
96
SimdDIBool(bool b)97 SimdDIBool(bool b) : simdInternal_(_mm_set1_epi32(b ? 0xFFFFFFFF : 0)) {}
98
99 // Internal utility constructor to simplify return statements
SimdDIBool(__m128i simd)100 SimdDIBool(__m128i simd) : simdInternal_(simd) {}
101
102 __m128i simdInternal_;
103 };
104
105 static inline SimdDouble gmx_simdcall simdLoad(const double* m, SimdDoubleTag = {})
106 {
107 assert(std::size_t(m) % 16 == 0);
108 return { _mm_load_pd(m) };
109 }
110
store(double * m,SimdDouble a)111 static inline void gmx_simdcall store(double* m, SimdDouble a)
112 {
113 assert(std::size_t(m) % 16 == 0);
114 _mm_store_pd(m, a.simdInternal_);
115 }
116
117 static inline SimdDouble gmx_simdcall simdLoadU(const double* m, SimdDoubleTag = {})
118 {
119 return { _mm_loadu_pd(m) };
120 }
121
storeU(double * m,SimdDouble a)122 static inline void gmx_simdcall storeU(double* m, SimdDouble a)
123 {
124 _mm_storeu_pd(m, a.simdInternal_);
125 }
126
setZeroD()127 static inline SimdDouble gmx_simdcall setZeroD()
128 {
129 return { _mm_setzero_pd() };
130 }
131
simdLoad(const std::int32_t * m,SimdDInt32Tag)132 static inline SimdDInt32 gmx_simdcall simdLoad(const std::int32_t* m, SimdDInt32Tag)
133 {
134 assert(std::size_t(m) % 8 == 0);
135 return { _mm_loadl_epi64(reinterpret_cast<const __m128i*>(m)) };
136 }
137
store(std::int32_t * m,SimdDInt32 a)138 static inline void gmx_simdcall store(std::int32_t* m, SimdDInt32 a)
139 {
140 assert(std::size_t(m) % 8 == 0);
141 _mm_storel_epi64(reinterpret_cast<__m128i*>(m), a.simdInternal_);
142 }
143
simdLoadU(const std::int32_t * m,SimdDInt32Tag)144 static inline SimdDInt32 gmx_simdcall simdLoadU(const std::int32_t* m, SimdDInt32Tag)
145 {
146 return { _mm_loadl_epi64(reinterpret_cast<const __m128i*>(m)) };
147 }
148
storeU(std::int32_t * m,SimdDInt32 a)149 static inline void gmx_simdcall storeU(std::int32_t* m, SimdDInt32 a)
150 {
151 _mm_storel_epi64(reinterpret_cast<__m128i*>(m), a.simdInternal_);
152 }
153
setZeroDI()154 static inline SimdDInt32 gmx_simdcall setZeroDI()
155 {
156 return { _mm_setzero_si128() };
157 }
158
159 // Override for SSE4.1 and higher
160 #if GMX_SIMD_X86_SSE2
161 template<int index>
extract(SimdDInt32 a)162 static inline std::int32_t gmx_simdcall extract(SimdDInt32 a)
163 {
164 return _mm_cvtsi128_si32(_mm_srli_si128(a.simdInternal_, 4 * index));
165 }
166 #endif
167
168 static inline SimdDouble gmx_simdcall operator&(SimdDouble a, SimdDouble b)
169 {
170 return { _mm_and_pd(a.simdInternal_, b.simdInternal_) };
171 }
172
andNot(SimdDouble a,SimdDouble b)173 static inline SimdDouble gmx_simdcall andNot(SimdDouble a, SimdDouble b)
174 {
175 return { _mm_andnot_pd(a.simdInternal_, b.simdInternal_) };
176 }
177
178 static inline SimdDouble gmx_simdcall operator|(SimdDouble a, SimdDouble b)
179 {
180 return { _mm_or_pd(a.simdInternal_, b.simdInternal_) };
181 }
182
183 static inline SimdDouble gmx_simdcall operator^(SimdDouble a, SimdDouble b)
184 {
185 return { _mm_xor_pd(a.simdInternal_, b.simdInternal_) };
186 }
187
188 static inline SimdDouble gmx_simdcall operator+(SimdDouble a, SimdDouble b)
189 {
190 return { _mm_add_pd(a.simdInternal_, b.simdInternal_) };
191 }
192
193 static inline SimdDouble gmx_simdcall operator-(SimdDouble a, SimdDouble b)
194 {
195 return { _mm_sub_pd(a.simdInternal_, b.simdInternal_) };
196 }
197
198 static inline SimdDouble gmx_simdcall operator-(SimdDouble x)
199 {
200 return { _mm_xor_pd(x.simdInternal_, _mm_set1_pd(GMX_DOUBLE_NEGZERO)) };
201 }
202
203 static inline SimdDouble gmx_simdcall operator*(SimdDouble a, SimdDouble b)
204 {
205 return { _mm_mul_pd(a.simdInternal_, b.simdInternal_) };
206 }
207
208 // Override for AVX-128-FMA and higher
209 #if GMX_SIMD_X86_SSE2 || GMX_SIMD_X86_SSE4_1
fma(SimdDouble a,SimdDouble b,SimdDouble c)210 static inline SimdDouble gmx_simdcall fma(SimdDouble a, SimdDouble b, SimdDouble c)
211 {
212 return { _mm_add_pd(_mm_mul_pd(a.simdInternal_, b.simdInternal_), c.simdInternal_) };
213 }
214
fms(SimdDouble a,SimdDouble b,SimdDouble c)215 static inline SimdDouble gmx_simdcall fms(SimdDouble a, SimdDouble b, SimdDouble c)
216 {
217 return { _mm_sub_pd(_mm_mul_pd(a.simdInternal_, b.simdInternal_), c.simdInternal_) };
218 }
219
fnma(SimdDouble a,SimdDouble b,SimdDouble c)220 static inline SimdDouble gmx_simdcall fnma(SimdDouble a, SimdDouble b, SimdDouble c)
221 {
222 return { _mm_sub_pd(c.simdInternal_, _mm_mul_pd(a.simdInternal_, b.simdInternal_)) };
223 }
224
fnms(SimdDouble a,SimdDouble b,SimdDouble c)225 static inline SimdDouble gmx_simdcall fnms(SimdDouble a, SimdDouble b, SimdDouble c)
226 {
227 return { _mm_sub_pd(_mm_setzero_pd(),
228 _mm_add_pd(_mm_mul_pd(a.simdInternal_, b.simdInternal_), c.simdInternal_)) };
229 }
230 #endif
231
rsqrt(SimdDouble x)232 static inline SimdDouble gmx_simdcall rsqrt(SimdDouble x)
233 {
234 return { _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(x.simdInternal_))) };
235 }
236
rcp(SimdDouble x)237 static inline SimdDouble gmx_simdcall rcp(SimdDouble x)
238 {
239 return { _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(x.simdInternal_))) };
240 }
241
maskAdd(SimdDouble a,SimdDouble b,SimdDBool m)242 static inline SimdDouble gmx_simdcall maskAdd(SimdDouble a, SimdDouble b, SimdDBool m)
243 {
244 return { _mm_add_pd(a.simdInternal_, _mm_and_pd(b.simdInternal_, m.simdInternal_)) };
245 }
246
maskzMul(SimdDouble a,SimdDouble b,SimdDBool m)247 static inline SimdDouble gmx_simdcall maskzMul(SimdDouble a, SimdDouble b, SimdDBool m)
248 {
249 return { _mm_and_pd(_mm_mul_pd(a.simdInternal_, b.simdInternal_), m.simdInternal_) };
250 }
251
maskzFma(SimdDouble a,SimdDouble b,SimdDouble c,SimdDBool m)252 static inline SimdDouble gmx_simdcall maskzFma(SimdDouble a, SimdDouble b, SimdDouble c, SimdDBool m)
253 {
254 return { _mm_and_pd(_mm_add_pd(_mm_mul_pd(a.simdInternal_, b.simdInternal_), c.simdInternal_),
255 m.simdInternal_) };
256 }
257
258 // Override for SSE4.1 and higher
259 #if GMX_SIMD_X86_SSE2
maskzRsqrt(SimdDouble x,SimdDBool m)260 static inline SimdDouble gmx_simdcall maskzRsqrt(SimdDouble x, SimdDBool m)
261 {
262 // The result will always be correct since we mask the result with m, but
263 // for debug builds we also want to make sure not to generate FP exceptions
264 # ifndef NDEBUG
265 x.simdInternal_ = _mm_or_pd(_mm_andnot_pd(m.simdInternal_, _mm_set1_pd(1.0)),
266 _mm_and_pd(m.simdInternal_, x.simdInternal_));
267 # endif
268 return { _mm_and_pd(_mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(x.simdInternal_))), m.simdInternal_) };
269 }
270
maskzRcp(SimdDouble x,SimdDBool m)271 static inline SimdDouble gmx_simdcall maskzRcp(SimdDouble x, SimdDBool m)
272 {
273 // The result will always be correct since we mask the result with m, but
274 // for debug builds we also want to make sure not to generate FP exceptions
275 # ifndef NDEBUG
276 x.simdInternal_ = _mm_or_pd(_mm_andnot_pd(m.simdInternal_, _mm_set1_pd(1.0)),
277 _mm_and_pd(m.simdInternal_, x.simdInternal_));
278 # endif
279 return { _mm_and_pd(_mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(x.simdInternal_))), m.simdInternal_) };
280 }
281 #endif
282
abs(SimdDouble x)283 static inline SimdDouble gmx_simdcall abs(SimdDouble x)
284 {
285 return { _mm_andnot_pd(_mm_set1_pd(GMX_DOUBLE_NEGZERO), x.simdInternal_) };
286 }
287
max(SimdDouble a,SimdDouble b)288 static inline SimdDouble gmx_simdcall max(SimdDouble a, SimdDouble b)
289 {
290 return { _mm_max_pd(a.simdInternal_, b.simdInternal_) };
291 }
292
min(SimdDouble a,SimdDouble b)293 static inline SimdDouble gmx_simdcall min(SimdDouble a, SimdDouble b)
294 {
295 return { _mm_min_pd(a.simdInternal_, b.simdInternal_) };
296 }
297
298 // Override for SSE4.1 and higher
299 #if GMX_SIMD_X86_SSE2
round(SimdDouble x)300 static inline SimdDouble gmx_simdcall round(SimdDouble x)
301 {
302 return { _mm_cvtepi32_pd(_mm_cvtpd_epi32(x.simdInternal_)) };
303 }
304
trunc(SimdDouble x)305 static inline SimdDouble gmx_simdcall trunc(SimdDouble x)
306 {
307 return { _mm_cvtepi32_pd(_mm_cvttpd_epi32(x.simdInternal_)) };
308 }
309
310 #endif
311
312 template<MathOptimization opt = MathOptimization::Safe>
frexp(SimdDouble value,SimdDInt32 * exponent)313 static inline SimdDouble frexp(SimdDouble value, SimdDInt32* exponent)
314 {
315 // Don't use _mm_set1_epi64x() - on MSVC it is only supported for 64-bit builds
316 const __m128d exponentMask =
317 _mm_castsi128_pd(_mm_set_epi32(0x7FF00000, 0x00000000, 0x7FF00000, 0x00000000));
318 const __m128d mantissaMask =
319 _mm_castsi128_pd(_mm_set_epi32(0x800FFFFF, 0xFFFFFFFF, 0x800FFFFF, 0xFFFFFFFF));
320 const __m128i exponentBias = _mm_set1_epi32(1022); // add 1 to make our definition identical to frexp()
321 const __m128d half = _mm_set1_pd(0.5);
322
323 __m128i iExponent = _mm_castpd_si128(_mm_and_pd(value.simdInternal_, exponentMask));
324 iExponent = _mm_sub_epi32(_mm_srli_epi64(iExponent, 52), exponentBias);
325
326 __m128d result = _mm_or_pd(_mm_and_pd(value.simdInternal_, mantissaMask), half);
327
328 if (opt == MathOptimization::Safe)
329 {
330 __m128d valueIsZero = _mm_cmpeq_pd(_mm_setzero_pd(), value.simdInternal_);
331 // Set the upper/lower 64-bit-fields of "iExponent" to 0-bits if the corresponding input value was +-0.0
332 iExponent = _mm_andnot_si128(_mm_castpd_si128(valueIsZero), iExponent);
333 // Set result to +-0 if the corresponding input value was +-0
334 result = _mm_or_pd(_mm_andnot_pd(valueIsZero, result),
335 _mm_and_pd(valueIsZero, value.simdInternal_));
336 }
337
338 // Shuffle exponent so that 32-bit-fields 0 & 1 contain the relevant exponent values to return
339 exponent->simdInternal_ = _mm_shuffle_epi32(iExponent, _MM_SHUFFLE(3, 1, 2, 0));
340
341 return { result };
342 }
343
344 // Override for SSE4.1
345 #if GMX_SIMD_X86_SSE2
346 template<MathOptimization opt = MathOptimization::Safe>
ldexp(SimdDouble value,SimdDInt32 exponent)347 static inline SimdDouble ldexp(SimdDouble value, SimdDInt32 exponent)
348 {
349 const __m128i exponentBias = _mm_set1_epi32(1023);
350 __m128i iExponent = _mm_add_epi32(exponent.simdInternal_, exponentBias);
351
352 if (opt == MathOptimization::Safe)
353 {
354 // Make sure biased argument is not negative
355 iExponent = _mm_and_si128(iExponent, _mm_cmpgt_epi32(iExponent, _mm_setzero_si128()));
356 }
357
358 // After conversion integers will be in slot 0,1. Move them to 0,2 so
359 // we can do a 64-bit shift and get them to the dp exponents.
360 iExponent = _mm_shuffle_epi32(iExponent, _MM_SHUFFLE(3, 1, 2, 0));
361 iExponent = _mm_slli_epi64(iExponent, 52);
362
363 return { _mm_mul_pd(value.simdInternal_, _mm_castsi128_pd(iExponent)) };
364 }
365 #endif
366
367 // Override for AVX-128-FMA and higher
368 #if GMX_SIMD_X86_SSE2 || GMX_SIMD_X86_SSE4_1
reduce(SimdDouble a)369 static inline double gmx_simdcall reduce(SimdDouble a)
370 {
371 __m128d b = _mm_add_sd(a.simdInternal_,
372 _mm_shuffle_pd(a.simdInternal_, a.simdInternal_, _MM_SHUFFLE2(1, 1)));
373 return *reinterpret_cast<double*>(&b);
374 }
375 #endif
376
377 static inline SimdDBool gmx_simdcall operator==(SimdDouble a, SimdDouble b)
378 {
379 return { _mm_cmpeq_pd(a.simdInternal_, b.simdInternal_) };
380 }
381
382 static inline SimdDBool gmx_simdcall operator!=(SimdDouble a, SimdDouble b)
383 {
384 return { _mm_cmpneq_pd(a.simdInternal_, b.simdInternal_) };
385 }
386
387 static inline SimdDBool gmx_simdcall operator<(SimdDouble a, SimdDouble b)
388 {
389 return { _mm_cmplt_pd(a.simdInternal_, b.simdInternal_) };
390 }
391
392 static inline SimdDBool gmx_simdcall operator<=(SimdDouble a, SimdDouble b)
393 {
394 return { _mm_cmple_pd(a.simdInternal_, b.simdInternal_) };
395 }
396
397 // Override for SSE4.1 and higher
398 #if GMX_SIMD_X86_SSE2
testBits(SimdDouble a)399 static inline SimdDBool gmx_simdcall testBits(SimdDouble a)
400 {
401 __m128i ia = _mm_castpd_si128(a.simdInternal_);
402 __m128i res = _mm_andnot_si128(_mm_cmpeq_epi32(ia, _mm_setzero_si128()), _mm_cmpeq_epi32(ia, ia));
403
404 // set each 64-bit element if low or high 32-bit part is set
405 res = _mm_or_si128(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(2, 3, 0, 1)));
406
407 return { _mm_castsi128_pd(res) };
408 }
409 #endif
410
411 static inline SimdDBool gmx_simdcall operator&&(SimdDBool a, SimdDBool b)
412 {
413 return { _mm_and_pd(a.simdInternal_, b.simdInternal_) };
414 }
415
416 static inline SimdDBool gmx_simdcall operator||(SimdDBool a, SimdDBool b)
417 {
418 return { _mm_or_pd(a.simdInternal_, b.simdInternal_) };
419 }
420
anyTrue(SimdDBool a)421 static inline bool gmx_simdcall anyTrue(SimdDBool a)
422 {
423 return _mm_movemask_pd(a.simdInternal_) != 0;
424 }
425
selectByMask(SimdDouble a,SimdDBool mask)426 static inline SimdDouble gmx_simdcall selectByMask(SimdDouble a, SimdDBool mask)
427 {
428 return { _mm_and_pd(a.simdInternal_, mask.simdInternal_) };
429 }
430
selectByNotMask(SimdDouble a,SimdDBool mask)431 static inline SimdDouble gmx_simdcall selectByNotMask(SimdDouble a, SimdDBool mask)
432 {
433 return { _mm_andnot_pd(mask.simdInternal_, a.simdInternal_) };
434 }
435
436 // Override for SSE4.1 and higher
437 #if GMX_SIMD_X86_SSE2
blend(SimdDouble a,SimdDouble b,SimdDBool sel)438 static inline SimdDouble gmx_simdcall blend(SimdDouble a, SimdDouble b, SimdDBool sel)
439 {
440 return { _mm_or_pd(_mm_andnot_pd(sel.simdInternal_, a.simdInternal_),
441 _mm_and_pd(sel.simdInternal_, b.simdInternal_)) };
442 }
443 #endif
444
445 static inline SimdDInt32 gmx_simdcall operator&(SimdDInt32 a, SimdDInt32 b)
446 {
447 return { _mm_and_si128(a.simdInternal_, b.simdInternal_) };
448 }
449
andNot(SimdDInt32 a,SimdDInt32 b)450 static inline SimdDInt32 gmx_simdcall andNot(SimdDInt32 a, SimdDInt32 b)
451 {
452 return { _mm_andnot_si128(a.simdInternal_, b.simdInternal_) };
453 }
454
455 static inline SimdDInt32 gmx_simdcall operator|(SimdDInt32 a, SimdDInt32 b)
456 {
457 return { _mm_or_si128(a.simdInternal_, b.simdInternal_) };
458 }
459
460 static inline SimdDInt32 gmx_simdcall operator^(SimdDInt32 a, SimdDInt32 b)
461 {
462 return { _mm_xor_si128(a.simdInternal_, b.simdInternal_) };
463 }
464
465 static inline SimdDInt32 gmx_simdcall operator+(SimdDInt32 a, SimdDInt32 b)
466 {
467 return { _mm_add_epi32(a.simdInternal_, b.simdInternal_) };
468 }
469
470 static inline SimdDInt32 gmx_simdcall operator-(SimdDInt32 a, SimdDInt32 b)
471 {
472 return { _mm_sub_epi32(a.simdInternal_, b.simdInternal_) };
473 }
474
475 // Override for SSE4.1 and higher
476 #if GMX_SIMD_X86_SSE2
477 static inline SimdDInt32 gmx_simdcall operator*(SimdDInt32 a, SimdDInt32 b)
478 {
479
480 __m128i tmpA = _mm_unpacklo_epi32(a.simdInternal_, _mm_setzero_si128()); // 0 a[1] 0 a[0]
481 __m128i tmpB = _mm_unpacklo_epi32(b.simdInternal_, _mm_setzero_si128()); // 0 b[1] 0 b[0]
482
483 __m128i tmpC = _mm_mul_epu32(tmpA, tmpB); // 0 a[1]*b[1] 0 a[0]*b[0]
484
485 return { _mm_shuffle_epi32(tmpC, _MM_SHUFFLE(3, 1, 2, 0)) };
486 }
487 #endif
488
489 static inline SimdDIBool gmx_simdcall operator==(SimdDInt32 a, SimdDInt32 b)
490 {
491 return { _mm_cmpeq_epi32(a.simdInternal_, b.simdInternal_) };
492 }
493
testBits(SimdDInt32 a)494 static inline SimdDIBool gmx_simdcall testBits(SimdDInt32 a)
495 {
496 __m128i x = a.simdInternal_;
497 __m128i res = _mm_andnot_si128(_mm_cmpeq_epi32(x, _mm_setzero_si128()), _mm_cmpeq_epi32(x, x));
498
499 return { res };
500 }
501
502 static inline SimdDIBool gmx_simdcall operator<(SimdDInt32 a, SimdDInt32 b)
503 {
504 return { _mm_cmplt_epi32(a.simdInternal_, b.simdInternal_) };
505 }
506
507 static inline SimdDIBool gmx_simdcall operator&&(SimdDIBool a, SimdDIBool b)
508 {
509 return { _mm_and_si128(a.simdInternal_, b.simdInternal_) };
510 }
511
512 static inline SimdDIBool gmx_simdcall operator||(SimdDIBool a, SimdDIBool b)
513 {
514 return { _mm_or_si128(a.simdInternal_, b.simdInternal_) };
515 }
516
anyTrue(SimdDIBool a)517 static inline bool gmx_simdcall anyTrue(SimdDIBool a)
518 {
519 return _mm_movemask_epi8(_mm_shuffle_epi32(a.simdInternal_, _MM_SHUFFLE(1, 0, 1, 0))) != 0;
520 }
521
selectByMask(SimdDInt32 a,SimdDIBool mask)522 static inline SimdDInt32 gmx_simdcall selectByMask(SimdDInt32 a, SimdDIBool mask)
523 {
524 return { _mm_and_si128(a.simdInternal_, mask.simdInternal_) };
525 }
526
selectByNotMask(SimdDInt32 a,SimdDIBool mask)527 static inline SimdDInt32 gmx_simdcall selectByNotMask(SimdDInt32 a, SimdDIBool mask)
528 {
529 return { _mm_andnot_si128(mask.simdInternal_, a.simdInternal_) };
530 }
531
532 // Override for SSE4.1 and higher
533 #if GMX_SIMD_X86_SSE2
blend(SimdDInt32 a,SimdDInt32 b,SimdDIBool sel)534 static inline SimdDInt32 gmx_simdcall blend(SimdDInt32 a, SimdDInt32 b, SimdDIBool sel)
535 {
536 return { _mm_or_si128(_mm_andnot_si128(sel.simdInternal_, a.simdInternal_),
537 _mm_and_si128(sel.simdInternal_, b.simdInternal_)) };
538 }
539 #endif
540
cvtR2I(SimdDouble a)541 static inline SimdDInt32 gmx_simdcall cvtR2I(SimdDouble a)
542 {
543 return { _mm_cvtpd_epi32(a.simdInternal_) };
544 }
545
cvttR2I(SimdDouble a)546 static inline SimdDInt32 gmx_simdcall cvttR2I(SimdDouble a)
547 {
548 return { _mm_cvttpd_epi32(a.simdInternal_) };
549 }
550
cvtI2R(SimdDInt32 a)551 static inline SimdDouble gmx_simdcall cvtI2R(SimdDInt32 a)
552 {
553 return { _mm_cvtepi32_pd(a.simdInternal_) };
554 }
555
cvtB2IB(SimdDBool a)556 static inline SimdDIBool gmx_simdcall cvtB2IB(SimdDBool a)
557 {
558 return { _mm_shuffle_epi32(_mm_castpd_si128(a.simdInternal_), _MM_SHUFFLE(2, 0, 2, 0)) };
559 }
560
cvtIB2B(SimdDIBool a)561 static inline SimdDBool gmx_simdcall cvtIB2B(SimdDIBool a)
562 {
563 return { _mm_castsi128_pd(_mm_shuffle_epi32(a.simdInternal_, _MM_SHUFFLE(1, 1, 0, 0))) };
564 }
565
cvtF2DD(SimdFloat f,SimdDouble * d0,SimdDouble * d1)566 static inline void gmx_simdcall cvtF2DD(SimdFloat f, SimdDouble* d0, SimdDouble* d1)
567 {
568 d0->simdInternal_ = _mm_cvtps_pd(f.simdInternal_);
569 d1->simdInternal_ = _mm_cvtps_pd(_mm_movehl_ps(f.simdInternal_, f.simdInternal_));
570 }
571
cvtDD2F(SimdDouble d0,SimdDouble d1)572 static inline SimdFloat gmx_simdcall cvtDD2F(SimdDouble d0, SimdDouble d1)
573 {
574 return { _mm_movelh_ps(_mm_cvtpd_ps(d0.simdInternal_), _mm_cvtpd_ps(d1.simdInternal_)) };
575 }
576
577 } // namespace gmx
578
579 #endif // GMX_SIMD_IMPL_X86_SSE2_SIMD_DOUBLE_H
580