1 /*===--------- avx512vlbf16intrin.h - AVX512_BF16 intrinsics ---------------===
2  *
3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  * See https://llvm.org/LICENSE.txt for license information.
5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  *
7  *===-----------------------------------------------------------------------===
8  */
9 #ifndef __IMMINTRIN_H
10 #error "Never use <avx512vlbf16intrin.h> directly; include <immintrin.h> instead."
11 #endif
12 
13 #ifdef __SSE2__
14 
15 #ifndef __AVX512VLBF16INTRIN_H
16 #define __AVX512VLBF16INTRIN_H
17 
18 #define __DEFAULT_FN_ATTRS128                                                  \
19   __attribute__((__always_inline__, __nodebug__,                               \
20                  __target__("avx512vl,avx512bf16,no-evex512"),                 \
21                  __min_vector_width__(128)))
22 #define __DEFAULT_FN_ATTRS256                                                  \
23   __attribute__((__always_inline__, __nodebug__,                               \
24                  __target__("avx512vl,avx512bf16,no-evex512"),                 \
25                  __min_vector_width__(256)))
26 
27 /// Convert Two Packed Single Data to One Packed BF16 Data.
28 ///
29 /// \headerfile <x86intrin.h>
30 ///
31 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
32 ///
33 /// \param __A
34 ///    A 128-bit vector of [4 x float].
35 /// \param __B
36 ///    A 128-bit vector of [4 x float].
37 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
38 ///    conversion of __B, and higher 64 bits come from conversion of __A.
39 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
_mm_cvtne2ps_pbh(__m128 __A,__m128 __B)40 _mm_cvtne2ps_pbh(__m128 __A, __m128 __B) {
41   return (__m128bh)__builtin_ia32_cvtne2ps2bf16_128((__v4sf) __A,
42                                                     (__v4sf) __B);
43 }
44 
45 /// Convert Two Packed Single Data to One Packed BF16 Data.
46 ///
47 /// \headerfile <x86intrin.h>
48 ///
49 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
50 ///
51 /// \param __A
52 ///    A 128-bit vector of [4 x float].
53 /// \param __B
54 ///    A 128-bit vector of [4 x float].
55 /// \param __W
56 ///    A 128-bit vector of [8 x bfloat].
57 /// \param __U
58 ///    A 8-bit mask value specifying what is chosen for each element.
59 ///    A 1 means conversion of __A or __B. A 0 means element from __W.
60 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
61 ///    conversion of __B, and higher 64 bits come from conversion of __A.
62 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
_mm_mask_cvtne2ps_pbh(__m128bh __W,__mmask8 __U,__m128 __A,__m128 __B)63 _mm_mask_cvtne2ps_pbh(__m128bh __W, __mmask8 __U, __m128 __A, __m128 __B) {
64   return (__m128bh)__builtin_ia32_selectpbf_128((__mmask8)__U,
65                                              (__v8bf)_mm_cvtne2ps_pbh(__A, __B),
66                                              (__v8bf)__W);
67 }
68 
69 /// Convert Two Packed Single Data to One Packed BF16 Data.
70 ///
71 /// \headerfile <x86intrin.h>
72 ///
73 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
74 ///
75 /// \param __A
76 ///    A 128-bit vector of [4 x float].
77 /// \param __B
78 ///    A 128-bit vector of [4 x float].
79 /// \param __U
80 ///    A 8-bit mask value specifying what is chosen for each element.
81 ///    A 1 means conversion of __A or __B. A 0 means element is zero.
82 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
83 ///    conversion of __B, and higher 64 bits come from conversion of __A.
84 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
_mm_maskz_cvtne2ps_pbh(__mmask8 __U,__m128 __A,__m128 __B)85 _mm_maskz_cvtne2ps_pbh(__mmask8 __U, __m128 __A, __m128 __B) {
86   return (__m128bh)__builtin_ia32_selectpbf_128((__mmask8)__U,
87                                              (__v8bf)_mm_cvtne2ps_pbh(__A, __B),
88                                              (__v8bf)_mm_setzero_si128());
89 }
90 
91 /// Convert Two Packed Single Data to One Packed BF16 Data.
92 ///
93 /// \headerfile <x86intrin.h>
94 ///
95 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
96 ///
97 /// \param __A
98 ///    A 256-bit vector of [8 x float].
99 /// \param __B
100 ///    A 256-bit vector of [8 x float].
101 /// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
102 ///    conversion of __B, and higher 128 bits come from conversion of __A.
103 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
_mm256_cvtne2ps_pbh(__m256 __A,__m256 __B)104 _mm256_cvtne2ps_pbh(__m256 __A, __m256 __B) {
105   return (__m256bh)__builtin_ia32_cvtne2ps2bf16_256((__v8sf) __A,
106                                                     (__v8sf) __B);
107 }
108 
109 /// Convert Two Packed Single Data to One Packed BF16 Data.
110 ///
111 /// \headerfile <x86intrin.h>
112 ///
113 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
114 ///
115 /// \param __A
116 ///    A 256-bit vector of [8 x float].
117 /// \param __B
118 ///    A 256-bit vector of [8 x float].
119 /// \param __W
120 ///    A 256-bit vector of [16 x bfloat].
121 /// \param __U
122 ///    A 16-bit mask value specifying what is chosen for each element.
123 ///    A 1 means conversion of __A or __B. A 0 means element from __W.
124 /// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
125 ///    conversion of __B, and higher 128 bits come from conversion of __A.
126 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
_mm256_mask_cvtne2ps_pbh(__m256bh __W,__mmask16 __U,__m256 __A,__m256 __B)127 _mm256_mask_cvtne2ps_pbh(__m256bh __W, __mmask16 __U, __m256 __A, __m256 __B) {
128   return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U,
129                                          (__v16bf)_mm256_cvtne2ps_pbh(__A, __B),
130                                          (__v16bf)__W);
131 }
132 
133 /// Convert Two Packed Single Data to One Packed BF16 Data.
134 ///
135 /// \headerfile <x86intrin.h>
136 ///
137 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
138 ///
139 /// \param __A
140 ///    A 256-bit vector of [8 x float].
141 /// \param __B
142 ///    A 256-bit vector of [8 x float].
143 /// \param __U
144 ///    A 16-bit mask value specifying what is chosen for each element.
145 ///    A 1 means conversion of __A or __B. A 0 means element is zero.
146 /// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
147 ///    conversion of __B, and higher 128 bits come from conversion of __A.
148 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
_mm256_maskz_cvtne2ps_pbh(__mmask16 __U,__m256 __A,__m256 __B)149 _mm256_maskz_cvtne2ps_pbh(__mmask16 __U, __m256 __A, __m256 __B) {
150   return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U,
151                                          (__v16bf)_mm256_cvtne2ps_pbh(__A, __B),
152                                          (__v16bf)_mm256_setzero_si256());
153 }
154 
155 /// Convert Packed Single Data to Packed BF16 Data.
156 ///
157 /// \headerfile <x86intrin.h>
158 ///
159 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
160 ///
161 /// \param __A
162 ///    A 128-bit vector of [4 x float].
163 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
164 ///    conversion of __A, and higher 64 bits are 0.
165 #define _mm_cvtneps_pbh(A)                                                     \
166   ((__m128bh)__builtin_ia32_vcvtneps2bf16128((__v4sf)(A)))
167 
168 /// Convert Packed Single Data to Packed BF16 Data.
169 ///
170 /// \headerfile <x86intrin.h>
171 ///
172 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
173 ///
174 /// \param __A
175 ///    A 128-bit vector of [4 x float].
176 /// \param __W
177 ///    A 128-bit vector of [8 x bfloat].
178 /// \param __U
179 ///    A 4-bit mask value specifying what is chosen for each element.
180 ///    A 1 means conversion of __A. A 0 means element from __W.
181 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
182 ///    conversion of __A, and higher 64 bits are 0.
183 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
_mm_mask_cvtneps_pbh(__m128bh __W,__mmask8 __U,__m128 __A)184 _mm_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m128 __A) {
185   return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
186                                                         (__v8bf)__W,
187                                                         (__mmask8)__U);
188 }
189 
190 /// Convert Packed Single Data to Packed BF16 Data.
191 ///
192 /// \headerfile <x86intrin.h>
193 ///
194 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
195 ///
196 /// \param __A
197 ///    A 128-bit vector of [4 x float].
198 /// \param __U
199 ///    A 4-bit mask value specifying what is chosen for each element.
200 ///    A 1 means conversion of __A. A 0 means element is zero.
201 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
202 ///    conversion of __A, and higher 64 bits are 0.
203 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
_mm_maskz_cvtneps_pbh(__mmask8 __U,__m128 __A)204 _mm_maskz_cvtneps_pbh(__mmask8 __U, __m128 __A) {
205   return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
206                                                     (__v8bf)_mm_setzero_si128(),
207                                                     (__mmask8)__U);
208 }
209 
210 /// Convert Packed Single Data to Packed BF16 Data.
211 ///
212 /// \headerfile <x86intrin.h>
213 ///
214 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
215 ///
216 /// \param __A
217 ///    A 256-bit vector of [8 x float].
218 /// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
219 #define _mm256_cvtneps_pbh(A)                                                  \
220   ((__m128bh)__builtin_ia32_vcvtneps2bf16256((__v8sf)(A)))
221 
222 /// Convert Packed Single Data to Packed BF16 Data.
223 ///
224 /// \headerfile <x86intrin.h>
225 ///
226 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
227 ///
228 /// \param __A
229 ///    A 256-bit vector of [8 x float].
230 /// \param __W
231 ///    A 256-bit vector of [8 x bfloat].
232 /// \param __U
233 ///    A 8-bit mask value specifying what is chosen for each element.
234 ///    A 1 means conversion of __A. A 0 means element from __W.
235 /// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
236 static __inline__ __m128bh __DEFAULT_FN_ATTRS256
_mm256_mask_cvtneps_pbh(__m128bh __W,__mmask8 __U,__m256 __A)237 _mm256_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m256 __A) {
238   return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
239                                                         (__v8bf)__W,
240                                                         (__mmask8)__U);
241 }
242 
243 /// Convert Packed Single Data to Packed BF16 Data.
244 ///
245 /// \headerfile <x86intrin.h>
246 ///
247 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
248 ///
249 /// \param __A
250 ///    A 256-bit vector of [8 x float].
251 /// \param __U
252 ///    A 8-bit mask value specifying what is chosen for each element.
253 ///    A 1 means conversion of __A. A 0 means element is zero.
254 /// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
255 static __inline__ __m128bh __DEFAULT_FN_ATTRS256
_mm256_maskz_cvtneps_pbh(__mmask8 __U,__m256 __A)256 _mm256_maskz_cvtneps_pbh(__mmask8 __U, __m256 __A) {
257   return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
258                                                     (__v8bf)_mm_setzero_si128(),
259                                                     (__mmask8)__U);
260 }
261 
262 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
263 ///
264 /// \headerfile <x86intrin.h>
265 ///
266 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
267 ///
268 /// \param __A
269 ///    A 128-bit vector of [8 x bfloat].
270 /// \param __B
271 ///    A 128-bit vector of [8 x bfloat].
272 /// \param __D
273 ///    A 128-bit vector of [4 x float].
274 /// \returns A 128-bit vector of [4 x float] comes from  Dot Product of
275 ///  __A, __B and __D
276 static __inline__ __m128 __DEFAULT_FN_ATTRS128
_mm_dpbf16_ps(__m128 __D,__m128bh __A,__m128bh __B)277 _mm_dpbf16_ps(__m128 __D, __m128bh __A, __m128bh __B) {
278   return (__m128)__builtin_ia32_dpbf16ps_128((__v4sf)__D,
279                                              (__v8bf)__A,
280                                              (__v8bf)__B);
281 }
282 
283 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
284 ///
285 /// \headerfile <x86intrin.h>
286 ///
287 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
288 ///
289 /// \param __A
290 ///    A 128-bit vector of [8 x bfloat].
291 /// \param __B
292 ///    A 128-bit vector of [8 x bfloat].
293 /// \param __D
294 ///    A 128-bit vector of [4 x float].
295 /// \param __U
296 ///    A 8-bit mask value specifying what is chosen for each element.
297 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
298 /// \returns A 128-bit vector of [4 x float] comes from  Dot Product of
299 ///  __A, __B and __D
300 static __inline__ __m128 __DEFAULT_FN_ATTRS128
_mm_mask_dpbf16_ps(__m128 __D,__mmask8 __U,__m128bh __A,__m128bh __B)301 _mm_mask_dpbf16_ps(__m128 __D, __mmask8 __U, __m128bh __A, __m128bh __B) {
302   return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
303                                            (__v4sf)_mm_dpbf16_ps(__D, __A, __B),
304                                            (__v4sf)__D);
305 }
306 
307 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
308 ///
309 /// \headerfile <x86intrin.h>
310 ///
311 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
312 ///
313 /// \param __A
314 ///    A 128-bit vector of [8 x bfloat].
315 /// \param __B
316 ///    A 128-bit vector of [8 x bfloat].
317 /// \param __D
318 ///    A 128-bit vector of [4 x float].
319 /// \param __U
320 ///    A 8-bit mask value specifying what is chosen for each element.
321 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
322 /// \returns A 128-bit vector of [4 x float] comes from  Dot Product of
323 ///  __A, __B and __D
324 static __inline__ __m128 __DEFAULT_FN_ATTRS128
_mm_maskz_dpbf16_ps(__mmask8 __U,__m128 __D,__m128bh __A,__m128bh __B)325 _mm_maskz_dpbf16_ps(__mmask8 __U, __m128 __D, __m128bh __A, __m128bh __B) {
326   return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
327                                            (__v4sf)_mm_dpbf16_ps(__D, __A, __B),
328                                            (__v4sf)_mm_setzero_si128());
329 }
330 
331 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
332 ///
333 /// \headerfile <x86intrin.h>
334 ///
335 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
336 ///
337 /// \param __A
338 ///    A 256-bit vector of [16 x bfloat].
339 /// \param __B
340 ///    A 256-bit vector of [16 x bfloat].
341 /// \param __D
342 ///    A 256-bit vector of [8 x float].
343 /// \returns A 256-bit vector of [8 x float] comes from  Dot Product of
344 ///  __A, __B and __D
345 static __inline__ __m256 __DEFAULT_FN_ATTRS256
_mm256_dpbf16_ps(__m256 __D,__m256bh __A,__m256bh __B)346 _mm256_dpbf16_ps(__m256 __D, __m256bh __A, __m256bh __B) {
347   return (__m256)__builtin_ia32_dpbf16ps_256((__v8sf)__D,
348                                              (__v16bf)__A,
349                                              (__v16bf)__B);
350 }
351 
352 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
353 ///
354 /// \headerfile <x86intrin.h>
355 ///
356 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
357 ///
358 /// \param __A
359 ///    A 256-bit vector of [16 x bfloat].
360 /// \param __B
361 ///    A 256-bit vector of [16 x bfloat].
362 /// \param __D
363 ///    A 256-bit vector of [8 x float].
364 /// \param __U
365 ///    A 16-bit mask value specifying what is chosen for each element.
366 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
367 /// \returns A 256-bit vector of [8 x float] comes from  Dot Product of
368 ///  __A, __B and __D
369 static __inline__ __m256 __DEFAULT_FN_ATTRS256
_mm256_mask_dpbf16_ps(__m256 __D,__mmask8 __U,__m256bh __A,__m256bh __B)370 _mm256_mask_dpbf16_ps(__m256 __D, __mmask8 __U, __m256bh __A, __m256bh __B) {
371   return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
372                                         (__v8sf)_mm256_dpbf16_ps(__D, __A, __B),
373                                         (__v8sf)__D);
374 }
375 
376 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
377 ///
378 /// \headerfile <x86intrin.h>
379 ///
380 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
381 ///
382 /// \param __A
383 ///    A 256-bit vector of [16 x bfloat].
384 /// \param __B
385 ///    A 256-bit vector of [16 x bfloat].
386 /// \param __D
387 ///    A 256-bit vector of [8 x float].
388 /// \param __U
389 ///    A 8-bit mask value specifying what is chosen for each element.
390 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
391 /// \returns A 256-bit vector of [8 x float] comes from  Dot Product of
392 ///  __A, __B and __D
393 static __inline__ __m256 __DEFAULT_FN_ATTRS256
_mm256_maskz_dpbf16_ps(__mmask8 __U,__m256 __D,__m256bh __A,__m256bh __B)394 _mm256_maskz_dpbf16_ps(__mmask8 __U, __m256 __D, __m256bh __A, __m256bh __B) {
395   return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
396                                         (__v8sf)_mm256_dpbf16_ps(__D, __A, __B),
397                                         (__v8sf)_mm256_setzero_si256());
398 }
399 
400 /// Convert One Single float Data to One BF16 Data.
401 ///
402 /// \headerfile <x86intrin.h>
403 ///
404 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
405 ///
406 /// \param __A
407 ///    A float data.
408 /// \returns A bf16 data whose sign field and exponent field keep unchanged,
409 ///    and fraction field is truncated to 7 bits.
_mm_cvtness_sbh(float __A)410 static __inline__ __bf16 __DEFAULT_FN_ATTRS128 _mm_cvtness_sbh(float __A) {
411   __v4sf __V = {__A, 0, 0, 0};
412   __v8bf __R = __builtin_ia32_cvtneps2bf16_128_mask(
413       (__v4sf)__V, (__v8bf)_mm_undefined_si128(), (__mmask8)-1);
414   return (__bf16)__R[0];
415 }
416 
417 /// Convert Packed BF16 Data to Packed float Data.
418 ///
419 /// \headerfile <x86intrin.h>
420 ///
421 /// \param __A
422 ///    A 128-bit vector of [4 x bfloat].
423 /// \returns A 128-bit vector of [4 x float] come from conversion of __A
_mm_cvtpbh_ps(__m128bh __A)424 static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_cvtpbh_ps(__m128bh __A) {
425   return _mm_castsi128_ps(
426       (__m128i)_mm_slli_epi32((__m128i)_mm_cvtepi16_epi32((__m128i)__A), 16));
427 }
428 
429 /// Convert Packed BF16 Data to Packed float Data.
430 ///
431 /// \headerfile <x86intrin.h>
432 ///
433 /// \param __A
434 ///    A 128-bit vector of [8 x bfloat].
435 /// \returns A 256-bit vector of [8 x float] come from conversion of __A
_mm256_cvtpbh_ps(__m128bh __A)436 static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_cvtpbh_ps(__m128bh __A) {
437   return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
438       (__m256i)_mm256_cvtepi16_epi32((__m128i)__A), 16));
439 }
440 
441 /// Convert Packed BF16 Data to Packed float Data using zeroing mask.
442 ///
443 /// \headerfile <x86intrin.h>
444 ///
445 /// \param __U
446 ///    A 4-bit mask. Elements are zeroed out when the corresponding mask
447 ///    bit is not set.
448 /// \param __A
449 ///    A 128-bit vector of [4 x bfloat].
450 /// \returns A 128-bit vector of [4 x float] come from conversion of __A
451 static __inline__ __m128 __DEFAULT_FN_ATTRS128
_mm_maskz_cvtpbh_ps(__mmask8 __U,__m128bh __A)452 _mm_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
453   return _mm_castsi128_ps((__m128i)_mm_slli_epi32(
454       (__m128i)_mm_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
455 }
456 
457 /// Convert Packed BF16 Data to Packed float Data using zeroing mask.
458 ///
459 /// \headerfile <x86intrin.h>
460 ///
461 /// \param __U
462 ///    A 8-bit mask. Elements are zeroed out when the corresponding mask
463 ///    bit is not set.
464 /// \param __A
465 ///    A 128-bit vector of [8 x bfloat].
466 /// \returns A 256-bit vector of [8 x float] come from conversion of __A
467 static __inline__ __m256 __DEFAULT_FN_ATTRS256
_mm256_maskz_cvtpbh_ps(__mmask8 __U,__m128bh __A)468 _mm256_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
469   return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
470       (__m256i)_mm256_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
471 }
472 
473 /// Convert Packed BF16 Data to Packed float Data using merging mask.
474 ///
475 /// \headerfile <x86intrin.h>
476 ///
477 /// \param __S
478 ///    A 128-bit vector of [4 x float]. Elements are copied from __S when
479 ///     the corresponding mask bit is not set.
480 /// \param __U
481 ///    A 4-bit mask. Elements are zeroed out when the corresponding mask
482 ///    bit is not set.
483 /// \param __A
484 ///    A 128-bit vector of [4 x bfloat].
485 /// \returns A 128-bit vector of [4 x float] come from conversion of __A
486 static __inline__ __m128 __DEFAULT_FN_ATTRS128
_mm_mask_cvtpbh_ps(__m128 __S,__mmask8 __U,__m128bh __A)487 _mm_mask_cvtpbh_ps(__m128 __S, __mmask8 __U, __m128bh __A) {
488   return _mm_castsi128_ps((__m128i)_mm_mask_slli_epi32(
489       (__m128i)__S, (__mmask8)__U, (__m128i)_mm_cvtepi16_epi32((__m128i)__A),
490       16));
491 }
492 
493 /// Convert Packed BF16 Data to Packed float Data using merging mask.
494 ///
495 /// \headerfile <x86intrin.h>
496 ///
497 /// \param __S
498 ///    A 256-bit vector of [8 x float]. Elements are copied from __S when
499 ///     the corresponding mask bit is not set.
500 /// \param __U
501 ///    A 8-bit mask. Elements are zeroed out when the corresponding mask
502 ///    bit is not set.
503 /// \param __A
504 ///    A 128-bit vector of [8 x bfloat].
505 /// \returns A 256-bit vector of [8 x float] come from conversion of __A
506 static __inline__ __m256 __DEFAULT_FN_ATTRS256
_mm256_mask_cvtpbh_ps(__m256 __S,__mmask8 __U,__m128bh __A)507 _mm256_mask_cvtpbh_ps(__m256 __S, __mmask8 __U, __m128bh __A) {
508   return _mm256_castsi256_ps((__m256i)_mm256_mask_slli_epi32(
509       (__m256i)__S, (__mmask8)__U, (__m256i)_mm256_cvtepi16_epi32((__m128i)__A),
510       16));
511 }
512 
513 #undef __DEFAULT_FN_ATTRS128
514 #undef __DEFAULT_FN_ATTRS256
515 
516 #endif
517 #endif
518