1 /*===------------ avx512bf16intrin.h - AVX512_BF16 intrinsics --------------===
2  *
3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  * See https://llvm.org/LICENSE.txt for license information.
5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  *
7  *===-----------------------------------------------------------------------===
8  */
9 #ifndef __IMMINTRIN_H
10 #error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead."
11 #endif
12 
13 #ifndef __AVX512BF16INTRIN_H
14 #define __AVX512BF16INTRIN_H
15 
16 typedef short __m512bh __attribute__((__vector_size__(64), __aligned__(64)));
17 typedef short __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
18 typedef unsigned short __bfloat16;
19 
20 #define __DEFAULT_FN_ATTRS512 \
21   __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \
22                  __min_vector_width__(512)))
23 #define __DEFAULT_FN_ATTRS                                                     \
24   __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))
25 
26 /// Convert One BF16 Data to One Single Float Data.
27 ///
28 /// \headerfile <x86intrin.h>
29 ///
30 /// This intrinsic does not correspond to a specific instruction.
31 ///
32 /// \param __A
33 ///    A bfloat data.
34 /// \returns A float data whose sign field and exponent field keep unchanged,
35 ///    and fraction field is extended to 23 bits.
36 static __inline__ float __DEFAULT_FN_ATTRS _mm_cvtsbh_ss(__bfloat16 __A) {
37   return __builtin_ia32_cvtsbf162ss_32(__A);
38 }
39 
40 /// Convert Two Packed Single Data to One Packed BF16 Data.
41 ///
42 /// \headerfile <x86intrin.h>
43 ///
44 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
45 ///
46 /// \param __A
47 ///    A 512-bit vector of [16 x float].
48 /// \param __B
49 ///    A 512-bit vector of [16 x float].
50 /// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
51 ///    conversion of __B, and higher 256 bits come from conversion of __A.
52 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
53 _mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) {
54   return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A,
55                                                     (__v16sf) __B);
56 }
57 
58 /// Convert Two Packed Single Data to One Packed BF16 Data.
59 ///
60 /// \headerfile <x86intrin.h>
61 ///
62 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
63 ///
64 /// \param __A
65 ///    A 512-bit vector of [16 x float].
66 /// \param __B
67 ///    A 512-bit vector of [16 x float].
68 /// \param __W
69 ///    A 512-bit vector of [32 x bfloat].
70 /// \param __U
71 ///    A 32-bit mask value specifying what is chosen for each element.
72 ///    A 1 means conversion of __A or __B. A 0 means element from __W.
73 /// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
74 ///    conversion of __B, and higher 256 bits come from conversion of __A.
75 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
76 _mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) {
77   return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U,
78                                         (__v32hi)_mm512_cvtne2ps_pbh(__A, __B),
79                                         (__v32hi)__W);
80 }
81 
82 /// Convert Two Packed Single Data to One Packed BF16 Data.
83 ///
84 /// \headerfile <x86intrin.h>
85 ///
86 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
87 ///
88 /// \param __A
89 ///    A 512-bit vector of [16 x float].
90 /// \param __B
91 ///    A 512-bit vector of [16 x float].
92 /// \param __U
93 ///    A 32-bit mask value specifying what is chosen for each element.
94 ///    A 1 means conversion of __A or __B. A 0 means element is zero.
95 /// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
96 ///    conversion of __B, and higher 256 bits come from conversion of __A.
97 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
98 _mm512_maskz_cvtne2ps_pbh(__mmask32 __U, __m512 __A, __m512 __B) {
99   return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U,
100                                         (__v32hi)_mm512_cvtne2ps_pbh(__A, __B),
101                                         (__v32hi)_mm512_setzero_si512());
102 }
103 
104 /// Convert Packed Single Data to Packed BF16 Data.
105 ///
106 /// \headerfile <x86intrin.h>
107 ///
108 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
109 ///
110 /// \param __A
111 ///    A 512-bit vector of [16 x float].
112 /// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
113 static __inline__ __m256bh __DEFAULT_FN_ATTRS512
114 _mm512_cvtneps_pbh(__m512 __A) {
115   return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
116                                               (__v16hi)_mm256_undefined_si256(),
117                                               (__mmask16)-1);
118 }
119 
120 /// Convert Packed Single Data to Packed BF16 Data.
121 ///
122 /// \headerfile <x86intrin.h>
123 ///
124 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
125 ///
126 /// \param __A
127 ///    A 512-bit vector of [16 x float].
128 /// \param __W
129 ///    A 256-bit vector of [16 x bfloat].
130 /// \param __U
131 ///    A 16-bit mask value specifying what is chosen for each element.
132 ///    A 1 means conversion of __A. A 0 means element from __W.
133 /// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
134 static __inline__ __m256bh __DEFAULT_FN_ATTRS512
135 _mm512_mask_cvtneps_pbh(__m256bh __W, __mmask16 __U, __m512 __A) {
136   return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
137                                                         (__v16hi)__W,
138                                                         (__mmask16)__U);
139 }
140 
141 /// Convert Packed Single Data to Packed BF16 Data.
142 ///
143 /// \headerfile <x86intrin.h>
144 ///
145 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
146 ///
147 /// \param __A
148 ///    A 512-bit vector of [16 x float].
149 /// \param __U
150 ///    A 16-bit mask value specifying what is chosen for each element.
151 ///    A 1 means conversion of __A. A 0 means element is zero.
152 /// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
153 static __inline__ __m256bh __DEFAULT_FN_ATTRS512
154 _mm512_maskz_cvtneps_pbh(__mmask16 __U, __m512 __A) {
155   return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
156                                                 (__v16hi)_mm256_setzero_si256(),
157                                                 (__mmask16)__U);
158 }
159 
160 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
161 ///
162 /// \headerfile <x86intrin.h>
163 ///
164 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
165 ///
166 /// \param __A
167 ///    A 512-bit vector of [32 x bfloat].
168 /// \param __B
169 ///    A 512-bit vector of [32 x bfloat].
170 /// \param __D
171 ///    A 512-bit vector of [16 x float].
172 /// \returns A 512-bit vector of [16 x float] comes from  Dot Product of
173 ///  __A, __B and __D
174 static __inline__ __m512 __DEFAULT_FN_ATTRS512
175 _mm512_dpbf16_ps(__m512 __D, __m512bh __A, __m512bh __B) {
176   return (__m512)__builtin_ia32_dpbf16ps_512((__v16sf) __D,
177                                              (__v16si) __A,
178                                              (__v16si) __B);
179 }
180 
181 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
182 ///
183 /// \headerfile <x86intrin.h>
184 ///
185 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
186 ///
187 /// \param __A
188 ///    A 512-bit vector of [32 x bfloat].
189 /// \param __B
190 ///    A 512-bit vector of [32 x bfloat].
191 /// \param __D
192 ///    A 512-bit vector of [16 x float].
193 /// \param __U
194 ///    A 16-bit mask value specifying what is chosen for each element.
195 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
196 /// \returns A 512-bit vector of [16 x float] comes from  Dot Product of
197 ///  __A, __B and __D
198 static __inline__ __m512 __DEFAULT_FN_ATTRS512
199 _mm512_mask_dpbf16_ps(__m512 __D, __mmask16 __U, __m512bh __A, __m512bh __B) {
200   return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
201                                        (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
202                                        (__v16sf)__D);
203 }
204 
205 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
206 ///
207 /// \headerfile <x86intrin.h>
208 ///
209 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
210 ///
211 /// \param __A
212 ///    A 512-bit vector of [32 x bfloat].
213 /// \param __B
214 ///    A 512-bit vector of [32 x bfloat].
215 /// \param __D
216 ///    A 512-bit vector of [16 x float].
217 /// \param __U
218 ///    A 16-bit mask value specifying what is chosen for each element.
219 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
220 /// \returns A 512-bit vector of [16 x float] comes from  Dot Product of
221 ///  __A, __B and __D
222 static __inline__ __m512 __DEFAULT_FN_ATTRS512
223 _mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {
224   return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
225                                        (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
226                                        (__v16sf)_mm512_setzero_si512());
227 }
228 
229 /// Convert Packed BF16 Data to Packed float Data.
230 ///
231 /// \headerfile <x86intrin.h>
232 ///
233 /// \param __A
234 ///    A 256-bit vector of [16 x bfloat].
235 /// \returns A 512-bit vector of [16 x float] come from conversion of __A
236 static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_cvtpbh_ps(__m256bh __A) {
237   return _mm512_castsi512_ps((__m512i)_mm512_slli_epi32(
238       (__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
239 }
240 
241 /// Convert Packed BF16 Data to Packed float Data using zeroing mask.
242 ///
243 /// \headerfile <x86intrin.h>
244 ///
245 /// \param __U
246 ///    A 16-bit mask. Elements are zeroed out when the corresponding mask
247 ///    bit is not set.
248 /// \param __A
249 ///    A 256-bit vector of [16 x bfloat].
250 /// \returns A 512-bit vector of [16 x float] come from conversion of __A
251 static __inline__ __m512 __DEFAULT_FN_ATTRS512
252 _mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {
253   return _mm512_castsi512_ps((__m512i)_mm512_slli_epi32(
254       (__m512i)_mm512_maskz_cvtepi16_epi32((__mmask16)__U, (__m256i)__A), 16));
255 }
256 
257 /// Convert Packed BF16 Data to Packed float Data using merging mask.
258 ///
259 /// \headerfile <x86intrin.h>
260 ///
261 /// \param __S
262 ///    A 512-bit vector of [16 x float]. Elements are copied from __S when
263 ///     the corresponding mask bit is not set.
264 /// \param __U
265 ///    A 16-bit mask.
266 /// \param __A
267 ///    A 256-bit vector of [16 x bfloat].
268 /// \returns A 512-bit vector of [16 x float] come from conversion of __A
269 static __inline__ __m512 __DEFAULT_FN_ATTRS512
270 _mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __U, __m256bh __A) {
271   return _mm512_castsi512_ps((__m512i)_mm512_mask_slli_epi32(
272       (__m512i)__S, (__mmask16)__U,
273       (__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
274 }
275 
276 #undef __DEFAULT_FN_ATTRS
277 #undef __DEFAULT_FN_ATTRS512
278 
279 #endif
280