1 /* Copyright (C) 1988-2021 Free Software Foundation, Inc.
2
3 This file is part of GCC.
4
5 GCC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3, or (at your option)
8 any later version.
9
10 GCC is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GCC; see the file COPYING3. If not see
17 <http://www.gnu.org/licenses/>. */
18
19 #define IN_TARGET_CODE 1
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "memmodel.h"
28 #include "gimple.h"
29 #include "cfghooks.h"
30 #include "cfgloop.h"
31 #include "df.h"
32 #include "tm_p.h"
33 #include "stringpool.h"
34 #include "expmed.h"
35 #include "optabs.h"
36 #include "regs.h"
37 #include "emit-rtl.h"
38 #include "recog.h"
39 #include "cgraph.h"
40 #include "diagnostic.h"
41 #include "cfgbuild.h"
42 #include "alias.h"
43 #include "fold-const.h"
44 #include "attribs.h"
45 #include "calls.h"
46 #include "stor-layout.h"
47 #include "varasm.h"
48 #include "output.h"
49 #include "insn-attr.h"
50 #include "flags.h"
51 #include "except.h"
52 #include "explow.h"
53 #include "expr.h"
54 #include "cfgrtl.h"
55 #include "common/common-target.h"
56 #include "langhooks.h"
57 #include "reload.h"
58 #include "gimplify.h"
59 #include "dwarf2.h"
60 #include "tm-constrs.h"
61 #include "cselib.h"
62 #include "sched-int.h"
63 #include "opts.h"
64 #include "tree-pass.h"
65 #include "context.h"
66 #include "pass_manager.h"
67 #include "target-globals.h"
68 #include "gimple-iterator.h"
69 #include "tree-vectorizer.h"
70 #include "shrink-wrap.h"
71 #include "builtins.h"
72 #include "rtl-iter.h"
73 #include "tree-iterator.h"
74 #include "dbgcnt.h"
75 #include "case-cfn-macros.h"
76 #include "dojump.h"
77 #include "fold-const-call.h"
78 #include "tree-vrp.h"
79 #include "tree-ssanames.h"
80 #include "selftest.h"
81 #include "selftest-rtl.h"
82 #include "print-rtl.h"
83 #include "intl.h"
84 #include "ifcvt.h"
85 #include "symbol-summary.h"
86 #include "ipa-prop.h"
87 #include "ipa-fnsummary.h"
88 #include "wide-int-bitmask.h"
89 #include "tree-vector-builder.h"
90 #include "debug.h"
91 #include "dwarf2out.h"
92 #include "i386-builtins.h"
93 #include "common/config/i386/i386-isas.h"
94
95 #undef BDESC
96 #undef BDESC_FIRST
97 #undef BDESC_END
98
99 /* Macros for verification of enum ix86_builtins order. */
100 #define BDESC_VERIFY(x, y, z) \
101 gcc_checking_assert ((x) == (enum ix86_builtins) ((y) + (z)))
102 #define BDESC_VERIFYS(x, y, z) \
103 STATIC_ASSERT ((x) == (enum ix86_builtins) ((y) + (z)))
104
105 BDESC_VERIFYS (IX86_BUILTIN__BDESC_PCMPESTR_FIRST,
106 IX86_BUILTIN__BDESC_COMI_LAST, 1);
107 BDESC_VERIFYS (IX86_BUILTIN__BDESC_PCMPISTR_FIRST,
108 IX86_BUILTIN__BDESC_PCMPESTR_LAST, 1);
109 BDESC_VERIFYS (IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST,
110 IX86_BUILTIN__BDESC_PCMPISTR_LAST, 1);
111 BDESC_VERIFYS (IX86_BUILTIN__BDESC_ARGS_FIRST,
112 IX86_BUILTIN__BDESC_SPECIAL_ARGS_LAST, 1);
113 BDESC_VERIFYS (IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST,
114 IX86_BUILTIN__BDESC_ARGS_LAST, 1);
115 BDESC_VERIFYS (IX86_BUILTIN__BDESC_MULTI_ARG_FIRST,
116 IX86_BUILTIN__BDESC_ROUND_ARGS_LAST, 1);
117 BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_FIRST,
118 IX86_BUILTIN__BDESC_MULTI_ARG_LAST, 1);
119 BDESC_VERIFYS (IX86_BUILTIN_MAX,
120 IX86_BUILTIN__BDESC_CET_LAST, 1);
121
122
123 /* Table for the ix86 builtin non-function types. */
124 static GTY(()) tree ix86_builtin_type_tab[(int) IX86_BT_LAST_CPTR + 1];
125
126 /* Retrieve an element from the above table, building some of
127 the types lazily. */
128
129 static tree
ix86_get_builtin_type(enum ix86_builtin_type tcode)130 ix86_get_builtin_type (enum ix86_builtin_type tcode)
131 {
132 unsigned int index;
133 tree type, itype;
134
135 gcc_assert ((unsigned)tcode < ARRAY_SIZE(ix86_builtin_type_tab));
136
137 type = ix86_builtin_type_tab[(int) tcode];
138 if (type != NULL)
139 return type;
140
141 gcc_assert (tcode > IX86_BT_LAST_PRIM);
142 if (tcode <= IX86_BT_LAST_VECT)
143 {
144 machine_mode mode;
145
146 index = tcode - IX86_BT_LAST_PRIM - 1;
147 itype = ix86_get_builtin_type (ix86_builtin_type_vect_base[index]);
148 mode = ix86_builtin_type_vect_mode[index];
149
150 type = build_vector_type_for_mode (itype, mode);
151 }
152 else
153 {
154 int quals;
155
156 index = tcode - IX86_BT_LAST_VECT - 1;
157 if (tcode <= IX86_BT_LAST_PTR)
158 quals = TYPE_UNQUALIFIED;
159 else
160 quals = TYPE_QUAL_CONST;
161
162 itype = ix86_get_builtin_type (ix86_builtin_type_ptr_base[index]);
163 if (quals != TYPE_UNQUALIFIED)
164 itype = build_qualified_type (itype, quals);
165
166 type = build_pointer_type (itype);
167 }
168
169 ix86_builtin_type_tab[(int) tcode] = type;
170 return type;
171 }
172
173 /* Table for the ix86 builtin function types. */
174 static GTY(()) tree ix86_builtin_func_type_tab[(int) IX86_BT_LAST_ALIAS + 1];
175
176 /* Retrieve an element from the above table, building some of
177 the types lazily. */
178
179 static tree
ix86_get_builtin_func_type(enum ix86_builtin_func_type tcode)180 ix86_get_builtin_func_type (enum ix86_builtin_func_type tcode)
181 {
182 tree type;
183
184 gcc_assert ((unsigned)tcode < ARRAY_SIZE (ix86_builtin_func_type_tab));
185
186 type = ix86_builtin_func_type_tab[(int) tcode];
187 if (type != NULL)
188 return type;
189
190 if (tcode <= IX86_BT_LAST_FUNC)
191 {
192 unsigned start = ix86_builtin_func_start[(int) tcode];
193 unsigned after = ix86_builtin_func_start[(int) tcode + 1];
194 tree rtype, atype, args = void_list_node;
195 unsigned i;
196
197 rtype = ix86_get_builtin_type (ix86_builtin_func_args[start]);
198 for (i = after - 1; i > start; --i)
199 {
200 atype = ix86_get_builtin_type (ix86_builtin_func_args[i]);
201 args = tree_cons (NULL, atype, args);
202 }
203
204 type = build_function_type (rtype, args);
205 }
206 else
207 {
208 unsigned index = tcode - IX86_BT_LAST_FUNC - 1;
209 enum ix86_builtin_func_type icode;
210
211 icode = ix86_builtin_func_alias_base[index];
212 type = ix86_get_builtin_func_type (icode);
213 }
214
215 ix86_builtin_func_type_tab[(int) tcode] = type;
216 return type;
217 }
218
219 /* Table for the ix86 builtin decls. */
220 static GTY(()) tree ix86_builtins[(int) IX86_BUILTIN_MAX];
221
222 struct builtin_isa ix86_builtins_isa[(int) IX86_BUILTIN_MAX];
223
get_ix86_builtin(enum ix86_builtins c)224 tree get_ix86_builtin (enum ix86_builtins c)
225 {
226 return ix86_builtins[c];
227 }
228
229 /* Bits that can still enable any inclusion of a builtin. */
230 HOST_WIDE_INT deferred_isa_values = 0;
231 HOST_WIDE_INT deferred_isa_values2 = 0;
232
233 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the
234 MASK and MASK2 of which isa_flags and ix86_isa_flags2 to use in the
235 ix86_builtins_isa array. Stores the function decl in the ix86_builtins
236 array. Returns the function decl or NULL_TREE, if the builtin was not
237 added.
238
239 If the front end has a special hook for builtin functions, delay adding
240 builtin functions that aren't in the current ISA until the ISA is changed
241 with function specific optimization. Doing so, can save about 300K for the
242 default compiler. When the builtin is expanded, check at that time whether
243 it is valid.
244
245 If the front end doesn't have a special hook, record all builtins, even if
246 it isn't an instruction set in the current ISA in case the user uses
247 function specific options for a different ISA, so that we don't get scope
248 errors if a builtin is added in the middle of a function scope. */
249
250 static inline tree
def_builtin(HOST_WIDE_INT mask,HOST_WIDE_INT mask2,const char * name,enum ix86_builtin_func_type tcode,enum ix86_builtins code)251 def_builtin (HOST_WIDE_INT mask, HOST_WIDE_INT mask2,
252 const char *name,
253 enum ix86_builtin_func_type tcode,
254 enum ix86_builtins code)
255 {
256 tree decl = NULL_TREE;
257
258 /* An instruction may be 64bit only regardless of ISAs. */
259 if (!(mask & OPTION_MASK_ISA_64BIT) || TARGET_64BIT)
260 {
261 ix86_builtins_isa[(int) code].isa = mask;
262 ix86_builtins_isa[(int) code].isa2 = mask2;
263
264 mask &= ~OPTION_MASK_ISA_64BIT;
265
266 /* Filter out the masks most often ored together with others. */
267 if ((mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512VL)
268 && mask != OPTION_MASK_ISA_AVX512VL)
269 mask &= ~OPTION_MASK_ISA_AVX512VL;
270 if ((mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512BW)
271 && mask != OPTION_MASK_ISA_AVX512BW)
272 mask &= ~OPTION_MASK_ISA_AVX512BW;
273
274 if (((mask2 == 0 || (mask2 & ix86_isa_flags2) != 0)
275 && (mask == 0 || (mask & ix86_isa_flags) != 0))
276 || ((mask & OPTION_MASK_ISA_MMX) != 0 && TARGET_MMX_WITH_SSE)
277 /* "Unified" builtin used by either AVXVNNI intrinsics or AVX512VNNIVL
278 non-mask intrinsics should be defined whenever avxvnni
279 or avx512vnni && avx512vl exist. */
280 || (mask2 == OPTION_MASK_ISA2_AVXVNNI)
281 || (lang_hooks.builtin_function
282 == lang_hooks.builtin_function_ext_scope))
283 {
284 tree type = ix86_get_builtin_func_type (tcode);
285 decl = add_builtin_function (name, type, code, BUILT_IN_MD,
286 NULL, NULL_TREE);
287 ix86_builtins[(int) code] = decl;
288 ix86_builtins_isa[(int) code].set_and_not_built_p = false;
289 }
290 else
291 {
292 /* Just MASK and MASK2 where set_and_not_built_p == true can potentially
293 include a builtin. */
294 deferred_isa_values |= mask;
295 deferred_isa_values2 |= mask2;
296 ix86_builtins[(int) code] = NULL_TREE;
297 ix86_builtins_isa[(int) code].tcode = tcode;
298 ix86_builtins_isa[(int) code].name = name;
299 ix86_builtins_isa[(int) code].const_p = false;
300 ix86_builtins_isa[(int) code].pure_p = false;
301 ix86_builtins_isa[(int) code].set_and_not_built_p = true;
302 }
303 }
304
305 return decl;
306 }
307
308 /* Like def_builtin, but also marks the function decl "const". */
309
310 static inline tree
def_builtin_const(HOST_WIDE_INT mask,HOST_WIDE_INT mask2,const char * name,enum ix86_builtin_func_type tcode,enum ix86_builtins code)311 def_builtin_const (HOST_WIDE_INT mask, HOST_WIDE_INT mask2, const char *name,
312 enum ix86_builtin_func_type tcode, enum ix86_builtins code)
313 {
314 tree decl = def_builtin (mask, mask2, name, tcode, code);
315 if (decl)
316 TREE_READONLY (decl) = 1;
317 else
318 ix86_builtins_isa[(int) code].const_p = true;
319
320 return decl;
321 }
322
323 /* Like def_builtin, but also marks the function decl "pure". */
324
325 static inline tree
def_builtin_pure(HOST_WIDE_INT mask,HOST_WIDE_INT mask2,const char * name,enum ix86_builtin_func_type tcode,enum ix86_builtins code)326 def_builtin_pure (HOST_WIDE_INT mask, HOST_WIDE_INT mask2, const char *name,
327 enum ix86_builtin_func_type tcode, enum ix86_builtins code)
328 {
329 tree decl = def_builtin (mask, mask2, name, tcode, code);
330 if (decl)
331 DECL_PURE_P (decl) = 1;
332 else
333 ix86_builtins_isa[(int) code].pure_p = true;
334
335 return decl;
336 }
337
338 /* Add any new builtin functions for a given ISA that may not have been
339 declared. This saves a bit of space compared to adding all of the
340 declarations to the tree, even if we didn't use them. */
341
342 void
ix86_add_new_builtins(HOST_WIDE_INT isa,HOST_WIDE_INT isa2)343 ix86_add_new_builtins (HOST_WIDE_INT isa, HOST_WIDE_INT isa2)
344 {
345 isa &= ~OPTION_MASK_ISA_64BIT;
346
347 if ((isa & deferred_isa_values) == 0
348 && (isa2 & deferred_isa_values2) == 0
349 && ((deferred_isa_values & OPTION_MASK_ISA_MMX) == 0
350 || !(TARGET_64BIT && (isa & OPTION_MASK_ISA_SSE2) != 0)))
351 return;
352
353 /* Bits in ISA value can be removed from potential isa values. */
354 deferred_isa_values &= ~isa;
355 deferred_isa_values2 &= ~isa2;
356 if (TARGET_64BIT && (isa & OPTION_MASK_ISA_SSE2) != 0)
357 deferred_isa_values &= ~OPTION_MASK_ISA_MMX;
358
359 int i;
360 tree saved_current_target_pragma = current_target_pragma;
361 current_target_pragma = NULL_TREE;
362
363 for (i = 0; i < (int)IX86_BUILTIN_MAX; i++)
364 {
365 if (((ix86_builtins_isa[i].isa & isa) != 0
366 || (ix86_builtins_isa[i].isa2 & isa2) != 0
367 || ((ix86_builtins_isa[i].isa & OPTION_MASK_ISA_MMX) != 0
368 && TARGET_64BIT
369 && (isa & OPTION_MASK_ISA_SSE2) != 0))
370 && ix86_builtins_isa[i].set_and_not_built_p)
371 {
372 tree decl, type;
373
374 /* Don't define the builtin again. */
375 ix86_builtins_isa[i].set_and_not_built_p = false;
376
377 type = ix86_get_builtin_func_type (ix86_builtins_isa[i].tcode);
378 decl = add_builtin_function_ext_scope (ix86_builtins_isa[i].name,
379 type, i, BUILT_IN_MD, NULL,
380 NULL_TREE);
381
382 ix86_builtins[i] = decl;
383 if (ix86_builtins_isa[i].const_p)
384 TREE_READONLY (decl) = 1;
385 }
386 }
387
388 current_target_pragma = saved_current_target_pragma;
389 }
390
391 /* TM vector builtins. */
392
393 /* Reuse the existing x86-specific `struct builtin_description' cause
394 we're lazy. Add casts to make them fit. */
395 static const struct builtin_description bdesc_tm[] =
396 {
397 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
398 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
399 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
400 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
401 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
402 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
403 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
404
405 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
406 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
407 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
408 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
409 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
410 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
411 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
412
413 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
414 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
415 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
416 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
417 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
418 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
419 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
420
421 { OPTION_MASK_ISA_MMX, 0, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
422 { OPTION_MASK_ISA_SSE, 0, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
423 { OPTION_MASK_ISA_AVX, 0, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
424 };
425
426 /* Initialize the transactional memory vector load/store builtins. */
427
428 static void
ix86_init_tm_builtins(void)429 ix86_init_tm_builtins (void)
430 {
431 enum ix86_builtin_func_type ftype;
432 const struct builtin_description *d;
433 size_t i;
434 tree decl;
435 tree attrs_load, attrs_type_load, attrs_store, attrs_type_store;
436 tree attrs_log, attrs_type_log;
437
438 if (!flag_tm)
439 return;
440
441 /* If there are no builtins defined, we must be compiling in a
442 language without trans-mem support. */
443 if (!builtin_decl_explicit_p (BUILT_IN_TM_LOAD_1))
444 return;
445
446 /* Use whatever attributes a normal TM load has. */
447 decl = builtin_decl_explicit (BUILT_IN_TM_LOAD_1);
448 attrs_load = DECL_ATTRIBUTES (decl);
449 attrs_type_load = TYPE_ATTRIBUTES (TREE_TYPE (decl));
450 /* Use whatever attributes a normal TM store has. */
451 decl = builtin_decl_explicit (BUILT_IN_TM_STORE_1);
452 attrs_store = DECL_ATTRIBUTES (decl);
453 attrs_type_store = TYPE_ATTRIBUTES (TREE_TYPE (decl));
454 /* Use whatever attributes a normal TM log has. */
455 decl = builtin_decl_explicit (BUILT_IN_TM_LOG);
456 attrs_log = DECL_ATTRIBUTES (decl);
457 attrs_type_log = TYPE_ATTRIBUTES (TREE_TYPE (decl));
458
459 for (i = 0, d = bdesc_tm;
460 i < ARRAY_SIZE (bdesc_tm);
461 i++, d++)
462 {
463 if ((d->mask & ix86_isa_flags) != 0
464 || ((d->mask & OPTION_MASK_ISA_MMX) != 0 && TARGET_MMX_WITH_SSE)
465 || (lang_hooks.builtin_function
466 == lang_hooks.builtin_function_ext_scope))
467 {
468 tree type, attrs, attrs_type;
469 enum built_in_function code = (enum built_in_function) d->code;
470
471 ftype = (enum ix86_builtin_func_type) d->flag;
472 type = ix86_get_builtin_func_type (ftype);
473
474 if (BUILTIN_TM_LOAD_P (code))
475 {
476 attrs = attrs_load;
477 attrs_type = attrs_type_load;
478 }
479 else if (BUILTIN_TM_STORE_P (code))
480 {
481 attrs = attrs_store;
482 attrs_type = attrs_type_store;
483 }
484 else
485 {
486 attrs = attrs_log;
487 attrs_type = attrs_type_log;
488 }
489 decl = add_builtin_function (d->name, type, code, BUILT_IN_NORMAL,
490 /* The builtin without the prefix for
491 calling it directly. */
492 d->name + strlen ("__builtin_"),
493 attrs);
494 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
495 set the TYPE_ATTRIBUTES. */
496 decl_attributes (&TREE_TYPE (decl), attrs_type, ATTR_FLAG_BUILT_IN);
497
498 set_builtin_decl (code, decl, false);
499 }
500 }
501 }
502
503 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
504 in the current target ISA to allow the user to compile particular modules
505 with different target specific options that differ from the command line
506 options. */
507 static void
ix86_init_mmx_sse_builtins(void)508 ix86_init_mmx_sse_builtins (void)
509 {
510 const struct builtin_description * d;
511 enum ix86_builtin_func_type ftype;
512 size_t i;
513
514 /* Add all special builtins with variable number of operands. */
515 for (i = 0, d = bdesc_special_args;
516 i < ARRAY_SIZE (bdesc_special_args);
517 i++, d++)
518 {
519 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST, i);
520 if (d->name == 0)
521 continue;
522
523 ftype = (enum ix86_builtin_func_type) d->flag;
524 def_builtin (d->mask, d->mask2, d->name, ftype, d->code);
525 }
526 BDESC_VERIFYS (IX86_BUILTIN__BDESC_SPECIAL_ARGS_LAST,
527 IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST,
528 ARRAY_SIZE (bdesc_special_args) - 1);
529
530 /* Add all builtins with variable number of operands. */
531 for (i = 0, d = bdesc_args;
532 i < ARRAY_SIZE (bdesc_args);
533 i++, d++)
534 {
535 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_ARGS_FIRST, i);
536 if (d->name == 0)
537 continue;
538
539 ftype = (enum ix86_builtin_func_type) d->flag;
540 def_builtin_const (d->mask, d->mask2, d->name, ftype, d->code);
541 }
542 BDESC_VERIFYS (IX86_BUILTIN__BDESC_ARGS_LAST,
543 IX86_BUILTIN__BDESC_ARGS_FIRST,
544 ARRAY_SIZE (bdesc_args) - 1);
545
546 /* Add all builtins with rounding. */
547 for (i = 0, d = bdesc_round_args;
548 i < ARRAY_SIZE (bdesc_round_args);
549 i++, d++)
550 {
551 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST, i);
552 if (d->name == 0)
553 continue;
554
555 ftype = (enum ix86_builtin_func_type) d->flag;
556 def_builtin_const (d->mask, d->mask2, d->name, ftype, d->code);
557 }
558 BDESC_VERIFYS (IX86_BUILTIN__BDESC_ROUND_ARGS_LAST,
559 IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST,
560 ARRAY_SIZE (bdesc_round_args) - 1);
561
562 /* pcmpestr[im] insns. */
563 for (i = 0, d = bdesc_pcmpestr;
564 i < ARRAY_SIZE (bdesc_pcmpestr);
565 i++, d++)
566 {
567 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_PCMPESTR_FIRST, i);
568 if (d->code == IX86_BUILTIN_PCMPESTRM128)
569 ftype = V16QI_FTYPE_V16QI_INT_V16QI_INT_INT;
570 else
571 ftype = INT_FTYPE_V16QI_INT_V16QI_INT_INT;
572 def_builtin_const (d->mask, d->mask2, d->name, ftype, d->code);
573 }
574 BDESC_VERIFYS (IX86_BUILTIN__BDESC_PCMPESTR_LAST,
575 IX86_BUILTIN__BDESC_PCMPESTR_FIRST,
576 ARRAY_SIZE (bdesc_pcmpestr) - 1);
577
578 /* pcmpistr[im] insns. */
579 for (i = 0, d = bdesc_pcmpistr;
580 i < ARRAY_SIZE (bdesc_pcmpistr);
581 i++, d++)
582 {
583 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_PCMPISTR_FIRST, i);
584 if (d->code == IX86_BUILTIN_PCMPISTRM128)
585 ftype = V16QI_FTYPE_V16QI_V16QI_INT;
586 else
587 ftype = INT_FTYPE_V16QI_V16QI_INT;
588 def_builtin_const (d->mask, d->mask2, d->name, ftype, d->code);
589 }
590 BDESC_VERIFYS (IX86_BUILTIN__BDESC_PCMPISTR_LAST,
591 IX86_BUILTIN__BDESC_PCMPISTR_FIRST,
592 ARRAY_SIZE (bdesc_pcmpistr) - 1);
593
594 /* comi/ucomi insns. */
595 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
596 {
597 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_COMI_FIRST, i);
598 if (d->mask == OPTION_MASK_ISA_SSE2)
599 ftype = INT_FTYPE_V2DF_V2DF;
600 else
601 ftype = INT_FTYPE_V4SF_V4SF;
602 def_builtin_const (d->mask, d->mask2, d->name, ftype, d->code);
603 }
604 BDESC_VERIFYS (IX86_BUILTIN__BDESC_COMI_LAST,
605 IX86_BUILTIN__BDESC_COMI_FIRST,
606 ARRAY_SIZE (bdesc_comi) - 1);
607
608 /* SSE */
609 def_builtin (OPTION_MASK_ISA_SSE, 0, "__builtin_ia32_ldmxcsr",
610 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_LDMXCSR);
611 def_builtin_pure (OPTION_MASK_ISA_SSE, 0, "__builtin_ia32_stmxcsr",
612 UNSIGNED_FTYPE_VOID, IX86_BUILTIN_STMXCSR);
613
614 /* SSE or 3DNow!A */
615 def_builtin (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
616 /* As it uses V4HImode, we have to require -mmmx too. */
617 | OPTION_MASK_ISA_MMX, 0,
618 "__builtin_ia32_maskmovq", VOID_FTYPE_V8QI_V8QI_PCHAR,
619 IX86_BUILTIN_MASKMOVQ);
620
621 /* SSE2 */
622 def_builtin (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_maskmovdqu",
623 VOID_FTYPE_V16QI_V16QI_PCHAR, IX86_BUILTIN_MASKMOVDQU);
624
625 def_builtin (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_clflush",
626 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLFLUSH);
627 x86_mfence = def_builtin (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_mfence",
628 VOID_FTYPE_VOID, IX86_BUILTIN_MFENCE);
629
630 /* SSE3. */
631 def_builtin (0, OPTION_MASK_ISA2_MWAIT, "__builtin_ia32_monitor",
632 VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITOR);
633 def_builtin (0, OPTION_MASK_ISA2_MWAIT, "__builtin_ia32_mwait",
634 VOID_FTYPE_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAIT);
635
636 /* AES */
637 def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, 0,
638 "__builtin_ia32_aesenc128",
639 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENC128);
640 def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, 0,
641 "__builtin_ia32_aesenclast128",
642 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENCLAST128);
643 def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, 0,
644 "__builtin_ia32_aesdec128",
645 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDEC128);
646 def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, 0,
647 "__builtin_ia32_aesdeclast128",
648 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDECLAST128);
649 def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, 0,
650 "__builtin_ia32_aesimc128",
651 V2DI_FTYPE_V2DI, IX86_BUILTIN_AESIMC128);
652 def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, 0,
653 "__builtin_ia32_aeskeygenassist128",
654 V2DI_FTYPE_V2DI_INT, IX86_BUILTIN_AESKEYGENASSIST128);
655
656 /* PCLMUL */
657 def_builtin_const (OPTION_MASK_ISA_PCLMUL | OPTION_MASK_ISA_SSE2, 0,
658 "__builtin_ia32_pclmulqdq128",
659 V2DI_FTYPE_V2DI_V2DI_INT, IX86_BUILTIN_PCLMULQDQ128);
660
661 /* RDRND */
662 def_builtin (OPTION_MASK_ISA_RDRND, 0, "__builtin_ia32_rdrand16_step",
663 INT_FTYPE_PUSHORT, IX86_BUILTIN_RDRAND16_STEP);
664 def_builtin (OPTION_MASK_ISA_RDRND, 0, "__builtin_ia32_rdrand32_step",
665 INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDRAND32_STEP);
666 def_builtin (OPTION_MASK_ISA_RDRND | OPTION_MASK_ISA_64BIT, 0,
667 "__builtin_ia32_rdrand64_step", INT_FTYPE_PULONGLONG,
668 IX86_BUILTIN_RDRAND64_STEP);
669
670 /* AVX2 */
671 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv2df",
672 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
673 IX86_BUILTIN_GATHERSIV2DF);
674
675 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv4df",
676 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
677 IX86_BUILTIN_GATHERSIV4DF);
678
679 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv2df",
680 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
681 IX86_BUILTIN_GATHERDIV2DF);
682
683 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv4df",
684 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
685 IX86_BUILTIN_GATHERDIV4DF);
686
687 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv4sf",
688 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
689 IX86_BUILTIN_GATHERSIV4SF);
690
691 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv8sf",
692 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
693 IX86_BUILTIN_GATHERSIV8SF);
694
695 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv4sf",
696 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
697 IX86_BUILTIN_GATHERDIV4SF);
698
699 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv4sf256",
700 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
701 IX86_BUILTIN_GATHERDIV8SF);
702
703 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv2di",
704 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
705 IX86_BUILTIN_GATHERSIV2DI);
706
707 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv4di",
708 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
709 IX86_BUILTIN_GATHERSIV4DI);
710
711 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv2di",
712 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
713 IX86_BUILTIN_GATHERDIV2DI);
714
715 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv4di",
716 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
717 IX86_BUILTIN_GATHERDIV4DI);
718
719 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv4si",
720 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
721 IX86_BUILTIN_GATHERSIV4SI);
722
723 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gathersiv8si",
724 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
725 IX86_BUILTIN_GATHERSIV8SI);
726
727 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv4si",
728 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
729 IX86_BUILTIN_GATHERDIV4SI);
730
731 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatherdiv4si256",
732 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
733 IX86_BUILTIN_GATHERDIV8SI);
734
735 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatheraltsiv4df ",
736 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
737 IX86_BUILTIN_GATHERALTSIV4DF);
738
739 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatheraltdiv8sf ",
740 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
741 IX86_BUILTIN_GATHERALTDIV8SF);
742
743 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatheraltsiv4di ",
744 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
745 IX86_BUILTIN_GATHERALTSIV4DI);
746
747 def_builtin_pure (OPTION_MASK_ISA_AVX2, 0, "__builtin_ia32_gatheraltdiv8si ",
748 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
749 IX86_BUILTIN_GATHERALTDIV8SI);
750
751 /* AVX512F */
752 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gathersiv16sf",
753 V16SF_FTYPE_V16SF_PCVOID_V16SI_HI_INT,
754 IX86_BUILTIN_GATHER3SIV16SF);
755
756 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gathersiv8df",
757 V8DF_FTYPE_V8DF_PCVOID_V8SI_QI_INT,
758 IX86_BUILTIN_GATHER3SIV8DF);
759
760 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gatherdiv16sf",
761 V8SF_FTYPE_V8SF_PCVOID_V8DI_QI_INT,
762 IX86_BUILTIN_GATHER3DIV16SF);
763
764 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gatherdiv8df",
765 V8DF_FTYPE_V8DF_PCVOID_V8DI_QI_INT,
766 IX86_BUILTIN_GATHER3DIV8DF);
767
768 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gathersiv16si",
769 V16SI_FTYPE_V16SI_PCVOID_V16SI_HI_INT,
770 IX86_BUILTIN_GATHER3SIV16SI);
771
772 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gathersiv8di",
773 V8DI_FTYPE_V8DI_PCVOID_V8SI_QI_INT,
774 IX86_BUILTIN_GATHER3SIV8DI);
775
776 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gatherdiv16si",
777 V8SI_FTYPE_V8SI_PCVOID_V8DI_QI_INT,
778 IX86_BUILTIN_GATHER3DIV16SI);
779
780 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gatherdiv8di",
781 V8DI_FTYPE_V8DI_PCVOID_V8DI_QI_INT,
782 IX86_BUILTIN_GATHER3DIV8DI);
783
784 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gather3altsiv8df ",
785 V8DF_FTYPE_V8DF_PCDOUBLE_V16SI_QI_INT,
786 IX86_BUILTIN_GATHER3ALTSIV8DF);
787
788 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gather3altdiv16sf ",
789 V16SF_FTYPE_V16SF_PCFLOAT_V8DI_HI_INT,
790 IX86_BUILTIN_GATHER3ALTDIV16SF);
791
792 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gather3altsiv8di ",
793 V8DI_FTYPE_V8DI_PCINT64_V16SI_QI_INT,
794 IX86_BUILTIN_GATHER3ALTSIV8DI);
795
796 def_builtin_pure (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_gather3altdiv16si ",
797 V16SI_FTYPE_V16SI_PCINT_V8DI_HI_INT,
798 IX86_BUILTIN_GATHER3ALTDIV16SI);
799
800 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scattersiv16sf",
801 VOID_FTYPE_PVOID_HI_V16SI_V16SF_INT,
802 IX86_BUILTIN_SCATTERSIV16SF);
803
804 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scattersiv8df",
805 VOID_FTYPE_PVOID_QI_V8SI_V8DF_INT,
806 IX86_BUILTIN_SCATTERSIV8DF);
807
808 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatterdiv16sf",
809 VOID_FTYPE_PVOID_QI_V8DI_V8SF_INT,
810 IX86_BUILTIN_SCATTERDIV16SF);
811
812 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatterdiv8df",
813 VOID_FTYPE_PVOID_QI_V8DI_V8DF_INT,
814 IX86_BUILTIN_SCATTERDIV8DF);
815
816 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scattersiv16si",
817 VOID_FTYPE_PVOID_HI_V16SI_V16SI_INT,
818 IX86_BUILTIN_SCATTERSIV16SI);
819
820 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scattersiv8di",
821 VOID_FTYPE_PVOID_QI_V8SI_V8DI_INT,
822 IX86_BUILTIN_SCATTERSIV8DI);
823
824 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatterdiv16si",
825 VOID_FTYPE_PVOID_QI_V8DI_V8SI_INT,
826 IX86_BUILTIN_SCATTERDIV16SI);
827
828 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatterdiv8di",
829 VOID_FTYPE_PVOID_QI_V8DI_V8DI_INT,
830 IX86_BUILTIN_SCATTERDIV8DI);
831
832 /* AVX512VL */
833 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv2df",
834 V2DF_FTYPE_V2DF_PCVOID_V4SI_QI_INT,
835 IX86_BUILTIN_GATHER3SIV2DF);
836
837 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv4df",
838 V4DF_FTYPE_V4DF_PCVOID_V4SI_QI_INT,
839 IX86_BUILTIN_GATHER3SIV4DF);
840
841 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div2df",
842 V2DF_FTYPE_V2DF_PCVOID_V2DI_QI_INT,
843 IX86_BUILTIN_GATHER3DIV2DF);
844
845 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div4df",
846 V4DF_FTYPE_V4DF_PCVOID_V4DI_QI_INT,
847 IX86_BUILTIN_GATHER3DIV4DF);
848
849 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv4sf",
850 V4SF_FTYPE_V4SF_PCVOID_V4SI_QI_INT,
851 IX86_BUILTIN_GATHER3SIV4SF);
852
853 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv8sf",
854 V8SF_FTYPE_V8SF_PCVOID_V8SI_QI_INT,
855 IX86_BUILTIN_GATHER3SIV8SF);
856
857 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div4sf",
858 V4SF_FTYPE_V4SF_PCVOID_V2DI_QI_INT,
859 IX86_BUILTIN_GATHER3DIV4SF);
860
861 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div8sf",
862 V4SF_FTYPE_V4SF_PCVOID_V4DI_QI_INT,
863 IX86_BUILTIN_GATHER3DIV8SF);
864
865 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv2di",
866 V2DI_FTYPE_V2DI_PCVOID_V4SI_QI_INT,
867 IX86_BUILTIN_GATHER3SIV2DI);
868
869 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv4di",
870 V4DI_FTYPE_V4DI_PCVOID_V4SI_QI_INT,
871 IX86_BUILTIN_GATHER3SIV4DI);
872
873 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div2di",
874 V2DI_FTYPE_V2DI_PCVOID_V2DI_QI_INT,
875 IX86_BUILTIN_GATHER3DIV2DI);
876
877 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div4di",
878 V4DI_FTYPE_V4DI_PCVOID_V4DI_QI_INT,
879 IX86_BUILTIN_GATHER3DIV4DI);
880
881 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv4si",
882 V4SI_FTYPE_V4SI_PCVOID_V4SI_QI_INT,
883 IX86_BUILTIN_GATHER3SIV4SI);
884
885 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3siv8si",
886 V8SI_FTYPE_V8SI_PCVOID_V8SI_QI_INT,
887 IX86_BUILTIN_GATHER3SIV8SI);
888
889 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div4si",
890 V4SI_FTYPE_V4SI_PCVOID_V2DI_QI_INT,
891 IX86_BUILTIN_GATHER3DIV4SI);
892
893 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3div8si",
894 V4SI_FTYPE_V4SI_PCVOID_V4DI_QI_INT,
895 IX86_BUILTIN_GATHER3DIV8SI);
896
897 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3altsiv4df ",
898 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_QI_INT,
899 IX86_BUILTIN_GATHER3ALTSIV4DF);
900
901 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3altdiv8sf ",
902 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_QI_INT,
903 IX86_BUILTIN_GATHER3ALTDIV8SF);
904
905 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3altsiv4di ",
906 V4DI_FTYPE_V4DI_PCINT64_V8SI_QI_INT,
907 IX86_BUILTIN_GATHER3ALTSIV4DI);
908
909 def_builtin_pure (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_gather3altdiv8si ",
910 V8SI_FTYPE_V8SI_PCINT_V4DI_QI_INT,
911 IX86_BUILTIN_GATHER3ALTDIV8SI);
912
913 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv8sf",
914 VOID_FTYPE_PVOID_QI_V8SI_V8SF_INT,
915 IX86_BUILTIN_SCATTERSIV8SF);
916
917 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv4sf",
918 VOID_FTYPE_PVOID_QI_V4SI_V4SF_INT,
919 IX86_BUILTIN_SCATTERSIV4SF);
920
921 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv4df",
922 VOID_FTYPE_PVOID_QI_V4SI_V4DF_INT,
923 IX86_BUILTIN_SCATTERSIV4DF);
924
925 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv2df",
926 VOID_FTYPE_PVOID_QI_V4SI_V2DF_INT,
927 IX86_BUILTIN_SCATTERSIV2DF);
928
929 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv8sf",
930 VOID_FTYPE_PVOID_QI_V4DI_V4SF_INT,
931 IX86_BUILTIN_SCATTERDIV8SF);
932
933 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv4sf",
934 VOID_FTYPE_PVOID_QI_V2DI_V4SF_INT,
935 IX86_BUILTIN_SCATTERDIV4SF);
936
937 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv4df",
938 VOID_FTYPE_PVOID_QI_V4DI_V4DF_INT,
939 IX86_BUILTIN_SCATTERDIV4DF);
940
941 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv2df",
942 VOID_FTYPE_PVOID_QI_V2DI_V2DF_INT,
943 IX86_BUILTIN_SCATTERDIV2DF);
944
945 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv8si",
946 VOID_FTYPE_PVOID_QI_V8SI_V8SI_INT,
947 IX86_BUILTIN_SCATTERSIV8SI);
948
949 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv4si",
950 VOID_FTYPE_PVOID_QI_V4SI_V4SI_INT,
951 IX86_BUILTIN_SCATTERSIV4SI);
952
953 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv4di",
954 VOID_FTYPE_PVOID_QI_V4SI_V4DI_INT,
955 IX86_BUILTIN_SCATTERSIV4DI);
956
957 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scattersiv2di",
958 VOID_FTYPE_PVOID_QI_V4SI_V2DI_INT,
959 IX86_BUILTIN_SCATTERSIV2DI);
960
961 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv8si",
962 VOID_FTYPE_PVOID_QI_V4DI_V4SI_INT,
963 IX86_BUILTIN_SCATTERDIV8SI);
964
965 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv4si",
966 VOID_FTYPE_PVOID_QI_V2DI_V4SI_INT,
967 IX86_BUILTIN_SCATTERDIV4SI);
968
969 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv4di",
970 VOID_FTYPE_PVOID_QI_V4DI_V4DI_INT,
971 IX86_BUILTIN_SCATTERDIV4DI);
972
973 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatterdiv2di",
974 VOID_FTYPE_PVOID_QI_V2DI_V2DI_INT,
975 IX86_BUILTIN_SCATTERDIV2DI);
976
977 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatteraltsiv8df ",
978 VOID_FTYPE_PDOUBLE_QI_V16SI_V8DF_INT,
979 IX86_BUILTIN_SCATTERALTSIV8DF);
980
981 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatteraltdiv16sf ",
982 VOID_FTYPE_PFLOAT_HI_V8DI_V16SF_INT,
983 IX86_BUILTIN_SCATTERALTDIV16SF);
984
985 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatteraltsiv8di ",
986 VOID_FTYPE_PLONGLONG_QI_V16SI_V8DI_INT,
987 IX86_BUILTIN_SCATTERALTSIV8DI);
988
989 def_builtin (OPTION_MASK_ISA_AVX512F, 0, "__builtin_ia32_scatteraltdiv16si ",
990 VOID_FTYPE_PINT_HI_V8DI_V16SI_INT,
991 IX86_BUILTIN_SCATTERALTDIV16SI);
992
993 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltsiv4df ",
994 VOID_FTYPE_PDOUBLE_QI_V8SI_V4DF_INT,
995 IX86_BUILTIN_SCATTERALTSIV4DF);
996
997 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltdiv8sf ",
998 VOID_FTYPE_PFLOAT_QI_V4DI_V8SF_INT,
999 IX86_BUILTIN_SCATTERALTDIV8SF);
1000
1001 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltsiv4di ",
1002 VOID_FTYPE_PLONGLONG_QI_V8SI_V4DI_INT,
1003 IX86_BUILTIN_SCATTERALTSIV4DI);
1004
1005 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltdiv8si ",
1006 VOID_FTYPE_PINT_QI_V4DI_V8SI_INT,
1007 IX86_BUILTIN_SCATTERALTDIV8SI);
1008
1009 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltsiv2df ",
1010 VOID_FTYPE_PDOUBLE_QI_V4SI_V2DF_INT,
1011 IX86_BUILTIN_SCATTERALTSIV2DF);
1012
1013 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltdiv4sf ",
1014 VOID_FTYPE_PFLOAT_QI_V2DI_V4SF_INT,
1015 IX86_BUILTIN_SCATTERALTDIV4SF);
1016
1017 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltsiv2di ",
1018 VOID_FTYPE_PLONGLONG_QI_V4SI_V2DI_INT,
1019 IX86_BUILTIN_SCATTERALTSIV2DI);
1020
1021 def_builtin (OPTION_MASK_ISA_AVX512VL, 0, "__builtin_ia32_scatteraltdiv4si ",
1022 VOID_FTYPE_PINT_QI_V2DI_V4SI_INT,
1023 IX86_BUILTIN_SCATTERALTDIV4SI);
1024
1025 /* AVX512PF */
1026 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_gatherpfdpd",
1027 VOID_FTYPE_QI_V8SI_PCVOID_INT_INT,
1028 IX86_BUILTIN_GATHERPFDPD);
1029 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_gatherpfdps",
1030 VOID_FTYPE_HI_V16SI_PCVOID_INT_INT,
1031 IX86_BUILTIN_GATHERPFDPS);
1032 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_gatherpfqpd",
1033 VOID_FTYPE_QI_V8DI_PCVOID_INT_INT,
1034 IX86_BUILTIN_GATHERPFQPD);
1035 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_gatherpfqps",
1036 VOID_FTYPE_QI_V8DI_PCVOID_INT_INT,
1037 IX86_BUILTIN_GATHERPFQPS);
1038 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_scatterpfdpd",
1039 VOID_FTYPE_QI_V8SI_PCVOID_INT_INT,
1040 IX86_BUILTIN_SCATTERPFDPD);
1041 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_scatterpfdps",
1042 VOID_FTYPE_HI_V16SI_PCVOID_INT_INT,
1043 IX86_BUILTIN_SCATTERPFDPS);
1044 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_scatterpfqpd",
1045 VOID_FTYPE_QI_V8DI_PCVOID_INT_INT,
1046 IX86_BUILTIN_SCATTERPFQPD);
1047 def_builtin (OPTION_MASK_ISA_AVX512PF, 0, "__builtin_ia32_scatterpfqps",
1048 VOID_FTYPE_QI_V8DI_PCVOID_INT_INT,
1049 IX86_BUILTIN_SCATTERPFQPS);
1050
1051 /* SHA */
1052 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha1msg1",
1053 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1MSG1);
1054 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha1msg2",
1055 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1MSG2);
1056 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha1nexte",
1057 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1NEXTE);
1058 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha1rnds4",
1059 V4SI_FTYPE_V4SI_V4SI_INT, IX86_BUILTIN_SHA1RNDS4);
1060 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha256msg1",
1061 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA256MSG1);
1062 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha256msg2",
1063 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA256MSG2);
1064 def_builtin_const (OPTION_MASK_ISA_SHA, 0, "__builtin_ia32_sha256rnds2",
1065 V4SI_FTYPE_V4SI_V4SI_V4SI, IX86_BUILTIN_SHA256RNDS2);
1066
1067 /* RTM. */
1068 def_builtin (OPTION_MASK_ISA_RTM, 0, "__builtin_ia32_xabort",
1069 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_XABORT);
1070
1071 /* MMX access to the vec_init patterns. */
1072 def_builtin_const (OPTION_MASK_ISA_MMX, 0,
1073 "__builtin_ia32_vec_init_v2si",
1074 V2SI_FTYPE_INT_INT, IX86_BUILTIN_VEC_INIT_V2SI);
1075
1076 def_builtin_const (OPTION_MASK_ISA_MMX, 0,
1077 "__builtin_ia32_vec_init_v4hi",
1078 V4HI_FTYPE_HI_HI_HI_HI,
1079 IX86_BUILTIN_VEC_INIT_V4HI);
1080
1081 def_builtin_const (OPTION_MASK_ISA_MMX, 0,
1082 "__builtin_ia32_vec_init_v8qi",
1083 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
1084 IX86_BUILTIN_VEC_INIT_V8QI);
1085
1086 /* Access to the vec_extract patterns. */
1087 def_builtin_const (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_vec_ext_v2df",
1088 DOUBLE_FTYPE_V2DF_INT, IX86_BUILTIN_VEC_EXT_V2DF);
1089 def_builtin_const (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_vec_ext_v2di",
1090 DI_FTYPE_V2DI_INT, IX86_BUILTIN_VEC_EXT_V2DI);
1091 def_builtin_const (OPTION_MASK_ISA_SSE, 0, "__builtin_ia32_vec_ext_v4sf",
1092 FLOAT_FTYPE_V4SF_INT, IX86_BUILTIN_VEC_EXT_V4SF);
1093 def_builtin_const (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_vec_ext_v4si",
1094 SI_FTYPE_V4SI_INT, IX86_BUILTIN_VEC_EXT_V4SI);
1095 def_builtin_const (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_vec_ext_v8hi",
1096 HI_FTYPE_V8HI_INT, IX86_BUILTIN_VEC_EXT_V8HI);
1097
1098 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
1099 /* As it uses V4HImode, we have to require -mmmx too. */
1100 | OPTION_MASK_ISA_MMX, 0,
1101 "__builtin_ia32_vec_ext_v4hi",
1102 HI_FTYPE_V4HI_INT, IX86_BUILTIN_VEC_EXT_V4HI);
1103
1104 def_builtin_const (OPTION_MASK_ISA_MMX, 0,
1105 "__builtin_ia32_vec_ext_v2si",
1106 SI_FTYPE_V2SI_INT, IX86_BUILTIN_VEC_EXT_V2SI);
1107
1108 def_builtin_const (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_vec_ext_v16qi",
1109 QI_FTYPE_V16QI_INT, IX86_BUILTIN_VEC_EXT_V16QI);
1110
1111 /* Access to the vec_set patterns. */
1112 def_builtin_const (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_64BIT, 0,
1113 "__builtin_ia32_vec_set_v2di",
1114 V2DI_FTYPE_V2DI_DI_INT, IX86_BUILTIN_VEC_SET_V2DI);
1115
1116 def_builtin_const (OPTION_MASK_ISA_SSE4_1, 0, "__builtin_ia32_vec_set_v4sf",
1117 V4SF_FTYPE_V4SF_FLOAT_INT, IX86_BUILTIN_VEC_SET_V4SF);
1118
1119 def_builtin_const (OPTION_MASK_ISA_SSE4_1, 0, "__builtin_ia32_vec_set_v4si",
1120 V4SI_FTYPE_V4SI_SI_INT, IX86_BUILTIN_VEC_SET_V4SI);
1121
1122 def_builtin_const (OPTION_MASK_ISA_SSE2, 0, "__builtin_ia32_vec_set_v8hi",
1123 V8HI_FTYPE_V8HI_HI_INT, IX86_BUILTIN_VEC_SET_V8HI);
1124
1125 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
1126 /* As it uses V4HImode, we have to require -mmmx too. */
1127 | OPTION_MASK_ISA_MMX, 0,
1128 "__builtin_ia32_vec_set_v4hi",
1129 V4HI_FTYPE_V4HI_HI_INT, IX86_BUILTIN_VEC_SET_V4HI);
1130
1131 def_builtin_const (OPTION_MASK_ISA_SSE4_1, 0, "__builtin_ia32_vec_set_v16qi",
1132 V16QI_FTYPE_V16QI_QI_INT, IX86_BUILTIN_VEC_SET_V16QI);
1133
1134 /* RDSEED */
1135 def_builtin (OPTION_MASK_ISA_RDSEED, 0, "__builtin_ia32_rdseed_hi_step",
1136 INT_FTYPE_PUSHORT, IX86_BUILTIN_RDSEED16_STEP);
1137 def_builtin (OPTION_MASK_ISA_RDSEED, 0, "__builtin_ia32_rdseed_si_step",
1138 INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDSEED32_STEP);
1139 def_builtin (OPTION_MASK_ISA_RDSEED | OPTION_MASK_ISA_64BIT, 0,
1140 "__builtin_ia32_rdseed_di_step",
1141 INT_FTYPE_PULONGLONG, IX86_BUILTIN_RDSEED64_STEP);
1142
1143 /* ADCX */
1144 def_builtin (0, 0, "__builtin_ia32_addcarryx_u32",
1145 UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED, IX86_BUILTIN_ADDCARRYX32);
1146 def_builtin (OPTION_MASK_ISA_64BIT, 0,
1147 "__builtin_ia32_addcarryx_u64",
1148 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
1149 IX86_BUILTIN_ADDCARRYX64);
1150
1151 /* SBB */
1152 def_builtin (0, 0, "__builtin_ia32_sbb_u32",
1153 UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED, IX86_BUILTIN_SBB32);
1154 def_builtin (OPTION_MASK_ISA_64BIT, 0,
1155 "__builtin_ia32_sbb_u64",
1156 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
1157 IX86_BUILTIN_SBB64);
1158
1159 /* Read/write FLAGS. */
1160 if (TARGET_64BIT)
1161 {
1162 def_builtin (OPTION_MASK_ISA_64BIT, 0, "__builtin_ia32_readeflags_u64",
1163 UINT64_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS);
1164 def_builtin (OPTION_MASK_ISA_64BIT, 0, "__builtin_ia32_writeeflags_u64",
1165 VOID_FTYPE_UINT64, IX86_BUILTIN_WRITE_FLAGS);
1166 }
1167 else
1168 {
1169 def_builtin (0, 0, "__builtin_ia32_readeflags_u32",
1170 UNSIGNED_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS);
1171 def_builtin (0, 0, "__builtin_ia32_writeeflags_u32",
1172 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_WRITE_FLAGS);
1173 }
1174
1175 /* CLFLUSHOPT. */
1176 def_builtin (OPTION_MASK_ISA_CLFLUSHOPT, 0, "__builtin_ia32_clflushopt",
1177 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLFLUSHOPT);
1178
1179 /* CLWB. */
1180 def_builtin (OPTION_MASK_ISA_CLWB, 0, "__builtin_ia32_clwb",
1181 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLWB);
1182
1183 /* MONITORX and MWAITX. */
1184 def_builtin (0, OPTION_MASK_ISA2_MWAITX, "__builtin_ia32_monitorx",
1185 VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITORX);
1186 def_builtin (0, OPTION_MASK_ISA2_MWAITX, "__builtin_ia32_mwaitx",
1187 VOID_FTYPE_UNSIGNED_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAITX);
1188
1189 /* CLZERO. */
1190 def_builtin (0, OPTION_MASK_ISA2_CLZERO, "__builtin_ia32_clzero",
1191 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLZERO);
1192
1193 /* WAITPKG. */
1194 def_builtin (0, OPTION_MASK_ISA2_WAITPKG, "__builtin_ia32_umonitor",
1195 VOID_FTYPE_PVOID, IX86_BUILTIN_UMONITOR);
1196 def_builtin (0, OPTION_MASK_ISA2_WAITPKG, "__builtin_ia32_umwait",
1197 UINT8_FTYPE_UNSIGNED_UINT64, IX86_BUILTIN_UMWAIT);
1198 def_builtin (0, OPTION_MASK_ISA2_WAITPKG, "__builtin_ia32_tpause",
1199 UINT8_FTYPE_UNSIGNED_UINT64, IX86_BUILTIN_TPAUSE);
1200
1201 /* UINTR. */
1202 def_builtin (OPTION_MASK_ISA_64BIT, OPTION_MASK_ISA2_UINTR,
1203 "__builtin_ia32_testui",
1204 UINT8_FTYPE_VOID, IX86_BUILTIN_TESTUI);
1205
1206 /* CLDEMOTE. */
1207 def_builtin (0, OPTION_MASK_ISA2_CLDEMOTE, "__builtin_ia32_cldemote",
1208 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLDEMOTE);
1209
1210 /* Add FMA4 multi-arg argument instructions */
1211 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
1212 {
1213 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_MULTI_ARG_FIRST, i);
1214 if (d->name == 0)
1215 continue;
1216
1217 ftype = (enum ix86_builtin_func_type) d->flag;
1218 def_builtin_const (d->mask, d->mask2, d->name, ftype, d->code);
1219 }
1220 BDESC_VERIFYS (IX86_BUILTIN__BDESC_MULTI_ARG_LAST,
1221 IX86_BUILTIN__BDESC_MULTI_ARG_FIRST,
1222 ARRAY_SIZE (bdesc_multi_arg) - 1);
1223
1224 /* Add CET inrinsics. */
1225 for (i = 0, d = bdesc_cet; i < ARRAY_SIZE (bdesc_cet); i++, d++)
1226 {
1227 BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_CET_FIRST, i);
1228 if (d->name == 0)
1229 continue;
1230
1231 ftype = (enum ix86_builtin_func_type) d->flag;
1232 def_builtin (d->mask, d->mask2, d->name, ftype, d->code);
1233 }
1234 BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_LAST,
1235 IX86_BUILTIN__BDESC_CET_FIRST,
1236 ARRAY_SIZE (bdesc_cet) - 1);
1237 }
1238
1239 #undef BDESC_VERIFY
1240 #undef BDESC_VERIFYS
1241
1242 /* Make builtins to detect cpu type and features supported. NAME is
1243 the builtin name, CODE is the builtin code, and FTYPE is the function
1244 type of the builtin. */
1245
1246 static void
make_cpu_type_builtin(const char * name,int code,enum ix86_builtin_func_type ftype,bool is_const)1247 make_cpu_type_builtin (const char* name, int code,
1248 enum ix86_builtin_func_type ftype, bool is_const)
1249 {
1250 tree decl;
1251 tree type;
1252
1253 type = ix86_get_builtin_func_type (ftype);
1254 decl = add_builtin_function (name, type, code, BUILT_IN_MD,
1255 NULL, NULL_TREE);
1256 gcc_assert (decl != NULL_TREE);
1257 ix86_builtins[(int) code] = decl;
1258 TREE_READONLY (decl) = is_const;
1259 }
1260
1261 /* Make builtins to get CPU type and features supported. The created
1262 builtins are :
1263
1264 __builtin_cpu_init (), to detect cpu type and features,
1265 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
1266 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
1267 */
1268
1269 static void
ix86_init_platform_type_builtins(void)1270 ix86_init_platform_type_builtins (void)
1271 {
1272 make_cpu_type_builtin ("__builtin_cpu_init", IX86_BUILTIN_CPU_INIT,
1273 INT_FTYPE_VOID, false);
1274 make_cpu_type_builtin ("__builtin_cpu_is", IX86_BUILTIN_CPU_IS,
1275 INT_FTYPE_PCCHAR, true);
1276 make_cpu_type_builtin ("__builtin_cpu_supports", IX86_BUILTIN_CPU_SUPPORTS,
1277 INT_FTYPE_PCCHAR, true);
1278 }
1279
1280 /* Internal method for ix86_init_builtins. */
1281
1282 static void
ix86_init_builtins_va_builtins_abi(void)1283 ix86_init_builtins_va_builtins_abi (void)
1284 {
1285 tree ms_va_ref, sysv_va_ref;
1286 tree fnvoid_va_end_ms, fnvoid_va_end_sysv;
1287 tree fnvoid_va_start_ms, fnvoid_va_start_sysv;
1288 tree fnvoid_va_copy_ms, fnvoid_va_copy_sysv;
1289 tree fnattr_ms = NULL_TREE, fnattr_sysv = NULL_TREE;
1290
1291 if (!TARGET_64BIT)
1292 return;
1293 fnattr_ms = build_tree_list (get_identifier ("ms_abi"), NULL_TREE);
1294 fnattr_sysv = build_tree_list (get_identifier ("sysv_abi"), NULL_TREE);
1295 ms_va_ref = build_reference_type (ms_va_list_type_node);
1296 sysv_va_ref = build_pointer_type (TREE_TYPE (sysv_va_list_type_node));
1297
1298 fnvoid_va_end_ms = build_function_type_list (void_type_node, ms_va_ref,
1299 NULL_TREE);
1300 fnvoid_va_start_ms
1301 = build_varargs_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
1302 fnvoid_va_end_sysv
1303 = build_function_type_list (void_type_node, sysv_va_ref, NULL_TREE);
1304 fnvoid_va_start_sysv
1305 = build_varargs_function_type_list (void_type_node, sysv_va_ref,
1306 NULL_TREE);
1307 fnvoid_va_copy_ms
1308 = build_function_type_list (void_type_node, ms_va_ref,
1309 ms_va_list_type_node, NULL_TREE);
1310 fnvoid_va_copy_sysv
1311 = build_function_type_list (void_type_node, sysv_va_ref,
1312 sysv_va_ref, NULL_TREE);
1313
1314 add_builtin_function ("__builtin_ms_va_start", fnvoid_va_start_ms,
1315 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_ms);
1316 add_builtin_function ("__builtin_ms_va_end", fnvoid_va_end_ms,
1317 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_ms);
1318 add_builtin_function ("__builtin_ms_va_copy", fnvoid_va_copy_ms,
1319 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_ms);
1320 add_builtin_function ("__builtin_sysv_va_start", fnvoid_va_start_sysv,
1321 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_sysv);
1322 add_builtin_function ("__builtin_sysv_va_end", fnvoid_va_end_sysv,
1323 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_sysv);
1324 add_builtin_function ("__builtin_sysv_va_copy", fnvoid_va_copy_sysv,
1325 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_sysv);
1326 }
1327
1328 static void
ix86_init_builtin_types(void)1329 ix86_init_builtin_types (void)
1330 {
1331 tree float80_type_node, const_string_type_node;
1332
1333 /* The __float80 type. */
1334 float80_type_node = long_double_type_node;
1335 if (TYPE_MODE (float80_type_node) != XFmode)
1336 {
1337 if (float64x_type_node != NULL_TREE
1338 && TYPE_MODE (float64x_type_node) == XFmode)
1339 float80_type_node = float64x_type_node;
1340 else
1341 {
1342 /* The __float80 type. */
1343 float80_type_node = make_node (REAL_TYPE);
1344
1345 TYPE_PRECISION (float80_type_node) = 80;
1346 layout_type (float80_type_node);
1347 }
1348 }
1349 lang_hooks.types.register_builtin_type (float80_type_node, "__float80");
1350
1351 /* The __float128 type. The node has already been created as
1352 _Float128, so we only need to register the __float128 name for
1353 it. */
1354 lang_hooks.types.register_builtin_type (float128_type_node, "__float128");
1355
1356 const_string_type_node
1357 = build_pointer_type (build_qualified_type
1358 (char_type_node, TYPE_QUAL_CONST));
1359
1360 /* This macro is built by i386-builtin-types.awk. */
1361 DEFINE_BUILTIN_PRIMITIVE_TYPES;
1362 }
1363
1364 void
ix86_init_builtins(void)1365 ix86_init_builtins (void)
1366 {
1367 tree ftype, decl;
1368
1369 ix86_init_builtin_types ();
1370
1371 /* Builtins to get CPU type and features. */
1372 ix86_init_platform_type_builtins ();
1373
1374 /* TFmode support builtins. */
1375 def_builtin_const (0, 0, "__builtin_infq",
1376 FLOAT128_FTYPE_VOID, IX86_BUILTIN_INFQ);
1377 def_builtin_const (0, 0, "__builtin_huge_valq",
1378 FLOAT128_FTYPE_VOID, IX86_BUILTIN_HUGE_VALQ);
1379
1380 ftype = ix86_get_builtin_func_type (FLOAT128_FTYPE_CONST_STRING);
1381 decl = add_builtin_function ("__builtin_nanq", ftype, IX86_BUILTIN_NANQ,
1382 BUILT_IN_MD, "nanq", NULL_TREE);
1383 TREE_READONLY (decl) = 1;
1384 ix86_builtins[(int) IX86_BUILTIN_NANQ] = decl;
1385
1386 decl = add_builtin_function ("__builtin_nansq", ftype, IX86_BUILTIN_NANSQ,
1387 BUILT_IN_MD, "nansq", NULL_TREE);
1388 TREE_READONLY (decl) = 1;
1389 ix86_builtins[(int) IX86_BUILTIN_NANSQ] = decl;
1390
1391 /* We will expand them to normal call if SSE isn't available since
1392 they are used by libgcc. */
1393 ftype = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128);
1394 decl = add_builtin_function ("__builtin_fabsq", ftype, IX86_BUILTIN_FABSQ,
1395 BUILT_IN_MD, "__fabstf2", NULL_TREE);
1396 TREE_READONLY (decl) = 1;
1397 ix86_builtins[(int) IX86_BUILTIN_FABSQ] = decl;
1398
1399 ftype = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128_FLOAT128);
1400 decl = add_builtin_function ("__builtin_copysignq", ftype,
1401 IX86_BUILTIN_COPYSIGNQ, BUILT_IN_MD,
1402 "__copysigntf3", NULL_TREE);
1403 TREE_READONLY (decl) = 1;
1404 ix86_builtins[(int) IX86_BUILTIN_COPYSIGNQ] = decl;
1405
1406 ix86_init_tm_builtins ();
1407 ix86_init_mmx_sse_builtins ();
1408
1409 if (TARGET_LP64)
1410 ix86_init_builtins_va_builtins_abi ();
1411
1412 #ifdef SUBTARGET_INIT_BUILTINS
1413 SUBTARGET_INIT_BUILTINS;
1414 #endif
1415 }
1416
1417 /* Return the ix86 builtin for CODE. */
1418
1419 tree
ix86_builtin_decl(unsigned code,bool)1420 ix86_builtin_decl (unsigned code, bool)
1421 {
1422 if (code >= IX86_BUILTIN_MAX)
1423 return error_mark_node;
1424
1425 return ix86_builtins[code];
1426 }
1427
1428 /* This returns the target-specific builtin with code CODE if
1429 current_function_decl has visibility on this builtin, which is checked
1430 using isa flags. Returns NULL_TREE otherwise. */
1431
ix86_get_builtin(enum ix86_builtins code)1432 static tree ix86_get_builtin (enum ix86_builtins code)
1433 {
1434 struct cl_target_option *opts;
1435 tree target_tree = NULL_TREE;
1436
1437 /* Determine the isa flags of current_function_decl. */
1438
1439 if (current_function_decl)
1440 target_tree = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
1441
1442 if (target_tree == NULL)
1443 target_tree = target_option_default_node;
1444
1445 opts = TREE_TARGET_OPTION (target_tree);
1446
1447 if ((ix86_builtins_isa[(int) code].isa & opts->x_ix86_isa_flags)
1448 || (ix86_builtins_isa[(int) code].isa2 & opts->x_ix86_isa_flags2))
1449 return ix86_builtin_decl (code, true);
1450 else
1451 return NULL_TREE;
1452 }
1453
1454 /* Vectorization library interface and handlers. */
1455 tree (*ix86_veclib_handler) (combined_fn, tree, tree);
1456
1457 /* Returns a function decl for a vectorized version of the combined function
1458 with combined_fn code FN and the result vector type TYPE, or NULL_TREE
1459 if it is not available. */
1460
1461 tree
ix86_builtin_vectorized_function(unsigned int fn,tree type_out,tree type_in)1462 ix86_builtin_vectorized_function (unsigned int fn, tree type_out,
1463 tree type_in)
1464 {
1465 machine_mode in_mode, out_mode;
1466 int in_n, out_n;
1467
1468 if (TREE_CODE (type_out) != VECTOR_TYPE
1469 || TREE_CODE (type_in) != VECTOR_TYPE)
1470 return NULL_TREE;
1471
1472 out_mode = TYPE_MODE (TREE_TYPE (type_out));
1473 out_n = TYPE_VECTOR_SUBPARTS (type_out);
1474 in_mode = TYPE_MODE (TREE_TYPE (type_in));
1475 in_n = TYPE_VECTOR_SUBPARTS (type_in);
1476
1477 switch (fn)
1478 {
1479 CASE_CFN_EXP2:
1480 if (out_mode == SFmode && in_mode == SFmode)
1481 {
1482 if (out_n == 16 && in_n == 16)
1483 return ix86_get_builtin (IX86_BUILTIN_EXP2PS);
1484 }
1485 break;
1486
1487 CASE_CFN_IFLOOR:
1488 CASE_CFN_LFLOOR:
1489 CASE_CFN_LLFLOOR:
1490 /* The round insn does not trap on denormals. */
1491 if (flag_trapping_math || !TARGET_SSE4_1)
1492 break;
1493
1494 if (out_mode == SImode && in_mode == DFmode)
1495 {
1496 if (out_n == 4 && in_n == 2)
1497 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX);
1498 else if (out_n == 8 && in_n == 4)
1499 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256);
1500 else if (out_n == 16 && in_n == 8)
1501 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512);
1502 }
1503 if (out_mode == SImode && in_mode == SFmode)
1504 {
1505 if (out_n == 4 && in_n == 4)
1506 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX);
1507 else if (out_n == 8 && in_n == 8)
1508 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX256);
1509 else if (out_n == 16 && in_n == 16)
1510 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX512);
1511 }
1512 break;
1513
1514 CASE_CFN_ICEIL:
1515 CASE_CFN_LCEIL:
1516 CASE_CFN_LLCEIL:
1517 /* The round insn does not trap on denormals. */
1518 if (flag_trapping_math || !TARGET_SSE4_1)
1519 break;
1520
1521 if (out_mode == SImode && in_mode == DFmode)
1522 {
1523 if (out_n == 4 && in_n == 2)
1524 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX);
1525 else if (out_n == 8 && in_n == 4)
1526 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256);
1527 else if (out_n == 16 && in_n == 8)
1528 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512);
1529 }
1530 if (out_mode == SImode && in_mode == SFmode)
1531 {
1532 if (out_n == 4 && in_n == 4)
1533 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX);
1534 else if (out_n == 8 && in_n == 8)
1535 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX256);
1536 else if (out_n == 16 && in_n == 16)
1537 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX512);
1538 }
1539 break;
1540
1541 CASE_CFN_IRINT:
1542 CASE_CFN_LRINT:
1543 CASE_CFN_LLRINT:
1544 if (out_mode == SImode && in_mode == DFmode)
1545 {
1546 if (out_n == 4 && in_n == 2)
1547 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX);
1548 else if (out_n == 8 && in_n == 4)
1549 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX256);
1550 else if (out_n == 16 && in_n == 8)
1551 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX512);
1552 }
1553 if (out_mode == SImode && in_mode == SFmode)
1554 {
1555 if (out_n == 4 && in_n == 4)
1556 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ);
1557 else if (out_n == 8 && in_n == 8)
1558 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ256);
1559 else if (out_n == 16 && in_n == 16)
1560 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ512);
1561 }
1562 break;
1563
1564 CASE_CFN_IROUND:
1565 CASE_CFN_LROUND:
1566 CASE_CFN_LLROUND:
1567 /* The round insn does not trap on denormals. */
1568 if (flag_trapping_math || !TARGET_SSE4_1)
1569 break;
1570
1571 if (out_mode == SImode && in_mode == DFmode)
1572 {
1573 if (out_n == 4 && in_n == 2)
1574 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX);
1575 else if (out_n == 8 && in_n == 4)
1576 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256);
1577 else if (out_n == 16 && in_n == 8)
1578 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512);
1579 }
1580 if (out_mode == SImode && in_mode == SFmode)
1581 {
1582 if (out_n == 4 && in_n == 4)
1583 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX);
1584 else if (out_n == 8 && in_n == 8)
1585 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX256);
1586 else if (out_n == 16 && in_n == 16)
1587 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX512);
1588 }
1589 break;
1590
1591 CASE_CFN_FLOOR:
1592 /* The round insn does not trap on denormals. */
1593 if (flag_trapping_math || !TARGET_SSE4_1)
1594 break;
1595
1596 if (out_mode == DFmode && in_mode == DFmode)
1597 {
1598 if (out_n == 2 && in_n == 2)
1599 return ix86_get_builtin (IX86_BUILTIN_FLOORPD);
1600 else if (out_n == 4 && in_n == 4)
1601 return ix86_get_builtin (IX86_BUILTIN_FLOORPD256);
1602 else if (out_n == 8 && in_n == 8)
1603 return ix86_get_builtin (IX86_BUILTIN_FLOORPD512);
1604 }
1605 if (out_mode == SFmode && in_mode == SFmode)
1606 {
1607 if (out_n == 4 && in_n == 4)
1608 return ix86_get_builtin (IX86_BUILTIN_FLOORPS);
1609 else if (out_n == 8 && in_n == 8)
1610 return ix86_get_builtin (IX86_BUILTIN_FLOORPS256);
1611 else if (out_n == 16 && in_n == 16)
1612 return ix86_get_builtin (IX86_BUILTIN_FLOORPS512);
1613 }
1614 break;
1615
1616 CASE_CFN_CEIL:
1617 /* The round insn does not trap on denormals. */
1618 if (flag_trapping_math || !TARGET_SSE4_1)
1619 break;
1620
1621 if (out_mode == DFmode && in_mode == DFmode)
1622 {
1623 if (out_n == 2 && in_n == 2)
1624 return ix86_get_builtin (IX86_BUILTIN_CEILPD);
1625 else if (out_n == 4 && in_n == 4)
1626 return ix86_get_builtin (IX86_BUILTIN_CEILPD256);
1627 else if (out_n == 8 && in_n == 8)
1628 return ix86_get_builtin (IX86_BUILTIN_CEILPD512);
1629 }
1630 if (out_mode == SFmode && in_mode == SFmode)
1631 {
1632 if (out_n == 4 && in_n == 4)
1633 return ix86_get_builtin (IX86_BUILTIN_CEILPS);
1634 else if (out_n == 8 && in_n == 8)
1635 return ix86_get_builtin (IX86_BUILTIN_CEILPS256);
1636 else if (out_n == 16 && in_n == 16)
1637 return ix86_get_builtin (IX86_BUILTIN_CEILPS512);
1638 }
1639 break;
1640
1641 CASE_CFN_TRUNC:
1642 /* The round insn does not trap on denormals. */
1643 if (flag_trapping_math || !TARGET_SSE4_1)
1644 break;
1645
1646 if (out_mode == DFmode && in_mode == DFmode)
1647 {
1648 if (out_n == 2 && in_n == 2)
1649 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD);
1650 else if (out_n == 4 && in_n == 4)
1651 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD256);
1652 else if (out_n == 8 && in_n == 8)
1653 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD512);
1654 }
1655 if (out_mode == SFmode && in_mode == SFmode)
1656 {
1657 if (out_n == 4 && in_n == 4)
1658 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS);
1659 else if (out_n == 8 && in_n == 8)
1660 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS256);
1661 else if (out_n == 16 && in_n == 16)
1662 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS512);
1663 }
1664 break;
1665
1666 CASE_CFN_FMA:
1667 if (out_mode == DFmode && in_mode == DFmode)
1668 {
1669 if (out_n == 2 && in_n == 2)
1670 return ix86_get_builtin (IX86_BUILTIN_VFMADDPD);
1671 if (out_n == 4 && in_n == 4)
1672 return ix86_get_builtin (IX86_BUILTIN_VFMADDPD256);
1673 }
1674 if (out_mode == SFmode && in_mode == SFmode)
1675 {
1676 if (out_n == 4 && in_n == 4)
1677 return ix86_get_builtin (IX86_BUILTIN_VFMADDPS);
1678 if (out_n == 8 && in_n == 8)
1679 return ix86_get_builtin (IX86_BUILTIN_VFMADDPS256);
1680 }
1681 break;
1682
1683 default:
1684 break;
1685 }
1686
1687 /* Dispatch to a handler for a vectorization library. */
1688 if (ix86_veclib_handler)
1689 return ix86_veclib_handler (combined_fn (fn), type_out, type_in);
1690
1691 return NULL_TREE;
1692 }
1693
1694 /* Returns a decl of a function that implements gather load with
1695 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
1696 Return NULL_TREE if it is not available. */
1697
1698 tree
ix86_vectorize_builtin_gather(const_tree mem_vectype,const_tree index_type,int scale)1699 ix86_vectorize_builtin_gather (const_tree mem_vectype,
1700 const_tree index_type, int scale)
1701 {
1702 bool si;
1703 enum ix86_builtins code;
1704
1705 if (! TARGET_AVX2 || !TARGET_USE_GATHER)
1706 return NULL_TREE;
1707
1708 if ((TREE_CODE (index_type) != INTEGER_TYPE
1709 && !POINTER_TYPE_P (index_type))
1710 || (TYPE_MODE (index_type) != SImode
1711 && TYPE_MODE (index_type) != DImode))
1712 return NULL_TREE;
1713
1714 if (TYPE_PRECISION (index_type) > POINTER_SIZE)
1715 return NULL_TREE;
1716
1717 /* v*gather* insn sign extends index to pointer mode. */
1718 if (TYPE_PRECISION (index_type) < POINTER_SIZE
1719 && TYPE_UNSIGNED (index_type))
1720 return NULL_TREE;
1721
1722 if (scale <= 0
1723 || scale > 8
1724 || (scale & (scale - 1)) != 0)
1725 return NULL_TREE;
1726
1727 si = TYPE_MODE (index_type) == SImode;
1728 switch (TYPE_MODE (mem_vectype))
1729 {
1730 case E_V2DFmode:
1731 if (TARGET_AVX512VL)
1732 code = si ? IX86_BUILTIN_GATHER3SIV2DF : IX86_BUILTIN_GATHER3DIV2DF;
1733 else
1734 code = si ? IX86_BUILTIN_GATHERSIV2DF : IX86_BUILTIN_GATHERDIV2DF;
1735 break;
1736 case E_V4DFmode:
1737 if (TARGET_AVX512VL)
1738 code = si ? IX86_BUILTIN_GATHER3ALTSIV4DF : IX86_BUILTIN_GATHER3DIV4DF;
1739 else
1740 code = si ? IX86_BUILTIN_GATHERALTSIV4DF : IX86_BUILTIN_GATHERDIV4DF;
1741 break;
1742 case E_V2DImode:
1743 if (TARGET_AVX512VL)
1744 code = si ? IX86_BUILTIN_GATHER3SIV2DI : IX86_BUILTIN_GATHER3DIV2DI;
1745 else
1746 code = si ? IX86_BUILTIN_GATHERSIV2DI : IX86_BUILTIN_GATHERDIV2DI;
1747 break;
1748 case E_V4DImode:
1749 if (TARGET_AVX512VL)
1750 code = si ? IX86_BUILTIN_GATHER3ALTSIV4DI : IX86_BUILTIN_GATHER3DIV4DI;
1751 else
1752 code = si ? IX86_BUILTIN_GATHERALTSIV4DI : IX86_BUILTIN_GATHERDIV4DI;
1753 break;
1754 case E_V4SFmode:
1755 if (TARGET_AVX512VL)
1756 code = si ? IX86_BUILTIN_GATHER3SIV4SF : IX86_BUILTIN_GATHER3DIV4SF;
1757 else
1758 code = si ? IX86_BUILTIN_GATHERSIV4SF : IX86_BUILTIN_GATHERDIV4SF;
1759 break;
1760 case E_V8SFmode:
1761 if (TARGET_AVX512VL)
1762 code = si ? IX86_BUILTIN_GATHER3SIV8SF : IX86_BUILTIN_GATHER3ALTDIV8SF;
1763 else
1764 code = si ? IX86_BUILTIN_GATHERSIV8SF : IX86_BUILTIN_GATHERALTDIV8SF;
1765 break;
1766 case E_V4SImode:
1767 if (TARGET_AVX512VL)
1768 code = si ? IX86_BUILTIN_GATHER3SIV4SI : IX86_BUILTIN_GATHER3DIV4SI;
1769 else
1770 code = si ? IX86_BUILTIN_GATHERSIV4SI : IX86_BUILTIN_GATHERDIV4SI;
1771 break;
1772 case E_V8SImode:
1773 if (TARGET_AVX512VL)
1774 code = si ? IX86_BUILTIN_GATHER3SIV8SI : IX86_BUILTIN_GATHER3ALTDIV8SI;
1775 else
1776 code = si ? IX86_BUILTIN_GATHERSIV8SI : IX86_BUILTIN_GATHERALTDIV8SI;
1777 break;
1778 case E_V8DFmode:
1779 if (TARGET_AVX512F)
1780 code = si ? IX86_BUILTIN_GATHER3ALTSIV8DF : IX86_BUILTIN_GATHER3DIV8DF;
1781 else
1782 return NULL_TREE;
1783 break;
1784 case E_V8DImode:
1785 if (TARGET_AVX512F)
1786 code = si ? IX86_BUILTIN_GATHER3ALTSIV8DI : IX86_BUILTIN_GATHER3DIV8DI;
1787 else
1788 return NULL_TREE;
1789 break;
1790 case E_V16SFmode:
1791 if (TARGET_AVX512F)
1792 code = si ? IX86_BUILTIN_GATHER3SIV16SF : IX86_BUILTIN_GATHER3ALTDIV16SF;
1793 else
1794 return NULL_TREE;
1795 break;
1796 case E_V16SImode:
1797 if (TARGET_AVX512F)
1798 code = si ? IX86_BUILTIN_GATHER3SIV16SI : IX86_BUILTIN_GATHER3ALTDIV16SI;
1799 else
1800 return NULL_TREE;
1801 break;
1802 default:
1803 return NULL_TREE;
1804 }
1805
1806 return ix86_get_builtin (code);
1807 }
1808
1809 /* Returns a code for a target-specific builtin that implements
1810 reciprocal of the function, or NULL_TREE if not available. */
1811
1812 tree
ix86_builtin_reciprocal(tree fndecl)1813 ix86_builtin_reciprocal (tree fndecl)
1814 {
1815 enum ix86_builtins fn_code
1816 = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
1817 switch (fn_code)
1818 {
1819 /* Vectorized version of sqrt to rsqrt conversion. */
1820 case IX86_BUILTIN_SQRTPS_NR:
1821 return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR);
1822
1823 case IX86_BUILTIN_SQRTPS_NR256:
1824 return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR256);
1825
1826 default:
1827 return NULL_TREE;
1828 }
1829 }
1830
1831 /* This parses the attribute arguments to target in DECL and determines
1832 the right builtin to use to match the platform specification.
1833 It returns the priority value for this version decl. If PREDICATE_LIST
1834 is not NULL, it stores the list of cpu features that need to be checked
1835 before dispatching this function. */
1836
1837 unsigned int
get_builtin_code_for_version(tree decl,tree * predicate_list)1838 get_builtin_code_for_version (tree decl, tree *predicate_list)
1839 {
1840 tree attrs;
1841 struct cl_target_option cur_target;
1842 tree target_node;
1843 struct cl_target_option *new_target;
1844 const char *arg_str = NULL;
1845 const char *attrs_str = NULL;
1846 char *tok_str = NULL;
1847 char *token;
1848
1849 enum feature_priority priority = P_NONE;
1850
1851 static unsigned int NUM_FEATURES
1852 = sizeof (isa_names_table) / sizeof (_isa_names_table);
1853
1854 unsigned int i;
1855
1856 tree predicate_chain = NULL_TREE;
1857 tree predicate_decl, predicate_arg;
1858
1859 attrs = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
1860 gcc_assert (attrs != NULL);
1861
1862 attrs = TREE_VALUE (TREE_VALUE (attrs));
1863
1864 gcc_assert (TREE_CODE (attrs) == STRING_CST);
1865 attrs_str = TREE_STRING_POINTER (attrs);
1866
1867 /* Return priority zero for default function. */
1868 if (strcmp (attrs_str, "default") == 0)
1869 return 0;
1870
1871 /* Handle arch= if specified. For priority, set it to be 1 more than
1872 the best instruction set the processor can handle. For instance, if
1873 there is a version for atom and a version for ssse3 (the highest ISA
1874 priority for atom), the atom version must be checked for dispatch
1875 before the ssse3 version. */
1876 if (strstr (attrs_str, "arch=") != NULL)
1877 {
1878 cl_target_option_save (&cur_target, &global_options,
1879 &global_options_set);
1880 target_node
1881 = ix86_valid_target_attribute_tree (decl, attrs, &global_options,
1882 &global_options_set, 0);
1883
1884 gcc_assert (target_node);
1885 if (target_node == error_mark_node)
1886 return 0;
1887 new_target = TREE_TARGET_OPTION (target_node);
1888 gcc_assert (new_target);
1889
1890 if (new_target->arch_specified && new_target->arch > 0)
1891 for (i = 0; i < pta_size; i++)
1892 if (processor_alias_table[i].processor == new_target->arch)
1893 {
1894 const pta *arch_info = &processor_alias_table[i];
1895 switch (arch_info->priority)
1896 {
1897 default:
1898 arg_str = arch_info->name;
1899 priority = arch_info->priority;
1900 break;
1901 case P_PROC_DYNAMIC:
1902 switch (new_target->arch)
1903 {
1904 case PROCESSOR_NEHALEM:
1905 if (TARGET_PCLMUL_P (new_target->x_ix86_isa_flags))
1906 {
1907 arg_str = "westmere";
1908 priority = P_PCLMUL;
1909 }
1910 else
1911 {
1912 /* We translate "arch=corei7" and "arch=nehalem"
1913 to "corei7" so that it will be mapped to
1914 M_INTEL_COREI7 as cpu type to cover all
1915 M_INTEL_COREI7_XXXs. */
1916 arg_str = "corei7";
1917 priority = P_PROC_SSE4_2;
1918 }
1919 break;
1920 case PROCESSOR_SANDYBRIDGE:
1921 if (TARGET_F16C_P (new_target->x_ix86_isa_flags))
1922 arg_str = "ivybridge";
1923 else
1924 arg_str = "sandybridge";
1925 priority = P_PROC_AVX;
1926 break;
1927 case PROCESSOR_HASWELL:
1928 if (TARGET_ADX_P (new_target->x_ix86_isa_flags))
1929 arg_str = "broadwell";
1930 else
1931 arg_str = "haswell";
1932 priority = P_PROC_AVX2;
1933 break;
1934 case PROCESSOR_AMDFAM10:
1935 arg_str = "amdfam10h";
1936 priority = P_PROC_SSE4_A;
1937 break;
1938 default:
1939 gcc_unreachable ();
1940 }
1941 break;
1942 case P_NONE:
1943 break;
1944 }
1945 break;
1946 }
1947
1948 cl_target_option_restore (&global_options, &global_options_set,
1949 &cur_target);
1950
1951 if (predicate_list && arg_str == NULL)
1952 {
1953 error_at (DECL_SOURCE_LOCATION (decl),
1954 "no dispatcher found for the versioning attributes");
1955 return 0;
1956 }
1957
1958 if (predicate_list)
1959 {
1960 predicate_decl = ix86_builtins [(int) IX86_BUILTIN_CPU_IS];
1961 /* For a C string literal the length includes the trailing NULL. */
1962 predicate_arg = build_string_literal (strlen (arg_str) + 1, arg_str);
1963 predicate_chain = tree_cons (predicate_decl, predicate_arg,
1964 predicate_chain);
1965 }
1966 }
1967
1968 /* Process feature name. */
1969 tok_str = (char *) xmalloc (strlen (attrs_str) + 1);
1970 strcpy (tok_str, attrs_str);
1971 token = strtok (tok_str, ",");
1972 predicate_decl = ix86_builtins [(int) IX86_BUILTIN_CPU_SUPPORTS];
1973
1974 while (token != NULL)
1975 {
1976 /* Do not process "arch=" */
1977 if (strncmp (token, "arch=", 5) == 0)
1978 {
1979 token = strtok (NULL, ",");
1980 continue;
1981 }
1982 for (i = 0; i < NUM_FEATURES; ++i)
1983 {
1984 if (strcmp (token, isa_names_table[i].name) == 0)
1985 {
1986 if (predicate_list)
1987 {
1988 predicate_arg = build_string_literal (
1989 strlen (isa_names_table[i].name) + 1,
1990 isa_names_table[i].name);
1991 predicate_chain = tree_cons (predicate_decl, predicate_arg,
1992 predicate_chain);
1993 }
1994 /* Find the maximum priority feature. */
1995 if (isa_names_table[i].priority > priority)
1996 priority = isa_names_table[i].priority;
1997
1998 break;
1999 }
2000 }
2001 if (predicate_list && priority == P_NONE)
2002 {
2003 error_at (DECL_SOURCE_LOCATION (decl),
2004 "ISA %qs is not supported in %<target%> attribute, "
2005 "use %<arch=%> syntax", token);
2006 return 0;
2007 }
2008 token = strtok (NULL, ",");
2009 }
2010 free (tok_str);
2011
2012 if (predicate_list && predicate_chain == NULL_TREE)
2013 {
2014 error_at (DECL_SOURCE_LOCATION (decl),
2015 "no dispatcher found for the versioning attributes: %s",
2016 attrs_str);
2017 return 0;
2018 }
2019 else if (predicate_list)
2020 {
2021 predicate_chain = nreverse (predicate_chain);
2022 *predicate_list = predicate_chain;
2023 }
2024
2025 return priority;
2026 }
2027
2028 /* This builds the processor_model struct type defined in
2029 libgcc/config/i386/cpuinfo.c */
2030
2031 static tree
build_processor_model_struct(void)2032 build_processor_model_struct (void)
2033 {
2034 const char *field_name[] = {"__cpu_vendor", "__cpu_type", "__cpu_subtype",
2035 "__cpu_features"};
2036 tree field = NULL_TREE, field_chain = NULL_TREE;
2037 int i;
2038 tree type = make_node (RECORD_TYPE);
2039
2040 /* The first 3 fields are unsigned int. */
2041 for (i = 0; i < 3; ++i)
2042 {
2043 field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
2044 get_identifier (field_name[i]), unsigned_type_node);
2045 if (field_chain != NULL_TREE)
2046 DECL_CHAIN (field) = field_chain;
2047 field_chain = field;
2048 }
2049
2050 /* The last field is an array of unsigned integers of size one. */
2051 field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
2052 get_identifier (field_name[3]),
2053 build_array_type (unsigned_type_node,
2054 build_index_type (size_one_node)));
2055 if (field_chain != NULL_TREE)
2056 DECL_CHAIN (field) = field_chain;
2057 field_chain = field;
2058
2059 finish_builtin_struct (type, "__processor_model", field_chain, NULL_TREE);
2060 return type;
2061 }
2062
2063 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
2064
2065 static tree
make_var_decl(tree type,const char * name)2066 make_var_decl (tree type, const char *name)
2067 {
2068 tree new_decl;
2069
2070 new_decl = build_decl (UNKNOWN_LOCATION,
2071 VAR_DECL,
2072 get_identifier(name),
2073 type);
2074
2075 DECL_EXTERNAL (new_decl) = 1;
2076 TREE_STATIC (new_decl) = 1;
2077 TREE_PUBLIC (new_decl) = 1;
2078 DECL_INITIAL (new_decl) = 0;
2079 DECL_ARTIFICIAL (new_decl) = 0;
2080 DECL_PRESERVE_P (new_decl) = 1;
2081
2082 make_decl_one_only (new_decl, DECL_ASSEMBLER_NAME (new_decl));
2083 assemble_variable (new_decl, 0, 0, 0);
2084
2085 return new_decl;
2086 }
2087
2088 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
2089 into an integer defined in libgcc/config/i386/cpuinfo.c */
2090
2091 tree
fold_builtin_cpu(tree fndecl,tree * args)2092 fold_builtin_cpu (tree fndecl, tree *args)
2093 {
2094 unsigned int i;
2095 enum ix86_builtins fn_code
2096 = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
2097 tree param_string_cst = NULL;
2098
2099 tree __processor_model_type = build_processor_model_struct ();
2100 tree __cpu_model_var = make_var_decl (__processor_model_type,
2101 "__cpu_model");
2102
2103
2104 varpool_node::add (__cpu_model_var);
2105
2106 gcc_assert ((args != NULL) && (*args != NULL));
2107
2108 param_string_cst = *args;
2109 while (param_string_cst
2110 && TREE_CODE (param_string_cst) != STRING_CST)
2111 {
2112 /* *args must be a expr that can contain other EXPRS leading to a
2113 STRING_CST. */
2114 if (!EXPR_P (param_string_cst))
2115 {
2116 error ("parameter to builtin must be a string constant or literal");
2117 return integer_zero_node;
2118 }
2119 param_string_cst = TREE_OPERAND (EXPR_CHECK (param_string_cst), 0);
2120 }
2121
2122 gcc_assert (param_string_cst);
2123
2124 if (fn_code == IX86_BUILTIN_CPU_IS)
2125 {
2126 tree ref;
2127 tree field;
2128 tree final;
2129
2130 unsigned int field_val = 0;
2131
2132 for (i = 0; i < num_arch_names; i++)
2133 if (processor_alias_table[i].model != 0
2134 && strcmp (processor_alias_table[i].name,
2135 TREE_STRING_POINTER (param_string_cst)) == 0)
2136 break;
2137
2138 if (i == num_arch_names)
2139 {
2140 error ("parameter to builtin not valid: %s",
2141 TREE_STRING_POINTER (param_string_cst));
2142 return integer_zero_node;
2143 }
2144
2145 field = TYPE_FIELDS (__processor_model_type);
2146 field_val = processor_alias_table[i].model;
2147
2148 /* CPU types are stored in the next field. */
2149 if (field_val > M_CPU_TYPE_START
2150 && field_val < M_CPU_SUBTYPE_START)
2151 {
2152 field = DECL_CHAIN (field);
2153 field_val -= M_CPU_TYPE_START;
2154 }
2155
2156 /* CPU subtypes are stored in the next field. */
2157 if (field_val > M_CPU_SUBTYPE_START)
2158 {
2159 field = DECL_CHAIN ( DECL_CHAIN (field));
2160 field_val -= M_CPU_SUBTYPE_START;
2161 }
2162
2163 /* Get the appropriate field in __cpu_model. */
2164 ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
2165 field, NULL_TREE);
2166
2167 /* Check the value. */
2168 final = build2 (EQ_EXPR, unsigned_type_node, ref,
2169 build_int_cstu (unsigned_type_node, field_val));
2170 return build1 (CONVERT_EXPR, integer_type_node, final);
2171 }
2172 else if (fn_code == IX86_BUILTIN_CPU_SUPPORTS)
2173 {
2174 tree ref;
2175 tree array_elt;
2176 tree field;
2177 tree final;
2178
2179 unsigned int field_val = 0;
2180 unsigned int NUM_ISA_NAMES
2181 = sizeof (isa_names_table) / sizeof (struct _isa_names_table);
2182
2183 for (i = 0; i < NUM_ISA_NAMES; i++)
2184 if (strcmp (isa_names_table[i].name,
2185 TREE_STRING_POINTER (param_string_cst)) == 0)
2186 break;
2187
2188 if (i == NUM_ISA_NAMES)
2189 {
2190 error ("parameter to builtin not valid: %s",
2191 TREE_STRING_POINTER (param_string_cst));
2192 return integer_zero_node;
2193 }
2194
2195 if (isa_names_table[i].feature >= 32)
2196 {
2197 tree index_type
2198 = build_index_type (size_int (SIZE_OF_CPU_FEATURES));
2199 tree type = build_array_type (unsigned_type_node, index_type);
2200 tree __cpu_features2_var = make_var_decl (type,
2201 "__cpu_features2");
2202
2203 varpool_node::add (__cpu_features2_var);
2204 for (unsigned int j = 0; j < SIZE_OF_CPU_FEATURES; j++)
2205 if (isa_names_table[i].feature < (32 + 32 + j * 32))
2206 {
2207 field_val = (1U << (isa_names_table[i].feature
2208 - (32 + j * 32)));
2209 tree index = size_int (j);
2210 array_elt = build4 (ARRAY_REF, unsigned_type_node,
2211 __cpu_features2_var,
2212 index, NULL_TREE, NULL_TREE);
2213 /* Return __cpu_features2[index] & field_val */
2214 final = build2 (BIT_AND_EXPR, unsigned_type_node,
2215 array_elt,
2216 build_int_cstu (unsigned_type_node,
2217 field_val));
2218 return build1 (CONVERT_EXPR, integer_type_node, final);
2219 }
2220 }
2221
2222 field = TYPE_FIELDS (__processor_model_type);
2223 /* Get the last field, which is __cpu_features. */
2224 while (DECL_CHAIN (field))
2225 field = DECL_CHAIN (field);
2226
2227 /* Get the appropriate field: __cpu_model.__cpu_features */
2228 ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
2229 field, NULL_TREE);
2230
2231 /* Access the 0th element of __cpu_features array. */
2232 array_elt = build4 (ARRAY_REF, unsigned_type_node, ref,
2233 integer_zero_node, NULL_TREE, NULL_TREE);
2234
2235 field_val = (1U << isa_names_table[i].feature);
2236 /* Return __cpu_model.__cpu_features[0] & field_val */
2237 final = build2 (BIT_AND_EXPR, unsigned_type_node, array_elt,
2238 build_int_cstu (unsigned_type_node, field_val));
2239 return build1 (CONVERT_EXPR, integer_type_node, final);
2240 }
2241 gcc_unreachable ();
2242 }
2243
2244 #include "gt-i386-builtins.h"
2245