1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2018 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Return vector of parameter types of function FNDECL. This uses
90 TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static vec<tree>
simd_clone_vector_of_formal_parm_types(tree fndecl)94 simd_clone_vector_of_formal_parm_types (tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
98 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
99 unsigned int i;
100 tree arg;
101 FOR_EACH_VEC_ELT (args, i, arg)
102 args[i] = TREE_TYPE (args[i]);
103 return args;
104 }
105
106 /* Given a simd function in NODE, extract the simd specific
107 information from the OMP clauses passed in CLAUSES, and return
108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
109 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
110 otherwise set to FALSE. */
111
112 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)113 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
114 bool *inbranch_specified)
115 {
116 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
117 tree t;
118 int n;
119 *inbranch_specified = false;
120
121 n = args.length ();
122 if (n > 0 && args.last () == void_type_node)
123 n--;
124
125 /* Allocate one more than needed just in case this is an in-branch
126 clone which will require a mask argument. */
127 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
128 clone_info->nargs = n;
129
130 if (!clauses)
131 goto out;
132
133 clauses = TREE_VALUE (clauses);
134 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
135 goto out;
136
137 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
138 {
139 switch (OMP_CLAUSE_CODE (t))
140 {
141 case OMP_CLAUSE_INBRANCH:
142 clone_info->inbranch = 1;
143 *inbranch_specified = true;
144 break;
145 case OMP_CLAUSE_NOTINBRANCH:
146 clone_info->inbranch = 0;
147 *inbranch_specified = true;
148 break;
149 case OMP_CLAUSE_SIMDLEN:
150 clone_info->simdlen
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
152 break;
153 case OMP_CLAUSE_LINEAR:
154 {
155 tree decl = OMP_CLAUSE_DECL (t);
156 tree step = OMP_CLAUSE_LINEAR_STEP (t);
157 int argno = TREE_INT_CST_LOW (decl);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
159 {
160 enum cgraph_simd_clone_arg_type arg_type;
161 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
162 switch (OMP_CLAUSE_LINEAR_KIND (t))
163 {
164 case OMP_CLAUSE_LINEAR_REF:
165 arg_type
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
167 break;
168 case OMP_CLAUSE_LINEAR_UVAL:
169 arg_type
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
171 break;
172 case OMP_CLAUSE_LINEAR_VAL:
173 case OMP_CLAUSE_LINEAR_DEFAULT:
174 arg_type
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
176 break;
177 default:
178 gcc_unreachable ();
179 }
180 else
181 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
182 clone_info->args[argno].arg_type = arg_type;
183 clone_info->args[argno].linear_step = tree_to_shwi (step);
184 gcc_assert (clone_info->args[argno].linear_step >= 0
185 && clone_info->args[argno].linear_step < n);
186 }
187 else
188 {
189 if (POINTER_TYPE_P (args[argno]))
190 step = fold_convert (ssizetype, step);
191 if (!tree_fits_shwi_p (step))
192 {
193 warning_at (OMP_CLAUSE_LOCATION (t), 0,
194 "ignoring large linear step");
195 args.release ();
196 return NULL;
197 }
198 else if (integer_zerop (step))
199 {
200 warning_at (OMP_CLAUSE_LOCATION (t), 0,
201 "ignoring zero linear step");
202 args.release ();
203 return NULL;
204 }
205 else
206 {
207 enum cgraph_simd_clone_arg_type arg_type;
208 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
209 switch (OMP_CLAUSE_LINEAR_KIND (t))
210 {
211 case OMP_CLAUSE_LINEAR_REF:
212 arg_type
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
214 break;
215 case OMP_CLAUSE_LINEAR_UVAL:
216 arg_type
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
218 break;
219 case OMP_CLAUSE_LINEAR_VAL:
220 case OMP_CLAUSE_LINEAR_DEFAULT:
221 arg_type
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
223 break;
224 default:
225 gcc_unreachable ();
226 }
227 else
228 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
229 clone_info->args[argno].arg_type = arg_type;
230 clone_info->args[argno].linear_step = tree_to_shwi (step);
231 }
232 }
233 break;
234 }
235 case OMP_CLAUSE_UNIFORM:
236 {
237 tree decl = OMP_CLAUSE_DECL (t);
238 int argno = tree_to_uhwi (decl);
239 clone_info->args[argno].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM;
241 break;
242 }
243 case OMP_CLAUSE_ALIGNED:
244 {
245 /* Ignore aligned (x) for declare simd, for the ABI we really
246 need an alignment specified. */
247 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
248 break;
249 tree decl = OMP_CLAUSE_DECL (t);
250 int argno = tree_to_uhwi (decl);
251 clone_info->args[argno].alignment
252 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
253 break;
254 }
255 default:
256 break;
257 }
258 }
259
260 out:
261 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
262 {
263 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
264 "ignoring %<#pragma omp declare simd%> on function "
265 "with %<_Atomic%> qualified return type");
266 args.release ();
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 args.release ();
282 return clone_info;
283 }
284
285 /* Given a SIMD clone in NODE, calculate the characteristic data
286 type and return the coresponding type. The characteristic data
287 type is computed as described in the Intel Vector ABI. */
288
289 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)290 simd_clone_compute_base_data_type (struct cgraph_node *node,
291 struct cgraph_simd_clone *clone_info)
292 {
293 tree type = integer_type_node;
294 tree fndecl = node->decl;
295
296 /* a) For non-void function, the characteristic data type is the
297 return type. */
298 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
299 type = TREE_TYPE (TREE_TYPE (fndecl));
300
301 /* b) If the function has any non-uniform, non-linear parameters,
302 then the characteristic data type is the type of the first
303 such parameter. */
304 else
305 {
306 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 map.release ();
314 }
315
316 /* c) If the characteristic data type determined by a) or b) above
317 is struct, union, or class type which is pass-by-value (except
318 for the type that maps to the built-in complex data type), the
319 characteristic data type is int. */
320 if (RECORD_OR_UNION_TYPE_P (type)
321 && !aggregate_value_p (type, NULL)
322 && TREE_CODE (type) != COMPLEX_TYPE)
323 return integer_type_node;
324
325 /* d) If none of the above three classes is applicable, the
326 characteristic data type is int. */
327
328 return type;
329
330 /* e) For Intel Xeon Phi native and offload compilation, if the
331 resulting characteristic data type is 8-bit or 16-bit integer
332 data type, the characteristic data type is int. */
333 /* Well, we don't handle Xeon Phi yet. */
334 }
335
336 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)337 simd_clone_mangle (struct cgraph_node *node,
338 struct cgraph_simd_clone *clone_info)
339 {
340 char vecsize_mangle = clone_info->vecsize_mangle;
341 char mask = clone_info->inbranch ? 'M' : 'N';
342 unsigned int simdlen = clone_info->simdlen;
343 unsigned int n;
344 pretty_printer pp;
345
346 gcc_assert (vecsize_mangle && simdlen);
347
348 pp_string (&pp, "_ZGV");
349 pp_character (&pp, vecsize_mangle);
350 pp_character (&pp, mask);
351 pp_decimal_int (&pp, simdlen);
352
353 for (n = 0; n < clone_info->nargs; ++n)
354 {
355 struct cgraph_simd_clone_arg arg = clone_info->args[n];
356
357 switch (arg.arg_type)
358 {
359 case SIMD_CLONE_ARG_TYPE_UNIFORM:
360 pp_character (&pp, 'u');
361 break;
362 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
363 pp_character (&pp, 'l');
364 goto mangle_linear;
365 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
366 pp_character (&pp, 'R');
367 goto mangle_linear;
368 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
369 pp_character (&pp, 'L');
370 goto mangle_linear;
371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
372 pp_character (&pp, 'U');
373 goto mangle_linear;
374 mangle_linear:
375 gcc_assert (arg.linear_step != 0);
376 if (arg.linear_step > 1)
377 pp_unsigned_wide_integer (&pp, arg.linear_step);
378 else if (arg.linear_step < 0)
379 {
380 pp_character (&pp, 'n');
381 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
382 arg.linear_step));
383 }
384 break;
385 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
386 pp_string (&pp, "ls");
387 pp_unsigned_wide_integer (&pp, arg.linear_step);
388 break;
389 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
390 pp_string (&pp, "Rs");
391 pp_unsigned_wide_integer (&pp, arg.linear_step);
392 break;
393 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
394 pp_string (&pp, "Ls");
395 pp_unsigned_wide_integer (&pp, arg.linear_step);
396 break;
397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
398 pp_string (&pp, "Us");
399 pp_unsigned_wide_integer (&pp, arg.linear_step);
400 break;
401 default:
402 pp_character (&pp, 'v');
403 }
404 if (arg.alignment)
405 {
406 pp_character (&pp, 'a');
407 pp_decimal_int (&pp, arg.alignment);
408 }
409 }
410
411 pp_underscore (&pp);
412 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
413 if (*str == '*')
414 ++str;
415 pp_string (&pp, str);
416 str = pp_formatted_text (&pp);
417
418 /* If there already is a SIMD clone with the same mangled name, don't
419 add another one. This can happen e.g. for
420 #pragma omp declare simd
421 #pragma omp declare simd simdlen(8)
422 int foo (int, int);
423 if the simdlen is assumed to be 8 for the first one, etc. */
424 for (struct cgraph_node *clone = node->simd_clones; clone;
425 clone = clone->simdclone->next_clone)
426 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
427 return NULL_TREE;
428
429 return get_identifier (str);
430 }
431
432 /* Create a simd clone of OLD_NODE and return it. */
433
434 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)435 simd_clone_create (struct cgraph_node *old_node)
436 {
437 struct cgraph_node *new_node;
438 if (old_node->definition)
439 {
440 if (!old_node->has_gimple_body_p ())
441 return NULL;
442 old_node->get_body ();
443 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
444 false, NULL, NULL,
445 "simdclone");
446 }
447 else
448 {
449 tree old_decl = old_node->decl;
450 tree new_decl = copy_node (old_node->decl);
451 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
452 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
453 SET_DECL_RTL (new_decl, NULL);
454 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
455 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
456 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
457 if (old_node->in_other_partition)
458 new_node->in_other_partition = 1;
459 }
460 if (new_node == NULL)
461 return new_node;
462
463 DECL_BUILT_IN_CLASS (new_node->decl) = NOT_BUILT_IN;
464 DECL_FUNCTION_CODE (new_node->decl) = (enum built_in_function) 0;
465 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
466 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
467 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
468 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
469 DECL_VISIBILITY_SPECIFIED (new_node->decl)
470 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
471 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
472 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
473 if (DECL_ONE_ONLY (old_node->decl))
474 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
475
476 /* The method cgraph_version_clone_with_body () will force the new
477 symbol local. Undo this, and inherit external visibility from
478 the old node. */
479 new_node->local.local = old_node->local.local;
480 new_node->externally_visible = old_node->externally_visible;
481
482 return new_node;
483 }
484
485 /* Adjust the return type of the given function to its appropriate
486 vector counterpart. Returns a simd array to be used throughout the
487 function as a return value. */
488
489 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)490 simd_clone_adjust_return_type (struct cgraph_node *node)
491 {
492 tree fndecl = node->decl;
493 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
494 unsigned int veclen;
495 tree t;
496
497 /* Adjust the function return type. */
498 if (orig_rettype == void_type_node)
499 return NULL_TREE;
500 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
501 t = TREE_TYPE (TREE_TYPE (fndecl));
502 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
503 veclen = node->simdclone->vecsize_int;
504 else
505 veclen = node->simdclone->vecsize_float;
506 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
507 if (veclen > node->simdclone->simdlen)
508 veclen = node->simdclone->simdlen;
509 if (POINTER_TYPE_P (t))
510 t = pointer_sized_int_node;
511 if (veclen == node->simdclone->simdlen)
512 t = build_vector_type (t, node->simdclone->simdlen);
513 else
514 {
515 t = build_vector_type (t, veclen);
516 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
517 }
518 TREE_TYPE (TREE_TYPE (fndecl)) = t;
519 if (!node->definition)
520 return NULL_TREE;
521
522 t = DECL_RESULT (fndecl);
523 /* Adjust the DECL_RESULT. */
524 gcc_assert (TREE_TYPE (t) != void_type_node);
525 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
526 relayout_decl (t);
527
528 tree atype = build_array_type_nelts (orig_rettype,
529 node->simdclone->simdlen);
530 if (veclen != node->simdclone->simdlen)
531 return build1 (VIEW_CONVERT_EXPR, atype, t);
532
533 /* Set up a SIMD array to use as the return value. */
534 tree retval = create_tmp_var_raw (atype, "retval");
535 gimple_add_tmp_var (retval);
536 return retval;
537 }
538
539 /* Each vector argument has a corresponding array to be used locally
540 as part of the eventual loop. Create such temporary array and
541 return it.
542
543 PREFIX is the prefix to be used for the temporary.
544
545 TYPE is the inner element type.
546
547 SIMDLEN is the number of elements. */
548
549 static tree
create_tmp_simd_array(const char * prefix,tree type,int simdlen)550 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
551 {
552 tree atype = build_array_type_nelts (type, simdlen);
553 tree avar = create_tmp_var_raw (atype, prefix);
554 gimple_add_tmp_var (avar);
555 return avar;
556 }
557
558 /* Modify the function argument types to their corresponding vector
559 counterparts if appropriate. Also, create one array for each simd
560 argument to be used locally when using the function arguments as
561 part of the loop.
562
563 NODE is the function whose arguments are to be adjusted.
564
565 Returns an adjustment vector that will be filled describing how the
566 argument types will be adjusted. */
567
568 static ipa_parm_adjustment_vec
simd_clone_adjust_argument_types(struct cgraph_node * node)569 simd_clone_adjust_argument_types (struct cgraph_node *node)
570 {
571 vec<tree> args;
572 ipa_parm_adjustment_vec adjustments;
573
574 if (node->definition)
575 args = ipa_get_vector_of_formal_parms (node->decl);
576 else
577 args = simd_clone_vector_of_formal_parm_types (node->decl);
578 adjustments.create (args.length ());
579 unsigned i, j, veclen;
580 struct ipa_parm_adjustment adj;
581 struct cgraph_simd_clone *sc = node->simdclone;
582
583 for (i = 0; i < sc->nargs; ++i)
584 {
585 memset (&adj, 0, sizeof (adj));
586 tree parm = args[i];
587 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
588 adj.base_index = i;
589 adj.base = parm;
590
591 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
592 sc->args[i].orig_type = parm_type;
593
594 switch (sc->args[i].arg_type)
595 {
596 default:
597 /* No adjustment necessary for scalar arguments. */
598 adj.op = IPA_PARM_OP_COPY;
599 break;
600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
602 if (node->definition)
603 sc->args[i].simd_array
604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
605 TREE_TYPE (parm_type),
606 sc->simdlen);
607 adj.op = IPA_PARM_OP_COPY;
608 break;
609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
611 case SIMD_CLONE_ARG_TYPE_VECTOR:
612 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
613 veclen = sc->vecsize_int;
614 else
615 veclen = sc->vecsize_float;
616 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
617 if (veclen > sc->simdlen)
618 veclen = sc->simdlen;
619 adj.arg_prefix = "simd";
620 if (POINTER_TYPE_P (parm_type))
621 adj.type = build_vector_type (pointer_sized_int_node, veclen);
622 else
623 adj.type = build_vector_type (parm_type, veclen);
624 sc->args[i].vector_type = adj.type;
625 for (j = veclen; j < sc->simdlen; j += veclen)
626 {
627 adjustments.safe_push (adj);
628 if (j == veclen)
629 {
630 memset (&adj, 0, sizeof (adj));
631 adj.op = IPA_PARM_OP_NEW;
632 adj.arg_prefix = "simd";
633 adj.base_index = i;
634 adj.type = sc->args[i].vector_type;
635 }
636 }
637
638 if (node->definition)
639 sc->args[i].simd_array
640 = create_tmp_simd_array (DECL_NAME (parm)
641 ? IDENTIFIER_POINTER (DECL_NAME (parm))
642 : NULL, parm_type, sc->simdlen);
643 }
644 adjustments.safe_push (adj);
645 }
646
647 if (sc->inbranch)
648 {
649 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
650
651 memset (&adj, 0, sizeof (adj));
652 adj.op = IPA_PARM_OP_NEW;
653 adj.arg_prefix = "mask";
654
655 adj.base_index = i;
656 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
657 veclen = sc->vecsize_int;
658 else
659 veclen = sc->vecsize_float;
660 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
661 if (veclen > sc->simdlen)
662 veclen = sc->simdlen;
663 if (sc->mask_mode != VOIDmode)
664 adj.type
665 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
666 else if (POINTER_TYPE_P (base_type))
667 adj.type = build_vector_type (pointer_sized_int_node, veclen);
668 else
669 adj.type = build_vector_type (base_type, veclen);
670 adjustments.safe_push (adj);
671
672 for (j = veclen; j < sc->simdlen; j += veclen)
673 adjustments.safe_push (adj);
674
675 /* We have previously allocated one extra entry for the mask. Use
676 it and fill it. */
677 sc->nargs++;
678 if (sc->mask_mode != VOIDmode)
679 base_type = boolean_type_node;
680 if (node->definition)
681 {
682 sc->args[i].orig_arg
683 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
684 if (sc->mask_mode == VOIDmode)
685 sc->args[i].simd_array
686 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
687 else if (veclen < sc->simdlen)
688 sc->args[i].simd_array
689 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
690 else
691 sc->args[i].simd_array = NULL_TREE;
692 }
693 sc->args[i].orig_type = base_type;
694 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
695 }
696
697 if (node->definition)
698 ipa_modify_formal_parameters (node->decl, adjustments);
699 else
700 {
701 tree new_arg_types = NULL_TREE, new_reversed;
702 bool last_parm_void = false;
703 if (args.length () > 0 && args.last () == void_type_node)
704 last_parm_void = true;
705
706 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
707 j = adjustments.length ();
708 for (i = 0; i < j; i++)
709 {
710 struct ipa_parm_adjustment *adj = &adjustments[i];
711 tree ptype;
712 if (adj->op == IPA_PARM_OP_COPY)
713 ptype = args[adj->base_index];
714 else
715 ptype = adj->type;
716 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
717 }
718 new_reversed = nreverse (new_arg_types);
719 if (last_parm_void)
720 {
721 if (new_reversed)
722 TREE_CHAIN (new_arg_types) = void_list_node;
723 else
724 new_reversed = void_list_node;
725 }
726
727 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
728 TYPE_ARG_TYPES (new_type) = new_reversed;
729 TREE_TYPE (node->decl) = new_type;
730
731 adjustments.release ();
732 }
733 args.release ();
734 return adjustments;
735 }
736
737 /* Initialize and copy the function arguments in NODE to their
738 corresponding local simd arrays. Returns a fresh gimple_seq with
739 the instruction sequence generated. */
740
741 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments)742 simd_clone_init_simd_arrays (struct cgraph_node *node,
743 ipa_parm_adjustment_vec adjustments)
744 {
745 gimple_seq seq = NULL;
746 unsigned i = 0, j = 0, k;
747
748 for (tree arg = DECL_ARGUMENTS (node->decl);
749 arg;
750 arg = DECL_CHAIN (arg), i++, j++)
751 {
752 if (adjustments[j].op == IPA_PARM_OP_COPY
753 || POINTER_TYPE_P (TREE_TYPE (arg)))
754 continue;
755
756 node->simdclone->args[i].vector_arg = arg;
757
758 tree array = node->simdclone->args[i].simd_array;
759 if (node->simdclone->mask_mode != VOIDmode
760 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
761 {
762 if (array == NULL_TREE)
763 continue;
764 unsigned int l
765 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
766 for (k = 0; k <= l; k++)
767 {
768 if (k)
769 {
770 arg = DECL_CHAIN (arg);
771 j++;
772 }
773 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
774 array, size_int (k), NULL, NULL);
775 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
776 gimplify_and_add (t, &seq);
777 }
778 continue;
779 }
780 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen)
781 {
782 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
783 tree ptr = build_fold_addr_expr (array);
784 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
785 build_int_cst (ptype, 0));
786 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
787 gimplify_and_add (t, &seq);
788 }
789 else
790 {
791 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
792 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
793 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
794 {
795 tree ptr = build_fold_addr_expr (array);
796 int elemsize;
797 if (k)
798 {
799 arg = DECL_CHAIN (arg);
800 j++;
801 }
802 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
803 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
804 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
805 build_int_cst (ptype, k * elemsize));
806 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
807 gimplify_and_add (t, &seq);
808 }
809 }
810 }
811 return seq;
812 }
813
814 /* Callback info for ipa_simd_modify_stmt_ops below. */
815
816 struct modify_stmt_info {
817 ipa_parm_adjustment_vec adjustments;
818 gimple *stmt;
819 gimple *after_stmt;
820 /* True if the parent statement was modified by
821 ipa_simd_modify_stmt_ops. */
822 bool modified;
823 };
824
825 /* Callback for walk_gimple_op.
826
827 Adjust operands from a given statement as specified in the
828 adjustments vector in the callback data. */
829
830 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)831 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
832 {
833 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
834 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
835 tree *orig_tp = tp;
836 if (TREE_CODE (*tp) == ADDR_EXPR)
837 tp = &TREE_OPERAND (*tp, 0);
838 struct ipa_parm_adjustment *cand = NULL;
839 if (TREE_CODE (*tp) == PARM_DECL)
840 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
841 else
842 {
843 if (TYPE_P (*tp))
844 *walk_subtrees = 0;
845 }
846
847 tree repl = NULL_TREE;
848 if (cand)
849 repl = unshare_expr (cand->new_decl);
850 else
851 {
852 if (tp != orig_tp)
853 {
854 *walk_subtrees = 0;
855 bool modified = info->modified;
856 info->modified = false;
857 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
858 if (!info->modified)
859 {
860 info->modified = modified;
861 return NULL_TREE;
862 }
863 info->modified = modified;
864 repl = *tp;
865 }
866 else
867 return NULL_TREE;
868 }
869
870 if (tp != orig_tp)
871 {
872 if (gimple_code (info->stmt) == GIMPLE_PHI
873 && cand
874 && TREE_CODE (*orig_tp) == ADDR_EXPR
875 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
876 && cand->alias_ptr_type)
877 {
878 gcc_assert (TREE_CODE (cand->alias_ptr_type) == SSA_NAME);
879 *orig_tp = cand->alias_ptr_type;
880 info->modified = true;
881 return NULL_TREE;
882 }
883
884 repl = build_fold_addr_expr (repl);
885 gimple *stmt;
886 if (is_gimple_debug (info->stmt))
887 {
888 tree vexpr = make_node (DEBUG_EXPR_DECL);
889 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
890 DECL_ARTIFICIAL (vexpr) = 1;
891 TREE_TYPE (vexpr) = TREE_TYPE (repl);
892 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
893 repl = vexpr;
894 }
895 else
896 {
897 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
898 repl = gimple_assign_lhs (stmt);
899 }
900 gimple_stmt_iterator gsi;
901 if (gimple_code (info->stmt) == GIMPLE_PHI)
902 {
903 if (info->after_stmt)
904 gsi = gsi_for_stmt (info->after_stmt);
905 else
906 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
907 /* Cache SSA_NAME for next time. */
908 if (cand
909 && TREE_CODE (*orig_tp) == ADDR_EXPR
910 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
911 cand->alias_ptr_type = repl;
912 }
913 else
914 gsi = gsi_for_stmt (info->stmt);
915 if (info->after_stmt)
916 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
917 else
918 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
919 if (gimple_code (info->stmt) == GIMPLE_PHI)
920 info->after_stmt = stmt;
921 *orig_tp = repl;
922 }
923 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
924 {
925 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
926 *tp = vce;
927 }
928 else
929 *tp = repl;
930
931 info->modified = true;
932 return NULL_TREE;
933 }
934
935 /* Traverse the function body and perform all modifications as
936 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
937 modified such that the replacement/reduction value will now be an
938 offset into the corresponding simd_array.
939
940 This function will replace all function argument uses with their
941 corresponding simd array elements, and ajust the return values
942 accordingly. */
943
944 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments,tree retval_array,tree iter)945 ipa_simd_modify_function_body (struct cgraph_node *node,
946 ipa_parm_adjustment_vec adjustments,
947 tree retval_array, tree iter)
948 {
949 basic_block bb;
950 unsigned int i, j, l;
951
952 /* Re-use the adjustments array, but this time use it to replace
953 every function argument use to an offset into the corresponding
954 simd_array. */
955 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
956 {
957 if (!node->simdclone->args[i].vector_arg)
958 continue;
959
960 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
961 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
962 adjustments[j].new_decl
963 = build4 (ARRAY_REF,
964 basetype,
965 node->simdclone->args[i].simd_array,
966 iter,
967 NULL_TREE, NULL_TREE);
968 if (adjustments[j].op == IPA_PARM_OP_NONE
969 && simd_clone_subparts (vectype) < node->simdclone->simdlen)
970 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1;
971 }
972
973 l = adjustments.length ();
974 tree name;
975
976 FOR_EACH_SSA_NAME (i, name, cfun)
977 {
978 if (SSA_NAME_VAR (name)
979 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
980 {
981 for (j = 0; j < l; j++)
982 if (SSA_NAME_VAR (name) == adjustments[j].base
983 && adjustments[j].new_decl)
984 {
985 tree base_var;
986 if (adjustments[j].new_ssa_base == NULL_TREE)
987 {
988 base_var
989 = copy_var_decl (adjustments[j].base,
990 DECL_NAME (adjustments[j].base),
991 TREE_TYPE (adjustments[j].base));
992 adjustments[j].new_ssa_base = base_var;
993 }
994 else
995 base_var = adjustments[j].new_ssa_base;
996 if (SSA_NAME_IS_DEFAULT_DEF (name))
997 {
998 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
999 gimple_stmt_iterator gsi = gsi_after_labels (bb);
1000 tree new_decl = unshare_expr (adjustments[j].new_decl);
1001 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
1002 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1003 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
1004 gimple *stmt = gimple_build_assign (name, new_decl);
1005 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1006 }
1007 else
1008 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1009 }
1010 }
1011 }
1012
1013 struct modify_stmt_info info;
1014 info.adjustments = adjustments;
1015
1016 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1017 {
1018 gimple_stmt_iterator gsi;
1019
1020 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1021 {
1022 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1023 int i, n = gimple_phi_num_args (phi);
1024 info.stmt = phi;
1025 info.after_stmt = NULL;
1026 struct walk_stmt_info wi;
1027 memset (&wi, 0, sizeof (wi));
1028 info.modified = false;
1029 wi.info = &info;
1030 for (i = 0; i < n; ++i)
1031 {
1032 int walk_subtrees = 1;
1033 tree arg = gimple_phi_arg_def (phi, i);
1034 tree op = arg;
1035 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1036 if (op != arg)
1037 {
1038 SET_PHI_ARG_DEF (phi, i, op);
1039 gcc_assert (TREE_CODE (op) == SSA_NAME);
1040 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1041 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1042 }
1043 }
1044 }
1045
1046 gsi = gsi_start_bb (bb);
1047 while (!gsi_end_p (gsi))
1048 {
1049 gimple *stmt = gsi_stmt (gsi);
1050 info.stmt = stmt;
1051 info.after_stmt = NULL;
1052 struct walk_stmt_info wi;
1053
1054 memset (&wi, 0, sizeof (wi));
1055 info.modified = false;
1056 wi.info = &info;
1057 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1058
1059 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1060 {
1061 tree retval = gimple_return_retval (return_stmt);
1062 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1063 e->flags |= EDGE_FALLTHRU;
1064 if (!retval)
1065 {
1066 gsi_remove (&gsi, true);
1067 continue;
1068 }
1069
1070 /* Replace `return foo' with `retval_array[iter] = foo'. */
1071 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1072 retval_array, iter, NULL, NULL);
1073 stmt = gimple_build_assign (ref, retval);
1074 gsi_replace (&gsi, stmt, true);
1075 info.modified = true;
1076 }
1077
1078 if (info.modified)
1079 {
1080 update_stmt (stmt);
1081 /* If the above changed the var of a debug bind into something
1082 different, remove the debug stmt. We could also for all the
1083 replaced parameters add VAR_DECLs for debug info purposes,
1084 add debug stmts for those to be the simd array accesses and
1085 replace debug stmt var operand with that var. Debugging of
1086 vectorized loops doesn't work too well, so don't bother for
1087 now. */
1088 if ((gimple_debug_bind_p (stmt)
1089 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1090 || (gimple_debug_source_bind_p (stmt)
1091 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1092 {
1093 gsi_remove (&gsi, true);
1094 continue;
1095 }
1096 if (maybe_clean_eh_stmt (stmt))
1097 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1098 }
1099 gsi_next (&gsi);
1100 }
1101 }
1102 }
1103
1104 /* Helper function of simd_clone_adjust, return linear step addend
1105 of Ith argument. */
1106
1107 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1108 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1109 tree addtype, basic_block entry_bb)
1110 {
1111 tree ptype = NULL_TREE;
1112 switch (node->simdclone->args[i].arg_type)
1113 {
1114 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1115 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1116 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1117 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1118 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1119 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1120 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1121 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1122 break;
1123 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1124 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1125 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1126 break;
1127 default:
1128 gcc_unreachable ();
1129 }
1130
1131 unsigned int idx = node->simdclone->args[i].linear_step;
1132 tree arg = node->simdclone->args[idx].orig_arg;
1133 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1134 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1135 gimple *g;
1136 tree ret;
1137 if (is_gimple_reg (arg))
1138 ret = get_or_create_ssa_default_def (cfun, arg);
1139 else
1140 {
1141 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1142 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1143 ret = gimple_assign_lhs (g);
1144 }
1145 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1146 {
1147 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1148 build_simple_mem_ref (ret));
1149 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1150 ret = gimple_assign_lhs (g);
1151 }
1152 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1153 {
1154 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1155 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1156 ret = gimple_assign_lhs (g);
1157 }
1158 if (POINTER_TYPE_P (ptype))
1159 {
1160 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1161 if (size && TREE_CODE (size) == INTEGER_CST)
1162 {
1163 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1164 ret, fold_convert (addtype, size));
1165 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1166 ret = gimple_assign_lhs (g);
1167 }
1168 }
1169 return ret;
1170 }
1171
1172 /* Adjust the argument types in NODE to their appropriate vector
1173 counterparts. */
1174
1175 static void
simd_clone_adjust(struct cgraph_node * node)1176 simd_clone_adjust (struct cgraph_node *node)
1177 {
1178 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1179
1180 targetm.simd_clone.adjust (node);
1181
1182 tree retval = simd_clone_adjust_return_type (node);
1183 ipa_parm_adjustment_vec adjustments
1184 = simd_clone_adjust_argument_types (node);
1185
1186 push_gimplify_context ();
1187
1188 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1189
1190 /* Adjust all uses of vector arguments accordingly. Adjust all
1191 return values accordingly. */
1192 tree iter = create_tmp_var (unsigned_type_node, "iter");
1193 tree iter1 = make_ssa_name (iter);
1194 tree iter2 = NULL_TREE;
1195 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1196 adjustments.release ();
1197
1198 /* Initialize the iteration variable. */
1199 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1200 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1201 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1202 /* Insert the SIMD array and iv initialization at function
1203 entry. */
1204 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1205
1206 pop_gimplify_context (NULL);
1207
1208 gimple *g;
1209 basic_block incr_bb = NULL;
1210 struct loop *loop = NULL;
1211
1212 /* Create a new BB right before the original exit BB, to hold the
1213 iteration increment and the condition/branch. */
1214 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1215 {
1216 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1217 incr_bb = create_empty_bb (orig_exit);
1218 incr_bb->count = profile_count::zero ();
1219 add_bb_to_loop (incr_bb, body_bb->loop_father);
1220 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1221 {
1222 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1223 redirect_edge_succ (e, incr_bb);
1224 incr_bb->count += e->count ();
1225 }
1226 }
1227 else if (node->simdclone->inbranch)
1228 {
1229 incr_bb = create_empty_bb (entry_bb);
1230 incr_bb->count = profile_count::zero ();
1231 add_bb_to_loop (incr_bb, body_bb->loop_father);
1232 }
1233
1234 if (incr_bb)
1235 {
1236 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1237 gsi = gsi_last_bb (incr_bb);
1238 iter2 = make_ssa_name (iter);
1239 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1240 build_int_cst (unsigned_type_node, 1));
1241 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1242
1243 /* Mostly annotate the loop for the vectorizer (the rest is done
1244 below). */
1245 loop = alloc_loop ();
1246 cfun->has_force_vectorize_loops = true;
1247 loop->safelen = node->simdclone->simdlen;
1248 loop->force_vectorize = true;
1249 loop->header = body_bb;
1250 }
1251
1252 /* Branch around the body if the mask applies. */
1253 if (node->simdclone->inbranch)
1254 {
1255 gsi = gsi_last_bb (loop->header);
1256 tree mask_array
1257 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1258 tree mask;
1259 if (node->simdclone->mask_mode != VOIDmode)
1260 {
1261 tree shift_cnt;
1262 if (mask_array == NULL_TREE)
1263 {
1264 tree arg = node->simdclone->args[node->simdclone->nargs
1265 - 1].vector_arg;
1266 mask = get_or_create_ssa_default_def (cfun, arg);
1267 shift_cnt = iter1;
1268 }
1269 else
1270 {
1271 tree maskt = TREE_TYPE (mask_array);
1272 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1273 c = node->simdclone->simdlen / (c + 1);
1274 int s = exact_log2 (c);
1275 gcc_assert (s > 0);
1276 c--;
1277 tree idx = make_ssa_name (TREE_TYPE (iter1));
1278 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1279 build_int_cst (NULL_TREE, s));
1280 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1281 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1282 tree aref = build4 (ARRAY_REF,
1283 TREE_TYPE (TREE_TYPE (mask_array)),
1284 mask_array, idx, NULL, NULL);
1285 g = gimple_build_assign (mask, aref);
1286 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1287 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1288 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1289 build_int_cst (TREE_TYPE (iter1), c));
1290 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1291 }
1292 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1293 RSHIFT_EXPR, mask, shift_cnt);
1294 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1295 mask = gimple_assign_lhs (g);
1296 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1297 BIT_AND_EXPR, mask,
1298 build_int_cst (TREE_TYPE (mask), 1));
1299 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1300 mask = gimple_assign_lhs (g);
1301 }
1302 else
1303 {
1304 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1305 tree aref = build4 (ARRAY_REF,
1306 TREE_TYPE (TREE_TYPE (mask_array)),
1307 mask_array, iter1, NULL, NULL);
1308 g = gimple_build_assign (mask, aref);
1309 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1310 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1311 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1312 {
1313 aref = build1 (VIEW_CONVERT_EXPR,
1314 build_nonstandard_integer_type (bitsize, 0),
1315 mask);
1316 mask = make_ssa_name (TREE_TYPE (aref));
1317 g = gimple_build_assign (mask, aref);
1318 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1319 }
1320 }
1321
1322 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1323 NULL, NULL);
1324 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1325 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1326 e->probability = profile_probability::unlikely ().guessed ();
1327 incr_bb->count += e->count ();
1328 edge fallthru = FALLTHRU_EDGE (loop->header);
1329 fallthru->flags = EDGE_FALSE_VALUE;
1330 fallthru->probability = profile_probability::likely ().guessed ();
1331 }
1332
1333 basic_block latch_bb = NULL;
1334 basic_block new_exit_bb = NULL;
1335
1336 /* Generate the condition. */
1337 if (incr_bb)
1338 {
1339 gsi = gsi_last_bb (incr_bb);
1340 g = gimple_build_cond (LT_EXPR, iter2,
1341 build_int_cst (unsigned_type_node,
1342 node->simdclone->simdlen),
1343 NULL, NULL);
1344 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1345 edge e = split_block (incr_bb, gsi_stmt (gsi));
1346 latch_bb = e->dest;
1347 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1348 loop->latch = latch_bb;
1349
1350 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1351
1352 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1353
1354 /* FIXME: Do we need to distribute probabilities for the conditional? */
1355 new_e->probability = profile_probability::guessed_never ();
1356 /* The successor of incr_bb is already pointing to latch_bb; just
1357 change the flags.
1358 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1359 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1360 }
1361
1362 gphi *phi = create_phi_node (iter1, body_bb);
1363 edge preheader_edge = find_edge (entry_bb, body_bb);
1364 edge latch_edge = NULL;
1365 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1366 UNKNOWN_LOCATION);
1367 if (incr_bb)
1368 {
1369 latch_edge = single_succ_edge (latch_bb);
1370 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1371
1372 /* Generate the new return. */
1373 gsi = gsi_last_bb (new_exit_bb);
1374 if (retval
1375 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1376 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1377 retval = TREE_OPERAND (retval, 0);
1378 else if (retval)
1379 {
1380 retval = build1 (VIEW_CONVERT_EXPR,
1381 TREE_TYPE (TREE_TYPE (node->decl)),
1382 retval);
1383 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1384 false, GSI_CONTINUE_LINKING);
1385 }
1386 g = gimple_build_return (retval);
1387 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1388 }
1389
1390 /* Handle aligned clauses by replacing default defs of the aligned
1391 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1392 lhs. Handle linear by adding PHIs. */
1393 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1394 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1395 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1396 || !is_gimple_reg_type
1397 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1398 {
1399 tree orig_arg = node->simdclone->args[i].orig_arg;
1400 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1401 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1402 else
1403 {
1404 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1405 gimple_add_tmp_var (iter1);
1406 }
1407 gsi = gsi_after_labels (entry_bb);
1408 g = gimple_build_assign (iter1, orig_arg);
1409 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1410 gsi = gsi_after_labels (body_bb);
1411 g = gimple_build_assign (orig_arg, iter1);
1412 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1413 }
1414 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1415 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1416 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1417 == REFERENCE_TYPE
1418 && TREE_ADDRESSABLE
1419 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1420 {
1421 tree orig_arg = node->simdclone->args[i].orig_arg;
1422 tree def = ssa_default_def (cfun, orig_arg);
1423 if (def && !has_zero_uses (def))
1424 {
1425 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1426 gimple_add_tmp_var (iter1);
1427 gsi = gsi_after_labels (entry_bb);
1428 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1429 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1430 gsi = gsi_after_labels (body_bb);
1431 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1432 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1433 }
1434 }
1435 else if (node->simdclone->args[i].alignment
1436 && node->simdclone->args[i].arg_type
1437 == SIMD_CLONE_ARG_TYPE_UNIFORM
1438 && (node->simdclone->args[i].alignment
1439 & (node->simdclone->args[i].alignment - 1)) == 0
1440 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1441 == POINTER_TYPE)
1442 {
1443 unsigned int alignment = node->simdclone->args[i].alignment;
1444 tree orig_arg = node->simdclone->args[i].orig_arg;
1445 tree def = ssa_default_def (cfun, orig_arg);
1446 if (def && !has_zero_uses (def))
1447 {
1448 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1449 gimple_seq seq = NULL;
1450 bool need_cvt = false;
1451 gcall *call
1452 = gimple_build_call (fn, 2, def, size_int (alignment));
1453 g = call;
1454 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1455 ptr_type_node))
1456 need_cvt = true;
1457 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1458 gimple_call_set_lhs (g, t);
1459 gimple_seq_add_stmt_without_update (&seq, g);
1460 if (need_cvt)
1461 {
1462 t = make_ssa_name (orig_arg);
1463 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1464 gimple_seq_add_stmt_without_update (&seq, g);
1465 }
1466 gsi_insert_seq_on_edge_immediate
1467 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1468
1469 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1470 node->create_edge (cgraph_node::get_create (fn),
1471 call, entry_bb->count);
1472
1473 imm_use_iterator iter;
1474 use_operand_p use_p;
1475 gimple *use_stmt;
1476 tree repl = gimple_get_lhs (g);
1477 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1478 if (is_gimple_debug (use_stmt) || use_stmt == call)
1479 continue;
1480 else
1481 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1482 SET_USE (use_p, repl);
1483 }
1484 }
1485 else if ((node->simdclone->args[i].arg_type
1486 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1487 || (node->simdclone->args[i].arg_type
1488 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1489 || (node->simdclone->args[i].arg_type
1490 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1491 || (node->simdclone->args[i].arg_type
1492 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1493 {
1494 tree orig_arg = node->simdclone->args[i].orig_arg;
1495 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1496 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1497 tree def = NULL_TREE;
1498 if (TREE_ADDRESSABLE (orig_arg))
1499 {
1500 def = make_ssa_name (TREE_TYPE (orig_arg));
1501 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1502 if (incr_bb)
1503 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1504 gsi = gsi_after_labels (entry_bb);
1505 g = gimple_build_assign (def, orig_arg);
1506 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1507 }
1508 else
1509 {
1510 def = ssa_default_def (cfun, orig_arg);
1511 if (!def || has_zero_uses (def))
1512 def = NULL_TREE;
1513 else
1514 {
1515 iter1 = make_ssa_name (orig_arg);
1516 if (incr_bb)
1517 iter2 = make_ssa_name (orig_arg);
1518 }
1519 }
1520 if (def)
1521 {
1522 phi = create_phi_node (iter1, body_bb);
1523 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1524 if (incr_bb)
1525 {
1526 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1527 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1528 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1529 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1530 ? TREE_TYPE (orig_arg) : sizetype;
1531 tree addcst = simd_clone_linear_addend (node, i, addtype,
1532 entry_bb);
1533 gsi = gsi_last_bb (incr_bb);
1534 g = gimple_build_assign (iter2, code, iter1, addcst);
1535 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1536 }
1537
1538 imm_use_iterator iter;
1539 use_operand_p use_p;
1540 gimple *use_stmt;
1541 if (TREE_ADDRESSABLE (orig_arg))
1542 {
1543 gsi = gsi_after_labels (body_bb);
1544 g = gimple_build_assign (orig_arg, iter1);
1545 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1546 }
1547 else
1548 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1549 if (use_stmt == phi)
1550 continue;
1551 else
1552 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1553 SET_USE (use_p, iter1);
1554 }
1555 }
1556 else if (node->simdclone->args[i].arg_type
1557 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1558 || (node->simdclone->args[i].arg_type
1559 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1560 {
1561 tree orig_arg = node->simdclone->args[i].orig_arg;
1562 tree def = ssa_default_def (cfun, orig_arg);
1563 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1564 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1565 if (def && !has_zero_uses (def))
1566 {
1567 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1568 iter1 = make_ssa_name (orig_arg);
1569 if (incr_bb)
1570 iter2 = make_ssa_name (orig_arg);
1571 tree iter3 = make_ssa_name (rtype);
1572 tree iter4 = make_ssa_name (rtype);
1573 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1574 gsi = gsi_after_labels (entry_bb);
1575 gimple *load
1576 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1577 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1578
1579 tree array = node->simdclone->args[i].simd_array;
1580 TREE_ADDRESSABLE (array) = 1;
1581 tree ptr = build_fold_addr_expr (array);
1582 phi = create_phi_node (iter1, body_bb);
1583 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1584 if (incr_bb)
1585 {
1586 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1587 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1588 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1589 gsi = gsi_last_bb (incr_bb);
1590 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1591 }
1592
1593 phi = create_phi_node (iter4, body_bb);
1594 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1595 if (incr_bb)
1596 {
1597 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1598 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1599 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1600 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1601 ? TREE_TYPE (iter3) : sizetype;
1602 tree addcst = simd_clone_linear_addend (node, i, addtype,
1603 entry_bb);
1604 g = gimple_build_assign (iter5, code, iter4, addcst);
1605 gsi = gsi_last_bb (incr_bb);
1606 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1607 }
1608
1609 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1610 gsi = gsi_after_labels (body_bb);
1611 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1612
1613 imm_use_iterator iter;
1614 use_operand_p use_p;
1615 gimple *use_stmt;
1616 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1617 if (use_stmt == load)
1618 continue;
1619 else
1620 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1621 SET_USE (use_p, iter1);
1622
1623 if (!TYPE_READONLY (rtype) && incr_bb)
1624 {
1625 tree v = make_ssa_name (rtype);
1626 tree aref = build4 (ARRAY_REF, rtype, array,
1627 size_zero_node, NULL_TREE,
1628 NULL_TREE);
1629 gsi = gsi_after_labels (new_exit_bb);
1630 g = gimple_build_assign (v, aref);
1631 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1632 g = gimple_build_assign (build_simple_mem_ref (def), v);
1633 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1634 }
1635 }
1636 }
1637
1638 calculate_dominance_info (CDI_DOMINATORS);
1639 if (loop)
1640 add_loop (loop, loop->header->loop_father);
1641 update_ssa (TODO_update_ssa);
1642
1643 pop_cfun ();
1644 }
1645
1646 /* If the function in NODE is tagged as an elemental SIMD function,
1647 create the appropriate SIMD clones. */
1648
1649 void
expand_simd_clones(struct cgraph_node * node)1650 expand_simd_clones (struct cgraph_node *node)
1651 {
1652 tree attr = lookup_attribute ("omp declare simd",
1653 DECL_ATTRIBUTES (node->decl));
1654 if (attr == NULL_TREE
1655 || node->global.inlined_to
1656 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1657 return;
1658
1659 /* Ignore
1660 #pragma omp declare simd
1661 extern int foo ();
1662 in C, there we don't know the argument types at all. */
1663 if (!node->definition
1664 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1665 return;
1666
1667 /* Call this before creating clone_info, as it might ggc_collect. */
1668 if (node->definition && node->has_gimple_body_p ())
1669 node->get_body ();
1670
1671 do
1672 {
1673 /* Start with parsing the "omp declare simd" attribute(s). */
1674 bool inbranch_clause_specified;
1675 struct cgraph_simd_clone *clone_info
1676 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1677 &inbranch_clause_specified);
1678 if (clone_info == NULL)
1679 continue;
1680
1681 int orig_simdlen = clone_info->simdlen;
1682 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1683 /* The target can return 0 (no simd clones should be created),
1684 1 (just one ISA of simd clones should be created) or higher
1685 count of ISA variants. In that case, clone_info is initialized
1686 for the first ISA variant. */
1687 int count
1688 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1689 base_type, 0);
1690 if (count == 0)
1691 continue;
1692
1693 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1694 also create one inbranch and one !inbranch clone of it. */
1695 for (int i = 0; i < count * 2; i++)
1696 {
1697 struct cgraph_simd_clone *clone = clone_info;
1698 if (inbranch_clause_specified && (i & 1) != 0)
1699 continue;
1700
1701 if (i != 0)
1702 {
1703 clone = simd_clone_struct_alloc (clone_info->nargs
1704 + ((i & 1) != 0));
1705 simd_clone_struct_copy (clone, clone_info);
1706 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1707 and simd_clone_adjust_argument_types did to the first
1708 clone's info. */
1709 clone->nargs -= clone_info->inbranch;
1710 clone->simdlen = orig_simdlen;
1711 /* And call the target hook again to get the right ISA. */
1712 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1713 base_type,
1714 i / 2);
1715 if ((i & 1) != 0)
1716 clone->inbranch = 1;
1717 }
1718
1719 /* simd_clone_mangle might fail if such a clone has been created
1720 already. */
1721 tree id = simd_clone_mangle (node, clone);
1722 if (id == NULL_TREE)
1723 {
1724 if (i == 0)
1725 clone->nargs += clone->inbranch;
1726 continue;
1727 }
1728
1729 /* Only when we are sure we want to create the clone actually
1730 clone the function (or definitions) or create another
1731 extern FUNCTION_DECL (for prototypes without definitions). */
1732 struct cgraph_node *n = simd_clone_create (node);
1733 if (n == NULL)
1734 {
1735 if (i == 0)
1736 clone->nargs += clone->inbranch;
1737 continue;
1738 }
1739
1740 n->simdclone = clone;
1741 clone->origin = node;
1742 clone->next_clone = NULL;
1743 if (node->simd_clones == NULL)
1744 {
1745 clone->prev_clone = n;
1746 node->simd_clones = n;
1747 }
1748 else
1749 {
1750 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1751 clone->prev_clone->simdclone->next_clone = n;
1752 node->simd_clones->simdclone->prev_clone = n;
1753 }
1754 symtab->change_decl_assembler_name (n->decl, id);
1755 /* And finally adjust the return type, parameters and for
1756 definitions also function body. */
1757 if (node->definition)
1758 simd_clone_adjust (n);
1759 else
1760 {
1761 simd_clone_adjust_return_type (n);
1762 simd_clone_adjust_argument_types (n);
1763 }
1764 }
1765 }
1766 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1767 }
1768
1769 /* Entry point for IPA simd clone creation pass. */
1770
1771 static unsigned int
ipa_omp_simd_clone(void)1772 ipa_omp_simd_clone (void)
1773 {
1774 struct cgraph_node *node;
1775 FOR_EACH_FUNCTION (node)
1776 expand_simd_clones (node);
1777 return 0;
1778 }
1779
1780 namespace {
1781
1782 const pass_data pass_data_omp_simd_clone =
1783 {
1784 SIMPLE_IPA_PASS, /* type */
1785 "simdclone", /* name */
1786 OPTGROUP_OMP, /* optinfo_flags */
1787 TV_NONE, /* tv_id */
1788 ( PROP_ssa | PROP_cfg ), /* properties_required */
1789 0, /* properties_provided */
1790 0, /* properties_destroyed */
1791 0, /* todo_flags_start */
1792 0, /* todo_flags_finish */
1793 };
1794
1795 class pass_omp_simd_clone : public simple_ipa_opt_pass
1796 {
1797 public:
pass_omp_simd_clone(gcc::context * ctxt)1798 pass_omp_simd_clone(gcc::context *ctxt)
1799 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1800 {}
1801
1802 /* opt_pass methods: */
1803 virtual bool gate (function *);
execute(function *)1804 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1805 };
1806
1807 bool
gate(function *)1808 pass_omp_simd_clone::gate (function *)
1809 {
1810 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1811 }
1812
1813 } // anon namespace
1814
1815 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1816 make_pass_omp_simd_clone (gcc::context *ctxt)
1817 {
1818 return new pass_omp_simd_clone (ctxt);
1819 }
1820