1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2019 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Return vector of parameter types of function FNDECL. This uses
90 TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static vec<tree>
simd_clone_vector_of_formal_parm_types(tree fndecl)94 simd_clone_vector_of_formal_parm_types (tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
98 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
99 unsigned int i;
100 tree arg;
101 FOR_EACH_VEC_ELT (args, i, arg)
102 args[i] = TREE_TYPE (args[i]);
103 return args;
104 }
105
106 /* Given a simd function in NODE, extract the simd specific
107 information from the OMP clauses passed in CLAUSES, and return
108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
109 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
110 otherwise set to FALSE. */
111
112 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)113 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
114 bool *inbranch_specified)
115 {
116 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
117 tree t;
118 int n;
119 *inbranch_specified = false;
120
121 n = args.length ();
122 if (n > 0 && args.last () == void_type_node)
123 n--;
124
125 /* Allocate one more than needed just in case this is an in-branch
126 clone which will require a mask argument. */
127 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
128 clone_info->nargs = n;
129
130 if (!clauses)
131 goto out;
132
133 clauses = TREE_VALUE (clauses);
134 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
135 goto out;
136
137 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
138 {
139 switch (OMP_CLAUSE_CODE (t))
140 {
141 case OMP_CLAUSE_INBRANCH:
142 clone_info->inbranch = 1;
143 *inbranch_specified = true;
144 break;
145 case OMP_CLAUSE_NOTINBRANCH:
146 clone_info->inbranch = 0;
147 *inbranch_specified = true;
148 break;
149 case OMP_CLAUSE_SIMDLEN:
150 clone_info->simdlen
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
152 break;
153 case OMP_CLAUSE_LINEAR:
154 {
155 tree decl = OMP_CLAUSE_DECL (t);
156 tree step = OMP_CLAUSE_LINEAR_STEP (t);
157 int argno = TREE_INT_CST_LOW (decl);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
159 {
160 enum cgraph_simd_clone_arg_type arg_type;
161 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
162 switch (OMP_CLAUSE_LINEAR_KIND (t))
163 {
164 case OMP_CLAUSE_LINEAR_REF:
165 arg_type
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
167 break;
168 case OMP_CLAUSE_LINEAR_UVAL:
169 arg_type
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
171 break;
172 case OMP_CLAUSE_LINEAR_VAL:
173 case OMP_CLAUSE_LINEAR_DEFAULT:
174 arg_type
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
176 break;
177 default:
178 gcc_unreachable ();
179 }
180 else
181 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
182 clone_info->args[argno].arg_type = arg_type;
183 clone_info->args[argno].linear_step = tree_to_shwi (step);
184 gcc_assert (clone_info->args[argno].linear_step >= 0
185 && clone_info->args[argno].linear_step < n);
186 }
187 else
188 {
189 if (POINTER_TYPE_P (args[argno]))
190 step = fold_convert (ssizetype, step);
191 if (!tree_fits_shwi_p (step))
192 {
193 warning_at (OMP_CLAUSE_LOCATION (t), 0,
194 "ignoring large linear step");
195 args.release ();
196 return NULL;
197 }
198 else if (integer_zerop (step))
199 {
200 warning_at (OMP_CLAUSE_LOCATION (t), 0,
201 "ignoring zero linear step");
202 args.release ();
203 return NULL;
204 }
205 else
206 {
207 enum cgraph_simd_clone_arg_type arg_type;
208 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
209 switch (OMP_CLAUSE_LINEAR_KIND (t))
210 {
211 case OMP_CLAUSE_LINEAR_REF:
212 arg_type
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
214 break;
215 case OMP_CLAUSE_LINEAR_UVAL:
216 arg_type
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
218 break;
219 case OMP_CLAUSE_LINEAR_VAL:
220 case OMP_CLAUSE_LINEAR_DEFAULT:
221 arg_type
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
223 break;
224 default:
225 gcc_unreachable ();
226 }
227 else
228 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
229 clone_info->args[argno].arg_type = arg_type;
230 clone_info->args[argno].linear_step = tree_to_shwi (step);
231 }
232 }
233 break;
234 }
235 case OMP_CLAUSE_UNIFORM:
236 {
237 tree decl = OMP_CLAUSE_DECL (t);
238 int argno = tree_to_uhwi (decl);
239 clone_info->args[argno].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM;
241 break;
242 }
243 case OMP_CLAUSE_ALIGNED:
244 {
245 /* Ignore aligned (x) for declare simd, for the ABI we really
246 need an alignment specified. */
247 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
248 break;
249 tree decl = OMP_CLAUSE_DECL (t);
250 int argno = tree_to_uhwi (decl);
251 clone_info->args[argno].alignment
252 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
253 break;
254 }
255 default:
256 break;
257 }
258 }
259
260 out:
261 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
262 {
263 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
264 "ignoring %<#pragma omp declare simd%> on function "
265 "with %<_Atomic%> qualified return type");
266 args.release ();
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 args.release ();
282 return clone_info;
283 }
284
285 /* Given a SIMD clone in NODE, calculate the characteristic data
286 type and return the coresponding type. The characteristic data
287 type is computed as described in the Intel Vector ABI. */
288
289 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)290 simd_clone_compute_base_data_type (struct cgraph_node *node,
291 struct cgraph_simd_clone *clone_info)
292 {
293 tree type = integer_type_node;
294 tree fndecl = node->decl;
295
296 /* a) For non-void function, the characteristic data type is the
297 return type. */
298 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
299 type = TREE_TYPE (TREE_TYPE (fndecl));
300
301 /* b) If the function has any non-uniform, non-linear parameters,
302 then the characteristic data type is the type of the first
303 such parameter. */
304 else
305 {
306 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 map.release ();
314 }
315
316 /* c) If the characteristic data type determined by a) or b) above
317 is struct, union, or class type which is pass-by-value (except
318 for the type that maps to the built-in complex data type), the
319 characteristic data type is int. */
320 if (RECORD_OR_UNION_TYPE_P (type)
321 && !aggregate_value_p (type, NULL)
322 && TREE_CODE (type) != COMPLEX_TYPE)
323 return integer_type_node;
324
325 /* d) If none of the above three classes is applicable, the
326 characteristic data type is int. */
327
328 return type;
329
330 /* e) For Intel Xeon Phi native and offload compilation, if the
331 resulting characteristic data type is 8-bit or 16-bit integer
332 data type, the characteristic data type is int. */
333 /* Well, we don't handle Xeon Phi yet. */
334 }
335
336 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)337 simd_clone_mangle (struct cgraph_node *node,
338 struct cgraph_simd_clone *clone_info)
339 {
340 char vecsize_mangle = clone_info->vecsize_mangle;
341 char mask = clone_info->inbranch ? 'M' : 'N';
342 unsigned int simdlen = clone_info->simdlen;
343 unsigned int n;
344 pretty_printer pp;
345
346 gcc_assert (vecsize_mangle && simdlen);
347
348 pp_string (&pp, "_ZGV");
349 pp_character (&pp, vecsize_mangle);
350 pp_character (&pp, mask);
351 pp_decimal_int (&pp, simdlen);
352
353 for (n = 0; n < clone_info->nargs; ++n)
354 {
355 struct cgraph_simd_clone_arg arg = clone_info->args[n];
356
357 switch (arg.arg_type)
358 {
359 case SIMD_CLONE_ARG_TYPE_UNIFORM:
360 pp_character (&pp, 'u');
361 break;
362 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
363 pp_character (&pp, 'l');
364 goto mangle_linear;
365 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
366 pp_character (&pp, 'R');
367 goto mangle_linear;
368 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
369 pp_character (&pp, 'L');
370 goto mangle_linear;
371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
372 pp_character (&pp, 'U');
373 goto mangle_linear;
374 mangle_linear:
375 gcc_assert (arg.linear_step != 0);
376 if (arg.linear_step > 1)
377 pp_unsigned_wide_integer (&pp, arg.linear_step);
378 else if (arg.linear_step < 0)
379 {
380 pp_character (&pp, 'n');
381 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
382 arg.linear_step));
383 }
384 break;
385 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
386 pp_string (&pp, "ls");
387 pp_unsigned_wide_integer (&pp, arg.linear_step);
388 break;
389 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
390 pp_string (&pp, "Rs");
391 pp_unsigned_wide_integer (&pp, arg.linear_step);
392 break;
393 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
394 pp_string (&pp, "Ls");
395 pp_unsigned_wide_integer (&pp, arg.linear_step);
396 break;
397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
398 pp_string (&pp, "Us");
399 pp_unsigned_wide_integer (&pp, arg.linear_step);
400 break;
401 default:
402 pp_character (&pp, 'v');
403 }
404 if (arg.alignment)
405 {
406 pp_character (&pp, 'a');
407 pp_decimal_int (&pp, arg.alignment);
408 }
409 }
410
411 pp_underscore (&pp);
412 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
413 if (*str == '*')
414 ++str;
415 pp_string (&pp, str);
416 str = pp_formatted_text (&pp);
417
418 /* If there already is a SIMD clone with the same mangled name, don't
419 add another one. This can happen e.g. for
420 #pragma omp declare simd
421 #pragma omp declare simd simdlen(8)
422 int foo (int, int);
423 if the simdlen is assumed to be 8 for the first one, etc. */
424 for (struct cgraph_node *clone = node->simd_clones; clone;
425 clone = clone->simdclone->next_clone)
426 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
427 return NULL_TREE;
428
429 return get_identifier (str);
430 }
431
432 /* Create a simd clone of OLD_NODE and return it. */
433
434 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)435 simd_clone_create (struct cgraph_node *old_node)
436 {
437 struct cgraph_node *new_node;
438 if (old_node->definition)
439 {
440 if (!old_node->has_gimple_body_p ())
441 return NULL;
442 old_node->get_body ();
443 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
444 false, NULL, NULL,
445 "simdclone");
446 }
447 else
448 {
449 tree old_decl = old_node->decl;
450 tree new_decl = copy_node (old_node->decl);
451 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl,
452 "simdclone");
453 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
454 SET_DECL_RTL (new_decl, NULL);
455 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
456 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
457 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
458 if (old_node->in_other_partition)
459 new_node->in_other_partition = 1;
460 }
461 if (new_node == NULL)
462 return new_node;
463
464 DECL_BUILT_IN_CLASS (new_node->decl) = NOT_BUILT_IN;
465 DECL_FUNCTION_CODE (new_node->decl) = (enum built_in_function) 0;
466 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
467 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
468 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
469 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
470 DECL_VISIBILITY_SPECIFIED (new_node->decl)
471 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
472 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
473 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
474 if (DECL_ONE_ONLY (old_node->decl))
475 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
476
477 /* The method cgraph_version_clone_with_body () will force the new
478 symbol local. Undo this, and inherit external visibility from
479 the old node. */
480 new_node->local.local = old_node->local.local;
481 new_node->externally_visible = old_node->externally_visible;
482
483 return new_node;
484 }
485
486 /* Adjust the return type of the given function to its appropriate
487 vector counterpart. Returns a simd array to be used throughout the
488 function as a return value. */
489
490 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)491 simd_clone_adjust_return_type (struct cgraph_node *node)
492 {
493 tree fndecl = node->decl;
494 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
495 unsigned int veclen;
496 tree t;
497
498 /* Adjust the function return type. */
499 if (orig_rettype == void_type_node)
500 return NULL_TREE;
501 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
502 t = TREE_TYPE (TREE_TYPE (fndecl));
503 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
504 veclen = node->simdclone->vecsize_int;
505 else
506 veclen = node->simdclone->vecsize_float;
507 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
508 if (veclen > node->simdclone->simdlen)
509 veclen = node->simdclone->simdlen;
510 if (POINTER_TYPE_P (t))
511 t = pointer_sized_int_node;
512 if (veclen == node->simdclone->simdlen)
513 t = build_vector_type (t, node->simdclone->simdlen);
514 else
515 {
516 t = build_vector_type (t, veclen);
517 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
518 }
519 TREE_TYPE (TREE_TYPE (fndecl)) = t;
520 if (!node->definition)
521 return NULL_TREE;
522
523 t = DECL_RESULT (fndecl);
524 /* Adjust the DECL_RESULT. */
525 gcc_assert (TREE_TYPE (t) != void_type_node);
526 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
527 relayout_decl (t);
528
529 tree atype = build_array_type_nelts (orig_rettype,
530 node->simdclone->simdlen);
531 if (veclen != node->simdclone->simdlen)
532 return build1 (VIEW_CONVERT_EXPR, atype, t);
533
534 /* Set up a SIMD array to use as the return value. */
535 tree retval = create_tmp_var_raw (atype, "retval");
536 gimple_add_tmp_var (retval);
537 return retval;
538 }
539
540 /* Each vector argument has a corresponding array to be used locally
541 as part of the eventual loop. Create such temporary array and
542 return it.
543
544 PREFIX is the prefix to be used for the temporary.
545
546 TYPE is the inner element type.
547
548 SIMDLEN is the number of elements. */
549
550 static tree
create_tmp_simd_array(const char * prefix,tree type,int simdlen)551 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
552 {
553 tree atype = build_array_type_nelts (type, simdlen);
554 tree avar = create_tmp_var_raw (atype, prefix);
555 gimple_add_tmp_var (avar);
556 return avar;
557 }
558
559 /* Modify the function argument types to their corresponding vector
560 counterparts if appropriate. Also, create one array for each simd
561 argument to be used locally when using the function arguments as
562 part of the loop.
563
564 NODE is the function whose arguments are to be adjusted.
565
566 Returns an adjustment vector that will be filled describing how the
567 argument types will be adjusted. */
568
569 static ipa_parm_adjustment_vec
simd_clone_adjust_argument_types(struct cgraph_node * node)570 simd_clone_adjust_argument_types (struct cgraph_node *node)
571 {
572 vec<tree> args;
573 ipa_parm_adjustment_vec adjustments;
574
575 if (node->definition)
576 args = ipa_get_vector_of_formal_parms (node->decl);
577 else
578 args = simd_clone_vector_of_formal_parm_types (node->decl);
579 adjustments.create (args.length ());
580 unsigned i, j, veclen;
581 struct ipa_parm_adjustment adj;
582 struct cgraph_simd_clone *sc = node->simdclone;
583
584 for (i = 0; i < sc->nargs; ++i)
585 {
586 memset (&adj, 0, sizeof (adj));
587 tree parm = args[i];
588 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
589 adj.base_index = i;
590 adj.base = parm;
591
592 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
593 sc->args[i].orig_type = parm_type;
594
595 switch (sc->args[i].arg_type)
596 {
597 default:
598 /* No adjustment necessary for scalar arguments. */
599 adj.op = IPA_PARM_OP_COPY;
600 break;
601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
602 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
603 if (node->definition)
604 sc->args[i].simd_array
605 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
606 TREE_TYPE (parm_type),
607 sc->simdlen);
608 adj.op = IPA_PARM_OP_COPY;
609 break;
610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
611 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
612 case SIMD_CLONE_ARG_TYPE_VECTOR:
613 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
614 veclen = sc->vecsize_int;
615 else
616 veclen = sc->vecsize_float;
617 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
618 if (veclen > sc->simdlen)
619 veclen = sc->simdlen;
620 adj.arg_prefix = "simd";
621 if (POINTER_TYPE_P (parm_type))
622 adj.type = build_vector_type (pointer_sized_int_node, veclen);
623 else
624 adj.type = build_vector_type (parm_type, veclen);
625 sc->args[i].vector_type = adj.type;
626 for (j = veclen; j < sc->simdlen; j += veclen)
627 {
628 adjustments.safe_push (adj);
629 if (j == veclen)
630 {
631 memset (&adj, 0, sizeof (adj));
632 adj.op = IPA_PARM_OP_NEW;
633 adj.arg_prefix = "simd";
634 adj.base_index = i;
635 adj.type = sc->args[i].vector_type;
636 }
637 }
638
639 if (node->definition)
640 sc->args[i].simd_array
641 = create_tmp_simd_array (DECL_NAME (parm)
642 ? IDENTIFIER_POINTER (DECL_NAME (parm))
643 : NULL, parm_type, sc->simdlen);
644 }
645 adjustments.safe_push (adj);
646 }
647
648 if (sc->inbranch)
649 {
650 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
651
652 memset (&adj, 0, sizeof (adj));
653 adj.op = IPA_PARM_OP_NEW;
654 adj.arg_prefix = "mask";
655
656 adj.base_index = i;
657 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
658 veclen = sc->vecsize_int;
659 else
660 veclen = sc->vecsize_float;
661 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
662 if (veclen > sc->simdlen)
663 veclen = sc->simdlen;
664 if (sc->mask_mode != VOIDmode)
665 adj.type
666 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
667 else if (POINTER_TYPE_P (base_type))
668 adj.type = build_vector_type (pointer_sized_int_node, veclen);
669 else
670 adj.type = build_vector_type (base_type, veclen);
671 adjustments.safe_push (adj);
672
673 for (j = veclen; j < sc->simdlen; j += veclen)
674 adjustments.safe_push (adj);
675
676 /* We have previously allocated one extra entry for the mask. Use
677 it and fill it. */
678 sc->nargs++;
679 if (sc->mask_mode != VOIDmode)
680 base_type = boolean_type_node;
681 if (node->definition)
682 {
683 sc->args[i].orig_arg
684 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
685 if (sc->mask_mode == VOIDmode)
686 sc->args[i].simd_array
687 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
688 else if (veclen < sc->simdlen)
689 sc->args[i].simd_array
690 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
691 else
692 sc->args[i].simd_array = NULL_TREE;
693 }
694 sc->args[i].orig_type = base_type;
695 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
696 }
697
698 if (node->definition)
699 ipa_modify_formal_parameters (node->decl, adjustments);
700 else
701 {
702 tree new_arg_types = NULL_TREE, new_reversed;
703 bool last_parm_void = false;
704 if (args.length () > 0 && args.last () == void_type_node)
705 last_parm_void = true;
706
707 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
708 j = adjustments.length ();
709 for (i = 0; i < j; i++)
710 {
711 struct ipa_parm_adjustment *adj = &adjustments[i];
712 tree ptype;
713 if (adj->op == IPA_PARM_OP_COPY)
714 ptype = args[adj->base_index];
715 else
716 ptype = adj->type;
717 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
718 }
719 new_reversed = nreverse (new_arg_types);
720 if (last_parm_void)
721 {
722 if (new_reversed)
723 TREE_CHAIN (new_arg_types) = void_list_node;
724 else
725 new_reversed = void_list_node;
726 }
727
728 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
729 TYPE_ARG_TYPES (new_type) = new_reversed;
730 TREE_TYPE (node->decl) = new_type;
731
732 adjustments.release ();
733 }
734 args.release ();
735 return adjustments;
736 }
737
738 /* Initialize and copy the function arguments in NODE to their
739 corresponding local simd arrays. Returns a fresh gimple_seq with
740 the instruction sequence generated. */
741
742 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments)743 simd_clone_init_simd_arrays (struct cgraph_node *node,
744 ipa_parm_adjustment_vec adjustments)
745 {
746 gimple_seq seq = NULL;
747 unsigned i = 0, j = 0, k;
748
749 for (tree arg = DECL_ARGUMENTS (node->decl);
750 arg;
751 arg = DECL_CHAIN (arg), i++, j++)
752 {
753 if (adjustments[j].op == IPA_PARM_OP_COPY
754 || POINTER_TYPE_P (TREE_TYPE (arg)))
755 continue;
756
757 node->simdclone->args[i].vector_arg = arg;
758
759 tree array = node->simdclone->args[i].simd_array;
760 if (node->simdclone->mask_mode != VOIDmode
761 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
762 {
763 if (array == NULL_TREE)
764 continue;
765 unsigned int l
766 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
767 for (k = 0; k <= l; k++)
768 {
769 if (k)
770 {
771 arg = DECL_CHAIN (arg);
772 j++;
773 }
774 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
775 array, size_int (k), NULL, NULL);
776 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
777 gimplify_and_add (t, &seq);
778 }
779 continue;
780 }
781 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen)
782 {
783 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
784 tree ptr = build_fold_addr_expr (array);
785 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
786 build_int_cst (ptype, 0));
787 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
788 gimplify_and_add (t, &seq);
789 }
790 else
791 {
792 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
793 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
794 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
795 {
796 tree ptr = build_fold_addr_expr (array);
797 int elemsize;
798 if (k)
799 {
800 arg = DECL_CHAIN (arg);
801 j++;
802 }
803 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
804 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
805 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
806 build_int_cst (ptype, k * elemsize));
807 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
808 gimplify_and_add (t, &seq);
809 }
810 }
811 }
812 return seq;
813 }
814
815 /* Callback info for ipa_simd_modify_stmt_ops below. */
816
817 struct modify_stmt_info {
818 ipa_parm_adjustment_vec adjustments;
819 gimple *stmt;
820 /* True if the parent statement was modified by
821 ipa_simd_modify_stmt_ops. */
822 bool modified;
823 };
824
825 /* Callback for walk_gimple_op.
826
827 Adjust operands from a given statement as specified in the
828 adjustments vector in the callback data. */
829
830 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)831 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
832 {
833 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
834 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
835 tree *orig_tp = tp;
836 if (TREE_CODE (*tp) == ADDR_EXPR)
837 tp = &TREE_OPERAND (*tp, 0);
838 struct ipa_parm_adjustment *cand = NULL;
839 if (TREE_CODE (*tp) == PARM_DECL)
840 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
841 else if (TYPE_P (*tp))
842 *walk_subtrees = 0;
843
844 tree repl = NULL_TREE;
845 if (cand)
846 repl = unshare_expr (cand->new_decl);
847 else
848 {
849 if (tp != orig_tp)
850 {
851 *walk_subtrees = 0;
852 bool modified = info->modified;
853 info->modified = false;
854 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
855 if (!info->modified)
856 {
857 info->modified = modified;
858 return NULL_TREE;
859 }
860 info->modified = modified;
861 repl = *tp;
862 }
863 else
864 return NULL_TREE;
865 }
866
867 if (tp != orig_tp)
868 {
869 if (gimple_code (info->stmt) == GIMPLE_PHI
870 && cand
871 && TREE_CODE (*orig_tp) == ADDR_EXPR
872 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
873 && cand->alias_ptr_type)
874 {
875 gcc_assert (TREE_CODE (cand->alias_ptr_type) == SSA_NAME);
876 *orig_tp = cand->alias_ptr_type;
877 info->modified = true;
878 return NULL_TREE;
879 }
880
881 repl = build_fold_addr_expr (repl);
882 gimple *stmt;
883 if (is_gimple_debug (info->stmt))
884 {
885 tree vexpr = make_node (DEBUG_EXPR_DECL);
886 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
887 DECL_ARTIFICIAL (vexpr) = 1;
888 TREE_TYPE (vexpr) = TREE_TYPE (repl);
889 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
890 repl = vexpr;
891 }
892 else
893 {
894 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
895 repl = gimple_assign_lhs (stmt);
896 }
897 gimple_stmt_iterator gsi;
898 if (gimple_code (info->stmt) == GIMPLE_PHI)
899 {
900 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
901 /* Cache SSA_NAME for next time. */
902 if (cand
903 && TREE_CODE (*orig_tp) == ADDR_EXPR
904 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
905 cand->alias_ptr_type = repl;
906 }
907 else
908 gsi = gsi_for_stmt (info->stmt);
909 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
910 *orig_tp = repl;
911 }
912 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
913 {
914 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
915 *tp = vce;
916 }
917 else
918 *tp = repl;
919
920 info->modified = true;
921 return NULL_TREE;
922 }
923
924 /* Traverse the function body and perform all modifications as
925 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
926 modified such that the replacement/reduction value will now be an
927 offset into the corresponding simd_array.
928
929 This function will replace all function argument uses with their
930 corresponding simd array elements, and ajust the return values
931 accordingly. */
932
933 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments,tree retval_array,tree iter)934 ipa_simd_modify_function_body (struct cgraph_node *node,
935 ipa_parm_adjustment_vec adjustments,
936 tree retval_array, tree iter)
937 {
938 basic_block bb;
939 unsigned int i, j, l;
940
941 /* Re-use the adjustments array, but this time use it to replace
942 every function argument use to an offset into the corresponding
943 simd_array. */
944 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
945 {
946 if (!node->simdclone->args[i].vector_arg)
947 continue;
948
949 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
950 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
951 adjustments[j].new_decl
952 = build4 (ARRAY_REF,
953 basetype,
954 node->simdclone->args[i].simd_array,
955 iter,
956 NULL_TREE, NULL_TREE);
957 if (adjustments[j].op == IPA_PARM_OP_NONE
958 && simd_clone_subparts (vectype) < node->simdclone->simdlen)
959 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1;
960 }
961
962 l = adjustments.length ();
963 tree name;
964
965 FOR_EACH_SSA_NAME (i, name, cfun)
966 {
967 if (SSA_NAME_VAR (name)
968 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
969 {
970 for (j = 0; j < l; j++)
971 if (SSA_NAME_VAR (name) == adjustments[j].base
972 && adjustments[j].new_decl)
973 {
974 tree base_var;
975 if (adjustments[j].new_ssa_base == NULL_TREE)
976 {
977 base_var
978 = copy_var_decl (adjustments[j].base,
979 DECL_NAME (adjustments[j].base),
980 TREE_TYPE (adjustments[j].base));
981 adjustments[j].new_ssa_base = base_var;
982 }
983 else
984 base_var = adjustments[j].new_ssa_base;
985 if (SSA_NAME_IS_DEFAULT_DEF (name))
986 {
987 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
988 gimple_stmt_iterator gsi = gsi_after_labels (bb);
989 tree new_decl = unshare_expr (adjustments[j].new_decl);
990 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
991 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
992 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
993 gimple *stmt = gimple_build_assign (name, new_decl);
994 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
995 }
996 else
997 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
998 }
999 }
1000 }
1001
1002 struct modify_stmt_info info;
1003 info.adjustments = adjustments;
1004
1005 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1006 {
1007 gimple_stmt_iterator gsi;
1008
1009 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1010 {
1011 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1012 int i, n = gimple_phi_num_args (phi);
1013 info.stmt = phi;
1014 struct walk_stmt_info wi;
1015 memset (&wi, 0, sizeof (wi));
1016 info.modified = false;
1017 wi.info = &info;
1018 for (i = 0; i < n; ++i)
1019 {
1020 int walk_subtrees = 1;
1021 tree arg = gimple_phi_arg_def (phi, i);
1022 tree op = arg;
1023 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1024 if (op != arg)
1025 {
1026 SET_PHI_ARG_DEF (phi, i, op);
1027 gcc_assert (TREE_CODE (op) == SSA_NAME);
1028 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1029 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1030 }
1031 }
1032 }
1033
1034 gsi = gsi_start_bb (bb);
1035 while (!gsi_end_p (gsi))
1036 {
1037 gimple *stmt = gsi_stmt (gsi);
1038 info.stmt = stmt;
1039 struct walk_stmt_info wi;
1040
1041 memset (&wi, 0, sizeof (wi));
1042 info.modified = false;
1043 wi.info = &info;
1044 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1045
1046 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1047 {
1048 tree retval = gimple_return_retval (return_stmt);
1049 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1050 e->flags |= EDGE_FALLTHRU;
1051 if (!retval)
1052 {
1053 gsi_remove (&gsi, true);
1054 continue;
1055 }
1056
1057 /* Replace `return foo' with `retval_array[iter] = foo'. */
1058 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1059 retval_array, iter, NULL, NULL);
1060 stmt = gimple_build_assign (ref, retval);
1061 gsi_replace (&gsi, stmt, true);
1062 info.modified = true;
1063 }
1064
1065 if (info.modified)
1066 {
1067 update_stmt (stmt);
1068 /* If the above changed the var of a debug bind into something
1069 different, remove the debug stmt. We could also for all the
1070 replaced parameters add VAR_DECLs for debug info purposes,
1071 add debug stmts for those to be the simd array accesses and
1072 replace debug stmt var operand with that var. Debugging of
1073 vectorized loops doesn't work too well, so don't bother for
1074 now. */
1075 if ((gimple_debug_bind_p (stmt)
1076 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1077 || (gimple_debug_source_bind_p (stmt)
1078 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1079 {
1080 gsi_remove (&gsi, true);
1081 continue;
1082 }
1083 if (maybe_clean_eh_stmt (stmt))
1084 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1085 }
1086 gsi_next (&gsi);
1087 }
1088 }
1089 }
1090
1091 /* Helper function of simd_clone_adjust, return linear step addend
1092 of Ith argument. */
1093
1094 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1095 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1096 tree addtype, basic_block entry_bb)
1097 {
1098 tree ptype = NULL_TREE;
1099 switch (node->simdclone->args[i].arg_type)
1100 {
1101 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1102 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1103 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1104 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1105 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1106 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1107 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1108 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1109 break;
1110 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1111 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1112 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1113 break;
1114 default:
1115 gcc_unreachable ();
1116 }
1117
1118 unsigned int idx = node->simdclone->args[i].linear_step;
1119 tree arg = node->simdclone->args[idx].orig_arg;
1120 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1121 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1122 gimple *g;
1123 tree ret;
1124 if (is_gimple_reg (arg))
1125 ret = get_or_create_ssa_default_def (cfun, arg);
1126 else
1127 {
1128 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1129 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1130 ret = gimple_assign_lhs (g);
1131 }
1132 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1133 {
1134 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1135 build_simple_mem_ref (ret));
1136 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1137 ret = gimple_assign_lhs (g);
1138 }
1139 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1140 {
1141 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1142 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1143 ret = gimple_assign_lhs (g);
1144 }
1145 if (POINTER_TYPE_P (ptype))
1146 {
1147 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1148 if (size && TREE_CODE (size) == INTEGER_CST)
1149 {
1150 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1151 ret, fold_convert (addtype, size));
1152 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1153 ret = gimple_assign_lhs (g);
1154 }
1155 }
1156 return ret;
1157 }
1158
1159 /* Adjust the argument types in NODE to their appropriate vector
1160 counterparts. */
1161
1162 static void
simd_clone_adjust(struct cgraph_node * node)1163 simd_clone_adjust (struct cgraph_node *node)
1164 {
1165 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1166
1167 targetm.simd_clone.adjust (node);
1168
1169 tree retval = simd_clone_adjust_return_type (node);
1170 ipa_parm_adjustment_vec adjustments
1171 = simd_clone_adjust_argument_types (node);
1172
1173 push_gimplify_context ();
1174
1175 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1176
1177 /* Adjust all uses of vector arguments accordingly. Adjust all
1178 return values accordingly. */
1179 tree iter = create_tmp_var (unsigned_type_node, "iter");
1180 tree iter1 = make_ssa_name (iter);
1181 tree iter2 = NULL_TREE;
1182 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1183 adjustments.release ();
1184
1185 /* Initialize the iteration variable. */
1186 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1187 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1188 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1189 /* Insert the SIMD array and iv initialization at function
1190 entry. */
1191 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1192
1193 pop_gimplify_context (NULL);
1194
1195 gimple *g;
1196 basic_block incr_bb = NULL;
1197 struct loop *loop = NULL;
1198
1199 /* Create a new BB right before the original exit BB, to hold the
1200 iteration increment and the condition/branch. */
1201 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1202 {
1203 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1204 incr_bb = create_empty_bb (orig_exit);
1205 incr_bb->count = profile_count::zero ();
1206 add_bb_to_loop (incr_bb, body_bb->loop_father);
1207 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1208 {
1209 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1210 redirect_edge_succ (e, incr_bb);
1211 incr_bb->count += e->count ();
1212 }
1213 }
1214 else if (node->simdclone->inbranch)
1215 {
1216 incr_bb = create_empty_bb (entry_bb);
1217 incr_bb->count = profile_count::zero ();
1218 add_bb_to_loop (incr_bb, body_bb->loop_father);
1219 }
1220
1221 if (incr_bb)
1222 {
1223 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1224 gsi = gsi_last_bb (incr_bb);
1225 iter2 = make_ssa_name (iter);
1226 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1227 build_int_cst (unsigned_type_node, 1));
1228 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1229
1230 /* Mostly annotate the loop for the vectorizer (the rest is done
1231 below). */
1232 loop = alloc_loop ();
1233 cfun->has_force_vectorize_loops = true;
1234 loop->safelen = node->simdclone->simdlen;
1235 loop->force_vectorize = true;
1236 loop->header = body_bb;
1237 }
1238
1239 /* Branch around the body if the mask applies. */
1240 if (node->simdclone->inbranch)
1241 {
1242 gsi = gsi_last_bb (loop->header);
1243 tree mask_array
1244 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1245 tree mask;
1246 if (node->simdclone->mask_mode != VOIDmode)
1247 {
1248 tree shift_cnt;
1249 if (mask_array == NULL_TREE)
1250 {
1251 tree arg = node->simdclone->args[node->simdclone->nargs
1252 - 1].vector_arg;
1253 mask = get_or_create_ssa_default_def (cfun, arg);
1254 shift_cnt = iter1;
1255 }
1256 else
1257 {
1258 tree maskt = TREE_TYPE (mask_array);
1259 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1260 c = node->simdclone->simdlen / (c + 1);
1261 int s = exact_log2 (c);
1262 gcc_assert (s > 0);
1263 c--;
1264 tree idx = make_ssa_name (TREE_TYPE (iter1));
1265 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1266 build_int_cst (NULL_TREE, s));
1267 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1268 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1269 tree aref = build4 (ARRAY_REF,
1270 TREE_TYPE (TREE_TYPE (mask_array)),
1271 mask_array, idx, NULL, NULL);
1272 g = gimple_build_assign (mask, aref);
1273 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1274 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1275 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1276 build_int_cst (TREE_TYPE (iter1), c));
1277 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1278 }
1279 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1280 RSHIFT_EXPR, mask, shift_cnt);
1281 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1282 mask = gimple_assign_lhs (g);
1283 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1284 BIT_AND_EXPR, mask,
1285 build_int_cst (TREE_TYPE (mask), 1));
1286 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1287 mask = gimple_assign_lhs (g);
1288 }
1289 else
1290 {
1291 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1292 tree aref = build4 (ARRAY_REF,
1293 TREE_TYPE (TREE_TYPE (mask_array)),
1294 mask_array, iter1, NULL, NULL);
1295 g = gimple_build_assign (mask, aref);
1296 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1297 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1298 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1299 {
1300 aref = build1 (VIEW_CONVERT_EXPR,
1301 build_nonstandard_integer_type (bitsize, 0),
1302 mask);
1303 mask = make_ssa_name (TREE_TYPE (aref));
1304 g = gimple_build_assign (mask, aref);
1305 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1306 }
1307 }
1308
1309 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1310 NULL, NULL);
1311 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1312 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1313 e->probability = profile_probability::unlikely ().guessed ();
1314 incr_bb->count += e->count ();
1315 edge fallthru = FALLTHRU_EDGE (loop->header);
1316 fallthru->flags = EDGE_FALSE_VALUE;
1317 fallthru->probability = profile_probability::likely ().guessed ();
1318 }
1319
1320 basic_block latch_bb = NULL;
1321 basic_block new_exit_bb = NULL;
1322
1323 /* Generate the condition. */
1324 if (incr_bb)
1325 {
1326 gsi = gsi_last_bb (incr_bb);
1327 g = gimple_build_cond (LT_EXPR, iter2,
1328 build_int_cst (unsigned_type_node,
1329 node->simdclone->simdlen),
1330 NULL, NULL);
1331 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1332 edge e = split_block (incr_bb, gsi_stmt (gsi));
1333 latch_bb = e->dest;
1334 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1335 loop->latch = latch_bb;
1336
1337 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1338
1339 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1340
1341 /* FIXME: Do we need to distribute probabilities for the conditional? */
1342 new_e->probability = profile_probability::guessed_never ();
1343 /* The successor of incr_bb is already pointing to latch_bb; just
1344 change the flags.
1345 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1346 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1347 }
1348
1349 gphi *phi = create_phi_node (iter1, body_bb);
1350 edge preheader_edge = find_edge (entry_bb, body_bb);
1351 edge latch_edge = NULL;
1352 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1353 UNKNOWN_LOCATION);
1354 if (incr_bb)
1355 {
1356 latch_edge = single_succ_edge (latch_bb);
1357 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1358
1359 /* Generate the new return. */
1360 gsi = gsi_last_bb (new_exit_bb);
1361 if (retval
1362 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1363 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1364 retval = TREE_OPERAND (retval, 0);
1365 else if (retval)
1366 {
1367 retval = build1 (VIEW_CONVERT_EXPR,
1368 TREE_TYPE (TREE_TYPE (node->decl)),
1369 retval);
1370 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1371 false, GSI_CONTINUE_LINKING);
1372 }
1373 g = gimple_build_return (retval);
1374 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1375 }
1376
1377 /* Handle aligned clauses by replacing default defs of the aligned
1378 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1379 lhs. Handle linear by adding PHIs. */
1380 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1381 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1382 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1383 || !is_gimple_reg_type
1384 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1385 {
1386 tree orig_arg = node->simdclone->args[i].orig_arg;
1387 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1388 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1389 else
1390 {
1391 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1392 gimple_add_tmp_var (iter1);
1393 }
1394 gsi = gsi_after_labels (entry_bb);
1395 g = gimple_build_assign (iter1, orig_arg);
1396 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1397 gsi = gsi_after_labels (body_bb);
1398 g = gimple_build_assign (orig_arg, iter1);
1399 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1400 }
1401 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1402 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1403 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1404 == REFERENCE_TYPE
1405 && TREE_ADDRESSABLE
1406 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1407 {
1408 tree orig_arg = node->simdclone->args[i].orig_arg;
1409 tree def = ssa_default_def (cfun, orig_arg);
1410 if (def && !has_zero_uses (def))
1411 {
1412 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1413 gimple_add_tmp_var (iter1);
1414 gsi = gsi_after_labels (entry_bb);
1415 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1416 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1417 gsi = gsi_after_labels (body_bb);
1418 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1419 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1420 }
1421 }
1422 else if (node->simdclone->args[i].alignment
1423 && node->simdclone->args[i].arg_type
1424 == SIMD_CLONE_ARG_TYPE_UNIFORM
1425 && (node->simdclone->args[i].alignment
1426 & (node->simdclone->args[i].alignment - 1)) == 0
1427 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1428 == POINTER_TYPE)
1429 {
1430 unsigned int alignment = node->simdclone->args[i].alignment;
1431 tree orig_arg = node->simdclone->args[i].orig_arg;
1432 tree def = ssa_default_def (cfun, orig_arg);
1433 if (def && !has_zero_uses (def))
1434 {
1435 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1436 gimple_seq seq = NULL;
1437 bool need_cvt = false;
1438 gcall *call
1439 = gimple_build_call (fn, 2, def, size_int (alignment));
1440 g = call;
1441 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1442 ptr_type_node))
1443 need_cvt = true;
1444 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1445 gimple_call_set_lhs (g, t);
1446 gimple_seq_add_stmt_without_update (&seq, g);
1447 if (need_cvt)
1448 {
1449 t = make_ssa_name (orig_arg);
1450 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1451 gimple_seq_add_stmt_without_update (&seq, g);
1452 }
1453 gsi_insert_seq_on_edge_immediate
1454 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1455
1456 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1457 node->create_edge (cgraph_node::get_create (fn),
1458 call, entry_bb->count);
1459
1460 imm_use_iterator iter;
1461 use_operand_p use_p;
1462 gimple *use_stmt;
1463 tree repl = gimple_get_lhs (g);
1464 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1465 if (is_gimple_debug (use_stmt) || use_stmt == call)
1466 continue;
1467 else
1468 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1469 SET_USE (use_p, repl);
1470 }
1471 }
1472 else if ((node->simdclone->args[i].arg_type
1473 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1474 || (node->simdclone->args[i].arg_type
1475 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1476 || (node->simdclone->args[i].arg_type
1477 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1478 || (node->simdclone->args[i].arg_type
1479 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1480 {
1481 tree orig_arg = node->simdclone->args[i].orig_arg;
1482 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1483 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1484 tree def = NULL_TREE;
1485 if (TREE_ADDRESSABLE (orig_arg))
1486 {
1487 def = make_ssa_name (TREE_TYPE (orig_arg));
1488 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1489 if (incr_bb)
1490 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1491 gsi = gsi_after_labels (entry_bb);
1492 g = gimple_build_assign (def, orig_arg);
1493 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1494 }
1495 else
1496 {
1497 def = ssa_default_def (cfun, orig_arg);
1498 if (!def || has_zero_uses (def))
1499 def = NULL_TREE;
1500 else
1501 {
1502 iter1 = make_ssa_name (orig_arg);
1503 if (incr_bb)
1504 iter2 = make_ssa_name (orig_arg);
1505 }
1506 }
1507 if (def)
1508 {
1509 phi = create_phi_node (iter1, body_bb);
1510 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1511 if (incr_bb)
1512 {
1513 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1514 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1515 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1516 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1517 ? TREE_TYPE (orig_arg) : sizetype;
1518 tree addcst = simd_clone_linear_addend (node, i, addtype,
1519 entry_bb);
1520 gsi = gsi_last_bb (incr_bb);
1521 g = gimple_build_assign (iter2, code, iter1, addcst);
1522 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1523 }
1524
1525 imm_use_iterator iter;
1526 use_operand_p use_p;
1527 gimple *use_stmt;
1528 if (TREE_ADDRESSABLE (orig_arg))
1529 {
1530 gsi = gsi_after_labels (body_bb);
1531 g = gimple_build_assign (orig_arg, iter1);
1532 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1533 }
1534 else
1535 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1536 if (use_stmt == phi)
1537 continue;
1538 else
1539 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1540 SET_USE (use_p, iter1);
1541 }
1542 }
1543 else if (node->simdclone->args[i].arg_type
1544 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1545 || (node->simdclone->args[i].arg_type
1546 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1547 {
1548 tree orig_arg = node->simdclone->args[i].orig_arg;
1549 tree def = ssa_default_def (cfun, orig_arg);
1550 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1551 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1552 if (def && !has_zero_uses (def))
1553 {
1554 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1555 iter1 = make_ssa_name (orig_arg);
1556 if (incr_bb)
1557 iter2 = make_ssa_name (orig_arg);
1558 tree iter3 = make_ssa_name (rtype);
1559 tree iter4 = make_ssa_name (rtype);
1560 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1561 gsi = gsi_after_labels (entry_bb);
1562 gimple *load
1563 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1564 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1565
1566 tree array = node->simdclone->args[i].simd_array;
1567 TREE_ADDRESSABLE (array) = 1;
1568 tree ptr = build_fold_addr_expr (array);
1569 phi = create_phi_node (iter1, body_bb);
1570 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1571 if (incr_bb)
1572 {
1573 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1574 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1575 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1576 gsi = gsi_last_bb (incr_bb);
1577 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1578 }
1579
1580 phi = create_phi_node (iter4, body_bb);
1581 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1582 if (incr_bb)
1583 {
1584 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1585 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1586 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1587 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1588 ? TREE_TYPE (iter3) : sizetype;
1589 tree addcst = simd_clone_linear_addend (node, i, addtype,
1590 entry_bb);
1591 g = gimple_build_assign (iter5, code, iter4, addcst);
1592 gsi = gsi_last_bb (incr_bb);
1593 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1594 }
1595
1596 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1597 gsi = gsi_after_labels (body_bb);
1598 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1599
1600 imm_use_iterator iter;
1601 use_operand_p use_p;
1602 gimple *use_stmt;
1603 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1604 if (use_stmt == load)
1605 continue;
1606 else
1607 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1608 SET_USE (use_p, iter1);
1609
1610 if (!TYPE_READONLY (rtype) && incr_bb)
1611 {
1612 tree v = make_ssa_name (rtype);
1613 tree aref = build4 (ARRAY_REF, rtype, array,
1614 size_zero_node, NULL_TREE,
1615 NULL_TREE);
1616 gsi = gsi_after_labels (new_exit_bb);
1617 g = gimple_build_assign (v, aref);
1618 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1619 g = gimple_build_assign (build_simple_mem_ref (def), v);
1620 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1621 }
1622 }
1623 }
1624
1625 calculate_dominance_info (CDI_DOMINATORS);
1626 if (loop)
1627 add_loop (loop, loop->header->loop_father);
1628 update_ssa (TODO_update_ssa);
1629
1630 pop_cfun ();
1631 }
1632
1633 /* If the function in NODE is tagged as an elemental SIMD function,
1634 create the appropriate SIMD clones. */
1635
1636 void
expand_simd_clones(struct cgraph_node * node)1637 expand_simd_clones (struct cgraph_node *node)
1638 {
1639 tree attr = lookup_attribute ("omp declare simd",
1640 DECL_ATTRIBUTES (node->decl));
1641 if (attr == NULL_TREE
1642 || node->global.inlined_to
1643 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1644 return;
1645
1646 /* Ignore
1647 #pragma omp declare simd
1648 extern int foo ();
1649 in C, there we don't know the argument types at all. */
1650 if (!node->definition
1651 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1652 return;
1653
1654 /* Call this before creating clone_info, as it might ggc_collect. */
1655 if (node->definition && node->has_gimple_body_p ())
1656 node->get_body ();
1657
1658 do
1659 {
1660 /* Start with parsing the "omp declare simd" attribute(s). */
1661 bool inbranch_clause_specified;
1662 struct cgraph_simd_clone *clone_info
1663 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1664 &inbranch_clause_specified);
1665 if (clone_info == NULL)
1666 continue;
1667
1668 int orig_simdlen = clone_info->simdlen;
1669 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1670 /* The target can return 0 (no simd clones should be created),
1671 1 (just one ISA of simd clones should be created) or higher
1672 count of ISA variants. In that case, clone_info is initialized
1673 for the first ISA variant. */
1674 int count
1675 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1676 base_type, 0);
1677 if (count == 0)
1678 continue;
1679
1680 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1681 also create one inbranch and one !inbranch clone of it. */
1682 for (int i = 0; i < count * 2; i++)
1683 {
1684 struct cgraph_simd_clone *clone = clone_info;
1685 if (inbranch_clause_specified && (i & 1) != 0)
1686 continue;
1687
1688 if (i != 0)
1689 {
1690 clone = simd_clone_struct_alloc (clone_info->nargs
1691 + ((i & 1) != 0));
1692 simd_clone_struct_copy (clone, clone_info);
1693 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1694 and simd_clone_adjust_argument_types did to the first
1695 clone's info. */
1696 clone->nargs -= clone_info->inbranch;
1697 clone->simdlen = orig_simdlen;
1698 /* And call the target hook again to get the right ISA. */
1699 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1700 base_type,
1701 i / 2);
1702 if ((i & 1) != 0)
1703 clone->inbranch = 1;
1704 }
1705
1706 /* simd_clone_mangle might fail if such a clone has been created
1707 already. */
1708 tree id = simd_clone_mangle (node, clone);
1709 if (id == NULL_TREE)
1710 continue;
1711
1712 /* Only when we are sure we want to create the clone actually
1713 clone the function (or definitions) or create another
1714 extern FUNCTION_DECL (for prototypes without definitions). */
1715 struct cgraph_node *n = simd_clone_create (node);
1716 if (n == NULL)
1717 continue;
1718
1719 n->simdclone = clone;
1720 clone->origin = node;
1721 clone->next_clone = NULL;
1722 if (node->simd_clones == NULL)
1723 {
1724 clone->prev_clone = n;
1725 node->simd_clones = n;
1726 }
1727 else
1728 {
1729 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1730 clone->prev_clone->simdclone->next_clone = n;
1731 node->simd_clones->simdclone->prev_clone = n;
1732 }
1733 symtab->change_decl_assembler_name (n->decl, id);
1734 /* And finally adjust the return type, parameters and for
1735 definitions also function body. */
1736 if (node->definition)
1737 simd_clone_adjust (n);
1738 else
1739 {
1740 targetm.simd_clone.adjust (n);
1741 simd_clone_adjust_return_type (n);
1742 simd_clone_adjust_argument_types (n);
1743 }
1744 }
1745 }
1746 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1747 }
1748
1749 /* Entry point for IPA simd clone creation pass. */
1750
1751 static unsigned int
ipa_omp_simd_clone(void)1752 ipa_omp_simd_clone (void)
1753 {
1754 struct cgraph_node *node;
1755 FOR_EACH_FUNCTION (node)
1756 expand_simd_clones (node);
1757 return 0;
1758 }
1759
1760 namespace {
1761
1762 const pass_data pass_data_omp_simd_clone =
1763 {
1764 SIMPLE_IPA_PASS, /* type */
1765 "simdclone", /* name */
1766 OPTGROUP_OMP, /* optinfo_flags */
1767 TV_NONE, /* tv_id */
1768 ( PROP_ssa | PROP_cfg ), /* properties_required */
1769 0, /* properties_provided */
1770 0, /* properties_destroyed */
1771 0, /* todo_flags_start */
1772 0, /* todo_flags_finish */
1773 };
1774
1775 class pass_omp_simd_clone : public simple_ipa_opt_pass
1776 {
1777 public:
pass_omp_simd_clone(gcc::context * ctxt)1778 pass_omp_simd_clone(gcc::context *ctxt)
1779 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1780 {}
1781
1782 /* opt_pass methods: */
1783 virtual bool gate (function *);
execute(function *)1784 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1785 };
1786
1787 bool
gate(function *)1788 pass_omp_simd_clone::gate (function *)
1789 {
1790 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1791 }
1792
1793 } // anon namespace
1794
1795 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1796 make_pass_omp_simd_clone (gcc::context *ctxt)
1797 {
1798 return new pass_omp_simd_clone (ctxt);
1799 }
1800