1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2020 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Fill an empty vector ARGS with parameter types of function FNDECL. This
90 uses TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static void
simd_clone_vector_of_formal_parm_types(vec<tree> * args,tree fndecl)94 simd_clone_vector_of_formal_parm_types (vec<tree> *args, tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 {
98 push_function_arg_types (args, TREE_TYPE (fndecl));
99 return;
100 }
101 push_function_arg_decls (args, fndecl);
102 unsigned int i;
103 tree arg;
104 FOR_EACH_VEC_ELT (*args, i, arg)
105 (*args)[i] = TREE_TYPE ((*args)[i]);
106 }
107
108 /* Given a simd function in NODE, extract the simd specific
109 information from the OMP clauses passed in CLAUSES, and return
110 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
111 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
112 otherwise set to FALSE. */
113
114 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)115 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
116 bool *inbranch_specified)
117 {
118 auto_vec<tree> args;
119 simd_clone_vector_of_formal_parm_types (&args, node->decl);
120 tree t;
121 int n;
122 *inbranch_specified = false;
123
124 n = args.length ();
125 if (n > 0 && args.last () == void_type_node)
126 n--;
127
128 /* Allocate one more than needed just in case this is an in-branch
129 clone which will require a mask argument. */
130 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
131 clone_info->nargs = n;
132
133 if (!clauses)
134 goto out;
135
136 clauses = TREE_VALUE (clauses);
137 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
138 goto out;
139
140 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
141 {
142 switch (OMP_CLAUSE_CODE (t))
143 {
144 case OMP_CLAUSE_INBRANCH:
145 clone_info->inbranch = 1;
146 *inbranch_specified = true;
147 break;
148 case OMP_CLAUSE_NOTINBRANCH:
149 clone_info->inbranch = 0;
150 *inbranch_specified = true;
151 break;
152 case OMP_CLAUSE_SIMDLEN:
153 clone_info->simdlen
154 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
155 break;
156 case OMP_CLAUSE_LINEAR:
157 {
158 tree decl = OMP_CLAUSE_DECL (t);
159 tree step = OMP_CLAUSE_LINEAR_STEP (t);
160 int argno = TREE_INT_CST_LOW (decl);
161 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
162 {
163 enum cgraph_simd_clone_arg_type arg_type;
164 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
165 switch (OMP_CLAUSE_LINEAR_KIND (t))
166 {
167 case OMP_CLAUSE_LINEAR_REF:
168 arg_type
169 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
170 break;
171 case OMP_CLAUSE_LINEAR_UVAL:
172 arg_type
173 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
174 break;
175 case OMP_CLAUSE_LINEAR_VAL:
176 case OMP_CLAUSE_LINEAR_DEFAULT:
177 arg_type
178 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
179 break;
180 default:
181 gcc_unreachable ();
182 }
183 else
184 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
185 clone_info->args[argno].arg_type = arg_type;
186 clone_info->args[argno].linear_step = tree_to_shwi (step);
187 gcc_assert (clone_info->args[argno].linear_step >= 0
188 && clone_info->args[argno].linear_step < n);
189 }
190 else
191 {
192 if (POINTER_TYPE_P (args[argno]))
193 step = fold_convert (ssizetype, step);
194 if (!tree_fits_shwi_p (step))
195 {
196 warning_at (OMP_CLAUSE_LOCATION (t), 0,
197 "ignoring large linear step");
198 return NULL;
199 }
200 else if (integer_zerop (step))
201 {
202 warning_at (OMP_CLAUSE_LOCATION (t), 0,
203 "ignoring zero linear step");
204 return NULL;
205 }
206 else
207 {
208 enum cgraph_simd_clone_arg_type arg_type;
209 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
210 switch (OMP_CLAUSE_LINEAR_KIND (t))
211 {
212 case OMP_CLAUSE_LINEAR_REF:
213 arg_type
214 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
215 break;
216 case OMP_CLAUSE_LINEAR_UVAL:
217 arg_type
218 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
219 break;
220 case OMP_CLAUSE_LINEAR_VAL:
221 case OMP_CLAUSE_LINEAR_DEFAULT:
222 arg_type
223 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
224 break;
225 default:
226 gcc_unreachable ();
227 }
228 else
229 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
230 clone_info->args[argno].arg_type = arg_type;
231 clone_info->args[argno].linear_step = tree_to_shwi (step);
232 }
233 }
234 break;
235 }
236 case OMP_CLAUSE_UNIFORM:
237 {
238 tree decl = OMP_CLAUSE_DECL (t);
239 int argno = tree_to_uhwi (decl);
240 clone_info->args[argno].arg_type
241 = SIMD_CLONE_ARG_TYPE_UNIFORM;
242 break;
243 }
244 case OMP_CLAUSE_ALIGNED:
245 {
246 /* Ignore aligned (x) for declare simd, for the ABI we really
247 need an alignment specified. */
248 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
249 break;
250 tree decl = OMP_CLAUSE_DECL (t);
251 int argno = tree_to_uhwi (decl);
252 clone_info->args[argno].alignment
253 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
254 break;
255 }
256 default:
257 break;
258 }
259 }
260
261 out:
262 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
263 {
264 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
265 "ignoring %<#pragma omp declare simd%> on function "
266 "with %<_Atomic%> qualified return type");
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 return clone_info;
282 }
283
284 /* Given a SIMD clone in NODE, calculate the characteristic data
285 type and return the coresponding type. The characteristic data
286 type is computed as described in the Intel Vector ABI. */
287
288 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)289 simd_clone_compute_base_data_type (struct cgraph_node *node,
290 struct cgraph_simd_clone *clone_info)
291 {
292 tree type = integer_type_node;
293 tree fndecl = node->decl;
294
295 /* a) For non-void function, the characteristic data type is the
296 return type. */
297 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
298 type = TREE_TYPE (TREE_TYPE (fndecl));
299
300 /* b) If the function has any non-uniform, non-linear parameters,
301 then the characteristic data type is the type of the first
302 such parameter. */
303 else
304 {
305 auto_vec<tree> map;
306 simd_clone_vector_of_formal_parm_types (&map, fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 }
314
315 /* c) If the characteristic data type determined by a) or b) above
316 is struct, union, or class type which is pass-by-value (except
317 for the type that maps to the built-in complex data type), the
318 characteristic data type is int. */
319 if (RECORD_OR_UNION_TYPE_P (type)
320 && !aggregate_value_p (type, NULL)
321 && TREE_CODE (type) != COMPLEX_TYPE)
322 return integer_type_node;
323
324 /* d) If none of the above three classes is applicable, the
325 characteristic data type is int. */
326
327 return type;
328
329 /* e) For Intel Xeon Phi native and offload compilation, if the
330 resulting characteristic data type is 8-bit or 16-bit integer
331 data type, the characteristic data type is int. */
332 /* Well, we don't handle Xeon Phi yet. */
333 }
334
335 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)336 simd_clone_mangle (struct cgraph_node *node,
337 struct cgraph_simd_clone *clone_info)
338 {
339 char vecsize_mangle = clone_info->vecsize_mangle;
340 char mask = clone_info->inbranch ? 'M' : 'N';
341 unsigned int simdlen = clone_info->simdlen;
342 unsigned int n;
343 pretty_printer pp;
344
345 gcc_assert (vecsize_mangle && simdlen);
346
347 pp_string (&pp, "_ZGV");
348 pp_character (&pp, vecsize_mangle);
349 pp_character (&pp, mask);
350 pp_decimal_int (&pp, simdlen);
351
352 for (n = 0; n < clone_info->nargs; ++n)
353 {
354 struct cgraph_simd_clone_arg arg = clone_info->args[n];
355
356 switch (arg.arg_type)
357 {
358 case SIMD_CLONE_ARG_TYPE_UNIFORM:
359 pp_character (&pp, 'u');
360 break;
361 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
362 pp_character (&pp, 'l');
363 goto mangle_linear;
364 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
365 pp_character (&pp, 'R');
366 goto mangle_linear;
367 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
368 pp_character (&pp, 'L');
369 goto mangle_linear;
370 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
371 pp_character (&pp, 'U');
372 goto mangle_linear;
373 mangle_linear:
374 gcc_assert (arg.linear_step != 0);
375 if (arg.linear_step > 1)
376 pp_unsigned_wide_integer (&pp, arg.linear_step);
377 else if (arg.linear_step < 0)
378 {
379 pp_character (&pp, 'n');
380 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
381 arg.linear_step));
382 }
383 break;
384 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
385 pp_string (&pp, "ls");
386 pp_unsigned_wide_integer (&pp, arg.linear_step);
387 break;
388 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
389 pp_string (&pp, "Rs");
390 pp_unsigned_wide_integer (&pp, arg.linear_step);
391 break;
392 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
393 pp_string (&pp, "Ls");
394 pp_unsigned_wide_integer (&pp, arg.linear_step);
395 break;
396 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
397 pp_string (&pp, "Us");
398 pp_unsigned_wide_integer (&pp, arg.linear_step);
399 break;
400 default:
401 pp_character (&pp, 'v');
402 }
403 if (arg.alignment)
404 {
405 pp_character (&pp, 'a');
406 pp_decimal_int (&pp, arg.alignment);
407 }
408 }
409
410 pp_underscore (&pp);
411 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
412 if (*str == '*')
413 ++str;
414 pp_string (&pp, str);
415 str = pp_formatted_text (&pp);
416
417 /* If there already is a SIMD clone with the same mangled name, don't
418 add another one. This can happen e.g. for
419 #pragma omp declare simd
420 #pragma omp declare simd simdlen(8)
421 int foo (int, int);
422 if the simdlen is assumed to be 8 for the first one, etc. */
423 for (struct cgraph_node *clone = node->simd_clones; clone;
424 clone = clone->simdclone->next_clone)
425 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
426 return NULL_TREE;
427
428 return get_identifier (str);
429 }
430
431 /* Create a simd clone of OLD_NODE and return it. */
432
433 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)434 simd_clone_create (struct cgraph_node *old_node)
435 {
436 struct cgraph_node *new_node;
437 if (old_node->definition)
438 {
439 if (!old_node->has_gimple_body_p ())
440 return NULL;
441 old_node->get_body ();
442 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
443 NULL, NULL,
444 "simdclone");
445 }
446 else
447 {
448 tree old_decl = old_node->decl;
449 tree new_decl = copy_node (old_node->decl);
450 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl,
451 "simdclone");
452 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
453 SET_DECL_RTL (new_decl, NULL);
454 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
455 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
456 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
457 if (old_node->in_other_partition)
458 new_node->in_other_partition = 1;
459 }
460 if (new_node == NULL)
461 return new_node;
462
463 set_decl_built_in_function (new_node->decl, NOT_BUILT_IN, 0);
464 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
465 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
466 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
467 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
468 DECL_VISIBILITY_SPECIFIED (new_node->decl)
469 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
470 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
471 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
472 if (DECL_ONE_ONLY (old_node->decl))
473 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
474
475 /* The method cgraph_version_clone_with_body () will force the new
476 symbol local. Undo this, and inherit external visibility from
477 the old node. */
478 new_node->local = old_node->local;
479 new_node->externally_visible = old_node->externally_visible;
480
481 return new_node;
482 }
483
484 /* Adjust the return type of the given function to its appropriate
485 vector counterpart. Returns a simd array to be used throughout the
486 function as a return value. */
487
488 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)489 simd_clone_adjust_return_type (struct cgraph_node *node)
490 {
491 tree fndecl = node->decl;
492 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
493 unsigned int veclen;
494 tree t;
495
496 /* Adjust the function return type. */
497 if (orig_rettype == void_type_node)
498 return NULL_TREE;
499 t = TREE_TYPE (TREE_TYPE (fndecl));
500 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
501 veclen = node->simdclone->vecsize_int;
502 else
503 veclen = node->simdclone->vecsize_float;
504 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
505 if (veclen > node->simdclone->simdlen)
506 veclen = node->simdclone->simdlen;
507 if (POINTER_TYPE_P (t))
508 t = pointer_sized_int_node;
509 if (veclen == node->simdclone->simdlen)
510 t = build_vector_type (t, node->simdclone->simdlen);
511 else
512 {
513 t = build_vector_type (t, veclen);
514 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
515 }
516 TREE_TYPE (TREE_TYPE (fndecl)) = t;
517 if (!node->definition)
518 return NULL_TREE;
519
520 t = DECL_RESULT (fndecl);
521 /* Adjust the DECL_RESULT. */
522 gcc_assert (TREE_TYPE (t) != void_type_node);
523 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
524 relayout_decl (t);
525
526 tree atype = build_array_type_nelts (orig_rettype,
527 node->simdclone->simdlen);
528 if (veclen != node->simdclone->simdlen)
529 return build1 (VIEW_CONVERT_EXPR, atype, t);
530
531 /* Set up a SIMD array to use as the return value. */
532 tree retval = create_tmp_var_raw (atype, "retval");
533 gimple_add_tmp_var (retval);
534 return retval;
535 }
536
537 /* Each vector argument has a corresponding array to be used locally
538 as part of the eventual loop. Create such temporary array and
539 return it.
540
541 PREFIX is the prefix to be used for the temporary.
542
543 TYPE is the inner element type.
544
545 SIMDLEN is the number of elements. */
546
547 static tree
create_tmp_simd_array(const char * prefix,tree type,int simdlen)548 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
549 {
550 tree atype = build_array_type_nelts (type, simdlen);
551 tree avar = create_tmp_var_raw (atype, prefix);
552 gimple_add_tmp_var (avar);
553 return avar;
554 }
555
556 /* Modify the function argument types to their corresponding vector
557 counterparts if appropriate. Also, create one array for each simd
558 argument to be used locally when using the function arguments as
559 part of the loop.
560
561 NODE is the function whose arguments are to be adjusted.
562
563 If NODE does not represent function definition, returns NULL. Otherwise
564 returns an adjustment class that will be filled describing how the argument
565 declarations will be remapped. New arguments which are not to be remapped
566 are marked with USER_FLAG. */
567
568 static ipa_param_body_adjustments *
simd_clone_adjust_argument_types(struct cgraph_node * node)569 simd_clone_adjust_argument_types (struct cgraph_node *node)
570 {
571 auto_vec<tree> args;
572
573 if (node->definition)
574 push_function_arg_decls (&args, node->decl);
575 else
576 simd_clone_vector_of_formal_parm_types (&args, node->decl);
577 struct cgraph_simd_clone *sc = node->simdclone;
578 vec<ipa_adjusted_param, va_gc> *new_params = NULL;
579 vec_safe_reserve (new_params, sc->nargs);
580 unsigned i, j, veclen;
581
582 for (i = 0; i < sc->nargs; ++i)
583 {
584 ipa_adjusted_param adj;
585 memset (&adj, 0, sizeof (adj));
586 tree parm = args[i];
587 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
588 adj.base_index = i;
589 adj.prev_clone_index = i;
590
591 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
592 sc->args[i].orig_type = parm_type;
593
594 switch (sc->args[i].arg_type)
595 {
596 default:
597 /* No adjustment necessary for scalar arguments. */
598 adj.op = IPA_PARAM_OP_COPY;
599 break;
600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
602 if (node->definition)
603 sc->args[i].simd_array
604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
605 TREE_TYPE (parm_type),
606 sc->simdlen);
607 adj.op = IPA_PARAM_OP_COPY;
608 break;
609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
611 case SIMD_CLONE_ARG_TYPE_VECTOR:
612 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
613 veclen = sc->vecsize_int;
614 else
615 veclen = sc->vecsize_float;
616 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
617 if (veclen > sc->simdlen)
618 veclen = sc->simdlen;
619 adj.op = IPA_PARAM_OP_NEW;
620 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
621 if (POINTER_TYPE_P (parm_type))
622 adj.type = build_vector_type (pointer_sized_int_node, veclen);
623 else
624 adj.type = build_vector_type (parm_type, veclen);
625 sc->args[i].vector_type = adj.type;
626 for (j = veclen; j < sc->simdlen; j += veclen)
627 {
628 vec_safe_push (new_params, adj);
629 if (j == veclen)
630 {
631 memset (&adj, 0, sizeof (adj));
632 adj.op = IPA_PARAM_OP_NEW;
633 adj.user_flag = 1;
634 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
635 adj.base_index = i;
636 adj.prev_clone_index = i;
637 adj.type = sc->args[i].vector_type;
638 }
639 }
640
641 if (node->definition)
642 sc->args[i].simd_array
643 = create_tmp_simd_array (DECL_NAME (parm)
644 ? IDENTIFIER_POINTER (DECL_NAME (parm))
645 : NULL, parm_type, sc->simdlen);
646 }
647 vec_safe_push (new_params, adj);
648 }
649
650 if (sc->inbranch)
651 {
652 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
653 ipa_adjusted_param adj;
654 memset (&adj, 0, sizeof (adj));
655 adj.op = IPA_PARAM_OP_NEW;
656 adj.user_flag = 1;
657 adj.param_prefix_index = IPA_PARAM_PREFIX_MASK;
658
659 adj.base_index = i;
660 adj.prev_clone_index = i;
661 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
662 veclen = sc->vecsize_int;
663 else
664 veclen = sc->vecsize_float;
665 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
666 if (veclen > sc->simdlen)
667 veclen = sc->simdlen;
668 if (sc->mask_mode != VOIDmode)
669 adj.type
670 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
671 else if (POINTER_TYPE_P (base_type))
672 adj.type = build_vector_type (pointer_sized_int_node, veclen);
673 else
674 adj.type = build_vector_type (base_type, veclen);
675 vec_safe_push (new_params, adj);
676
677 for (j = veclen; j < sc->simdlen; j += veclen)
678 vec_safe_push (new_params, adj);
679
680 /* We have previously allocated one extra entry for the mask. Use
681 it and fill it. */
682 sc->nargs++;
683 if (sc->mask_mode != VOIDmode)
684 base_type = boolean_type_node;
685 if (node->definition)
686 {
687 sc->args[i].orig_arg
688 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
689 if (sc->mask_mode == VOIDmode)
690 sc->args[i].simd_array
691 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
692 else if (veclen < sc->simdlen)
693 sc->args[i].simd_array
694 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
695 else
696 sc->args[i].simd_array = NULL_TREE;
697 }
698 sc->args[i].orig_type = base_type;
699 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
700 }
701
702 if (node->definition)
703 {
704 ipa_param_body_adjustments *adjustments
705 = new ipa_param_body_adjustments (new_params, node->decl);
706
707 adjustments->modify_formal_parameters ();
708 return adjustments;
709 }
710 else
711 {
712 tree new_arg_types = NULL_TREE, new_reversed;
713 bool last_parm_void = false;
714 if (args.length () > 0 && args.last () == void_type_node)
715 last_parm_void = true;
716
717 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
718 j = vec_safe_length (new_params);
719 for (i = 0; i < j; i++)
720 {
721 struct ipa_adjusted_param *adj = &(*new_params)[i];
722 tree ptype;
723 if (adj->op == IPA_PARAM_OP_COPY)
724 ptype = args[adj->base_index];
725 else
726 ptype = adj->type;
727 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
728 }
729 new_reversed = nreverse (new_arg_types);
730 if (last_parm_void)
731 {
732 if (new_reversed)
733 TREE_CHAIN (new_arg_types) = void_list_node;
734 else
735 new_reversed = void_list_node;
736 }
737 TYPE_ARG_TYPES (TREE_TYPE (node->decl)) = new_reversed;
738 return NULL;
739 }
740 }
741
742 /* Initialize and copy the function arguments in NODE to their
743 corresponding local simd arrays. Returns a fresh gimple_seq with
744 the instruction sequence generated. */
745
746 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_param_body_adjustments * adjustments)747 simd_clone_init_simd_arrays (struct cgraph_node *node,
748 ipa_param_body_adjustments *adjustments)
749 {
750 gimple_seq seq = NULL;
751 unsigned i = 0, j = 0, k;
752
753 for (tree arg = DECL_ARGUMENTS (node->decl);
754 arg;
755 arg = DECL_CHAIN (arg), i++, j++)
756 {
757 if ((*adjustments->m_adj_params)[j].op == IPA_PARAM_OP_COPY
758 || POINTER_TYPE_P (TREE_TYPE (arg)))
759 continue;
760
761 node->simdclone->args[i].vector_arg = arg;
762
763 tree array = node->simdclone->args[i].simd_array;
764 if (node->simdclone->mask_mode != VOIDmode
765 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
766 {
767 if (array == NULL_TREE)
768 continue;
769 unsigned int l
770 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
771 for (k = 0; k <= l; k++)
772 {
773 if (k)
774 {
775 arg = DECL_CHAIN (arg);
776 j++;
777 }
778 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
779 array, size_int (k), NULL, NULL);
780 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
781 gimplify_and_add (t, &seq);
782 }
783 continue;
784 }
785 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen)
786 {
787 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
788 tree ptr = build_fold_addr_expr (array);
789 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
790 build_int_cst (ptype, 0));
791 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
792 gimplify_and_add (t, &seq);
793 }
794 else
795 {
796 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
797 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
798 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
799 {
800 tree ptr = build_fold_addr_expr (array);
801 int elemsize;
802 if (k)
803 {
804 arg = DECL_CHAIN (arg);
805 j++;
806 }
807 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
808 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
809 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
810 build_int_cst (ptype, k * elemsize));
811 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
812 gimplify_and_add (t, &seq);
813 }
814 }
815 }
816 return seq;
817 }
818
819 /* Callback info for ipa_simd_modify_stmt_ops below. */
820
821 struct modify_stmt_info {
822 ipa_param_body_adjustments *adjustments;
823 gimple *stmt;
824 gimple *after_stmt;
825 /* True if the parent statement was modified by
826 ipa_simd_modify_stmt_ops. */
827 bool modified;
828 };
829
830 /* Callback for walk_gimple_op.
831
832 Adjust operands from a given statement as specified in the
833 adjustments vector in the callback data. */
834
835 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)836 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
837 {
838 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
839 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
840 tree *orig_tp = tp;
841 if (TREE_CODE (*tp) == ADDR_EXPR)
842 tp = &TREE_OPERAND (*tp, 0);
843
844 if (TREE_CODE (*tp) == BIT_FIELD_REF
845 || TREE_CODE (*tp) == IMAGPART_EXPR
846 || TREE_CODE (*tp) == REALPART_EXPR)
847 tp = &TREE_OPERAND (*tp, 0);
848
849 tree repl = NULL_TREE;
850 ipa_param_body_replacement *pbr = NULL;
851
852 if (TREE_CODE (*tp) == PARM_DECL)
853 {
854 pbr = info->adjustments->get_expr_replacement (*tp, true);
855 if (pbr)
856 repl = pbr->repl;
857 }
858 else if (TYPE_P (*tp))
859 *walk_subtrees = 0;
860
861 if (repl)
862 repl = unshare_expr (repl);
863 else
864 {
865 if (tp != orig_tp)
866 {
867 *walk_subtrees = 0;
868 bool modified = info->modified;
869 info->modified = false;
870 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
871 if (!info->modified)
872 {
873 info->modified = modified;
874 return NULL_TREE;
875 }
876 info->modified = modified;
877 repl = *tp;
878 }
879 else
880 return NULL_TREE;
881 }
882
883 if (tp != orig_tp)
884 {
885 if (gimple_code (info->stmt) == GIMPLE_PHI
886 && pbr
887 && TREE_CODE (*orig_tp) == ADDR_EXPR
888 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
889 && pbr->dummy)
890 {
891 gcc_assert (TREE_CODE (pbr->dummy) == SSA_NAME);
892 *orig_tp = pbr->dummy;
893 info->modified = true;
894 return NULL_TREE;
895 }
896
897 repl = build_fold_addr_expr (repl);
898 gimple *stmt;
899 if (is_gimple_debug (info->stmt))
900 {
901 tree vexpr = make_node (DEBUG_EXPR_DECL);
902 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
903 DECL_ARTIFICIAL (vexpr) = 1;
904 TREE_TYPE (vexpr) = TREE_TYPE (repl);
905 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
906 repl = vexpr;
907 }
908 else
909 {
910 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
911 repl = gimple_assign_lhs (stmt);
912 }
913 gimple_stmt_iterator gsi;
914 if (gimple_code (info->stmt) == GIMPLE_PHI)
915 {
916 if (info->after_stmt)
917 gsi = gsi_for_stmt (info->after_stmt);
918 else
919 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
920 /* Cache SSA_NAME for next time. */
921 if (pbr
922 && TREE_CODE (*orig_tp) == ADDR_EXPR
923 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
924 {
925 gcc_assert (!pbr->dummy);
926 pbr->dummy = repl;
927 }
928 }
929 else
930 gsi = gsi_for_stmt (info->stmt);
931 if (info->after_stmt)
932 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
933 else
934 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
935 if (gimple_code (info->stmt) == GIMPLE_PHI)
936 info->after_stmt = stmt;
937 *orig_tp = repl;
938 }
939 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
940 {
941 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
942 *tp = vce;
943 }
944 else
945 *tp = repl;
946
947 info->modified = true;
948 return NULL_TREE;
949 }
950
951 /* Traverse the function body and perform all modifications as
952 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
953 modified such that the replacement/reduction value will now be an
954 offset into the corresponding simd_array.
955
956 This function will replace all function argument uses with their
957 corresponding simd array elements, and ajust the return values
958 accordingly. */
959
960 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_param_body_adjustments * adjustments,tree retval_array,tree iter)961 ipa_simd_modify_function_body (struct cgraph_node *node,
962 ipa_param_body_adjustments *adjustments,
963 tree retval_array, tree iter)
964 {
965 basic_block bb;
966 unsigned int i, j;
967
968
969 /* Register replacements for every function argument use to an offset into
970 the corresponding simd_array. */
971 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
972 {
973 if (!node->simdclone->args[i].vector_arg
974 || (*adjustments->m_adj_params)[j].user_flag)
975 continue;
976
977 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
978 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
979 tree r = build4 (ARRAY_REF, basetype, node->simdclone->args[i].simd_array,
980 iter, NULL_TREE, NULL_TREE);
981 adjustments->register_replacement (&(*adjustments->m_adj_params)[j], r);
982
983 if (simd_clone_subparts (vectype) < node->simdclone->simdlen)
984 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1;
985 }
986
987 tree name;
988 FOR_EACH_SSA_NAME (i, name, cfun)
989 {
990 tree base_var;
991 if (SSA_NAME_VAR (name)
992 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
993 && (base_var
994 = adjustments->get_replacement_ssa_base (SSA_NAME_VAR (name))))
995 {
996 if (SSA_NAME_IS_DEFAULT_DEF (name))
997 {
998 tree old_decl = SSA_NAME_VAR (name);
999 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1000 gimple_stmt_iterator gsi = gsi_after_labels (bb);
1001 tree repl = adjustments->lookup_replacement (old_decl, 0);
1002 gcc_checking_assert (repl);
1003 repl = unshare_expr (repl);
1004 set_ssa_default_def (cfun, old_decl, NULL_TREE);
1005 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1006 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
1007 gimple *stmt = gimple_build_assign (name, repl);
1008 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1009 }
1010 else
1011 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1012 }
1013 }
1014
1015 struct modify_stmt_info info;
1016 info.adjustments = adjustments;
1017
1018 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1019 {
1020 gimple_stmt_iterator gsi;
1021
1022 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1023 {
1024 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1025 int i, n = gimple_phi_num_args (phi);
1026 info.stmt = phi;
1027 info.after_stmt = NULL;
1028 struct walk_stmt_info wi;
1029 memset (&wi, 0, sizeof (wi));
1030 info.modified = false;
1031 wi.info = &info;
1032 for (i = 0; i < n; ++i)
1033 {
1034 int walk_subtrees = 1;
1035 tree arg = gimple_phi_arg_def (phi, i);
1036 tree op = arg;
1037 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1038 if (op != arg)
1039 {
1040 SET_PHI_ARG_DEF (phi, i, op);
1041 gcc_assert (TREE_CODE (op) == SSA_NAME);
1042 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1043 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1044 }
1045 }
1046 }
1047
1048 gsi = gsi_start_bb (bb);
1049 while (!gsi_end_p (gsi))
1050 {
1051 gimple *stmt = gsi_stmt (gsi);
1052 info.stmt = stmt;
1053 info.after_stmt = NULL;
1054 struct walk_stmt_info wi;
1055
1056 memset (&wi, 0, sizeof (wi));
1057 info.modified = false;
1058 wi.info = &info;
1059 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1060
1061 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1062 {
1063 tree retval = gimple_return_retval (return_stmt);
1064 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1065 e->flags |= EDGE_FALLTHRU;
1066 if (!retval)
1067 {
1068 gsi_remove (&gsi, true);
1069 continue;
1070 }
1071
1072 /* Replace `return foo' with `retval_array[iter] = foo'. */
1073 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1074 retval_array, iter, NULL, NULL);
1075 stmt = gimple_build_assign (ref, retval);
1076 gsi_replace (&gsi, stmt, true);
1077 info.modified = true;
1078 }
1079
1080 if (info.modified)
1081 {
1082 update_stmt (stmt);
1083 /* If the above changed the var of a debug bind into something
1084 different, remove the debug stmt. We could also for all the
1085 replaced parameters add VAR_DECLs for debug info purposes,
1086 add debug stmts for those to be the simd array accesses and
1087 replace debug stmt var operand with that var. Debugging of
1088 vectorized loops doesn't work too well, so don't bother for
1089 now. */
1090 if ((gimple_debug_bind_p (stmt)
1091 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1092 || (gimple_debug_source_bind_p (stmt)
1093 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1094 {
1095 gsi_remove (&gsi, true);
1096 continue;
1097 }
1098 if (maybe_clean_eh_stmt (stmt))
1099 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1100 }
1101 gsi_next (&gsi);
1102 }
1103 }
1104 }
1105
1106 /* Helper function of simd_clone_adjust, return linear step addend
1107 of Ith argument. */
1108
1109 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1110 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1111 tree addtype, basic_block entry_bb)
1112 {
1113 tree ptype = NULL_TREE;
1114 switch (node->simdclone->args[i].arg_type)
1115 {
1116 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1117 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1118 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1119 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1120 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1121 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1122 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1123 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1124 break;
1125 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1126 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1127 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1128 break;
1129 default:
1130 gcc_unreachable ();
1131 }
1132
1133 unsigned int idx = node->simdclone->args[i].linear_step;
1134 tree arg = node->simdclone->args[idx].orig_arg;
1135 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1136 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1137 gimple *g;
1138 tree ret;
1139 if (is_gimple_reg (arg))
1140 ret = get_or_create_ssa_default_def (cfun, arg);
1141 else
1142 {
1143 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1144 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1145 ret = gimple_assign_lhs (g);
1146 }
1147 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1148 {
1149 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1150 build_simple_mem_ref (ret));
1151 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1152 ret = gimple_assign_lhs (g);
1153 }
1154 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1155 {
1156 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1157 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1158 ret = gimple_assign_lhs (g);
1159 }
1160 if (POINTER_TYPE_P (ptype))
1161 {
1162 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1163 if (size && TREE_CODE (size) == INTEGER_CST)
1164 {
1165 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1166 ret, fold_convert (addtype, size));
1167 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1168 ret = gimple_assign_lhs (g);
1169 }
1170 }
1171 return ret;
1172 }
1173
1174 /* Adjust the argument types in NODE to their appropriate vector
1175 counterparts. */
1176
1177 static void
simd_clone_adjust(struct cgraph_node * node)1178 simd_clone_adjust (struct cgraph_node *node)
1179 {
1180 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1181
1182 TREE_TYPE (node->decl) = build_distinct_type_copy (TREE_TYPE (node->decl));
1183 targetm.simd_clone.adjust (node);
1184
1185 tree retval = simd_clone_adjust_return_type (node);
1186 ipa_param_body_adjustments *adjustments
1187 = simd_clone_adjust_argument_types (node);
1188 gcc_assert (adjustments);
1189
1190 push_gimplify_context ();
1191
1192 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1193
1194 /* Adjust all uses of vector arguments accordingly. Adjust all
1195 return values accordingly. */
1196 tree iter = create_tmp_var (unsigned_type_node, "iter");
1197 tree iter1 = make_ssa_name (iter);
1198 tree iter2 = NULL_TREE;
1199 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1200 delete adjustments;
1201
1202 /* Initialize the iteration variable. */
1203 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1204 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1205 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1206 /* Insert the SIMD array and iv initialization at function
1207 entry. */
1208 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1209
1210 pop_gimplify_context (NULL);
1211
1212 gimple *g;
1213 basic_block incr_bb = NULL;
1214 class loop *loop = NULL;
1215
1216 /* Create a new BB right before the original exit BB, to hold the
1217 iteration increment and the condition/branch. */
1218 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1219 {
1220 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1221 incr_bb = create_empty_bb (orig_exit);
1222 incr_bb->count = profile_count::zero ();
1223 add_bb_to_loop (incr_bb, body_bb->loop_father);
1224 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1225 {
1226 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1227 redirect_edge_succ (e, incr_bb);
1228 incr_bb->count += e->count ();
1229 }
1230 }
1231 else if (node->simdclone->inbranch)
1232 {
1233 incr_bb = create_empty_bb (entry_bb);
1234 incr_bb->count = profile_count::zero ();
1235 add_bb_to_loop (incr_bb, body_bb->loop_father);
1236 }
1237
1238 if (incr_bb)
1239 {
1240 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1241 gsi = gsi_last_bb (incr_bb);
1242 iter2 = make_ssa_name (iter);
1243 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1244 build_int_cst (unsigned_type_node, 1));
1245 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1246
1247 /* Mostly annotate the loop for the vectorizer (the rest is done
1248 below). */
1249 loop = alloc_loop ();
1250 cfun->has_force_vectorize_loops = true;
1251 loop->safelen = node->simdclone->simdlen;
1252 loop->force_vectorize = true;
1253 loop->header = body_bb;
1254 }
1255
1256 /* Branch around the body if the mask applies. */
1257 if (node->simdclone->inbranch)
1258 {
1259 gsi = gsi_last_bb (loop->header);
1260 tree mask_array
1261 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1262 tree mask;
1263 if (node->simdclone->mask_mode != VOIDmode)
1264 {
1265 tree shift_cnt;
1266 if (mask_array == NULL_TREE)
1267 {
1268 tree arg = node->simdclone->args[node->simdclone->nargs
1269 - 1].vector_arg;
1270 mask = get_or_create_ssa_default_def (cfun, arg);
1271 shift_cnt = iter1;
1272 }
1273 else
1274 {
1275 tree maskt = TREE_TYPE (mask_array);
1276 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1277 c = node->simdclone->simdlen / (c + 1);
1278 int s = exact_log2 (c);
1279 gcc_assert (s > 0);
1280 c--;
1281 tree idx = make_ssa_name (TREE_TYPE (iter1));
1282 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1283 build_int_cst (NULL_TREE, s));
1284 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1285 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1286 tree aref = build4 (ARRAY_REF,
1287 TREE_TYPE (TREE_TYPE (mask_array)),
1288 mask_array, idx, NULL, NULL);
1289 g = gimple_build_assign (mask, aref);
1290 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1291 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1292 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1293 build_int_cst (TREE_TYPE (iter1), c));
1294 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1295 }
1296 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1297 RSHIFT_EXPR, mask, shift_cnt);
1298 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1299 mask = gimple_assign_lhs (g);
1300 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1301 BIT_AND_EXPR, mask,
1302 build_int_cst (TREE_TYPE (mask), 1));
1303 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1304 mask = gimple_assign_lhs (g);
1305 }
1306 else
1307 {
1308 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1309 tree aref = build4 (ARRAY_REF,
1310 TREE_TYPE (TREE_TYPE (mask_array)),
1311 mask_array, iter1, NULL, NULL);
1312 g = gimple_build_assign (mask, aref);
1313 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1314 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1315 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1316 {
1317 aref = build1 (VIEW_CONVERT_EXPR,
1318 build_nonstandard_integer_type (bitsize, 0),
1319 mask);
1320 mask = make_ssa_name (TREE_TYPE (aref));
1321 g = gimple_build_assign (mask, aref);
1322 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1323 }
1324 }
1325
1326 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1327 NULL, NULL);
1328 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1329 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1330 e->probability = profile_probability::unlikely ().guessed ();
1331 incr_bb->count += e->count ();
1332 edge fallthru = FALLTHRU_EDGE (loop->header);
1333 fallthru->flags = EDGE_FALSE_VALUE;
1334 fallthru->probability = profile_probability::likely ().guessed ();
1335 }
1336
1337 basic_block latch_bb = NULL;
1338 basic_block new_exit_bb = NULL;
1339
1340 /* Generate the condition. */
1341 if (incr_bb)
1342 {
1343 gsi = gsi_last_bb (incr_bb);
1344 g = gimple_build_cond (LT_EXPR, iter2,
1345 build_int_cst (unsigned_type_node,
1346 node->simdclone->simdlen),
1347 NULL, NULL);
1348 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1349 edge e = split_block (incr_bb, gsi_stmt (gsi));
1350 latch_bb = e->dest;
1351 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1352 loop->latch = latch_bb;
1353
1354 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1355
1356 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1357
1358 /* FIXME: Do we need to distribute probabilities for the conditional? */
1359 new_e->probability = profile_probability::guessed_never ();
1360 /* The successor of incr_bb is already pointing to latch_bb; just
1361 change the flags.
1362 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1363 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1364 }
1365
1366 gphi *phi = create_phi_node (iter1, body_bb);
1367 edge preheader_edge = find_edge (entry_bb, body_bb);
1368 edge latch_edge = NULL;
1369 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1370 UNKNOWN_LOCATION);
1371 if (incr_bb)
1372 {
1373 latch_edge = single_succ_edge (latch_bb);
1374 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1375
1376 /* Generate the new return. */
1377 gsi = gsi_last_bb (new_exit_bb);
1378 if (retval
1379 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1380 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1381 retval = TREE_OPERAND (retval, 0);
1382 else if (retval)
1383 {
1384 retval = build1 (VIEW_CONVERT_EXPR,
1385 TREE_TYPE (TREE_TYPE (node->decl)),
1386 retval);
1387 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1388 false, GSI_CONTINUE_LINKING);
1389 }
1390 g = gimple_build_return (retval);
1391 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1392 }
1393
1394 /* Handle aligned clauses by replacing default defs of the aligned
1395 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1396 lhs. Handle linear by adding PHIs. */
1397 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1398 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1399 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1400 || !is_gimple_reg_type
1401 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1402 {
1403 tree orig_arg = node->simdclone->args[i].orig_arg;
1404 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1405 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1406 else
1407 {
1408 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1409 gimple_add_tmp_var (iter1);
1410 }
1411 gsi = gsi_after_labels (entry_bb);
1412 g = gimple_build_assign (iter1, orig_arg);
1413 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1414 gsi = gsi_after_labels (body_bb);
1415 g = gimple_build_assign (orig_arg, iter1);
1416 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1417 }
1418 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1419 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1420 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1421 == REFERENCE_TYPE
1422 && TREE_ADDRESSABLE
1423 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1424 {
1425 tree orig_arg = node->simdclone->args[i].orig_arg;
1426 tree def = ssa_default_def (cfun, orig_arg);
1427 if (def && !has_zero_uses (def))
1428 {
1429 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1430 gimple_add_tmp_var (iter1);
1431 gsi = gsi_after_labels (entry_bb);
1432 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1433 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1434 gsi = gsi_after_labels (body_bb);
1435 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1436 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1437 }
1438 }
1439 else if (node->simdclone->args[i].alignment
1440 && node->simdclone->args[i].arg_type
1441 == SIMD_CLONE_ARG_TYPE_UNIFORM
1442 && (node->simdclone->args[i].alignment
1443 & (node->simdclone->args[i].alignment - 1)) == 0
1444 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1445 == POINTER_TYPE)
1446 {
1447 unsigned int alignment = node->simdclone->args[i].alignment;
1448 tree orig_arg = node->simdclone->args[i].orig_arg;
1449 tree def = ssa_default_def (cfun, orig_arg);
1450 if (def && !has_zero_uses (def))
1451 {
1452 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1453 gimple_seq seq = NULL;
1454 bool need_cvt = false;
1455 gcall *call
1456 = gimple_build_call (fn, 2, def, size_int (alignment));
1457 g = call;
1458 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1459 ptr_type_node))
1460 need_cvt = true;
1461 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1462 gimple_call_set_lhs (g, t);
1463 gimple_seq_add_stmt_without_update (&seq, g);
1464 if (need_cvt)
1465 {
1466 t = make_ssa_name (orig_arg);
1467 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1468 gimple_seq_add_stmt_without_update (&seq, g);
1469 }
1470 gsi_insert_seq_on_edge_immediate
1471 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1472
1473 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1474 node->create_edge (cgraph_node::get_create (fn),
1475 call, entry_bb->count);
1476
1477 imm_use_iterator iter;
1478 use_operand_p use_p;
1479 gimple *use_stmt;
1480 tree repl = gimple_get_lhs (g);
1481 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1482 if (is_gimple_debug (use_stmt) || use_stmt == call)
1483 continue;
1484 else
1485 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1486 SET_USE (use_p, repl);
1487 }
1488 }
1489 else if ((node->simdclone->args[i].arg_type
1490 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1491 || (node->simdclone->args[i].arg_type
1492 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1493 || (node->simdclone->args[i].arg_type
1494 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1495 || (node->simdclone->args[i].arg_type
1496 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1497 {
1498 tree orig_arg = node->simdclone->args[i].orig_arg;
1499 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1500 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1501 tree def = NULL_TREE;
1502 if (TREE_ADDRESSABLE (orig_arg))
1503 {
1504 def = make_ssa_name (TREE_TYPE (orig_arg));
1505 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1506 if (incr_bb)
1507 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1508 gsi = gsi_after_labels (entry_bb);
1509 g = gimple_build_assign (def, orig_arg);
1510 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1511 }
1512 else
1513 {
1514 def = ssa_default_def (cfun, orig_arg);
1515 if (!def || has_zero_uses (def))
1516 def = NULL_TREE;
1517 else
1518 {
1519 iter1 = make_ssa_name (orig_arg);
1520 if (incr_bb)
1521 iter2 = make_ssa_name (orig_arg);
1522 }
1523 }
1524 if (def)
1525 {
1526 phi = create_phi_node (iter1, body_bb);
1527 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1528 if (incr_bb)
1529 {
1530 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1531 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1532 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1533 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1534 ? TREE_TYPE (orig_arg) : sizetype;
1535 tree addcst = simd_clone_linear_addend (node, i, addtype,
1536 entry_bb);
1537 gsi = gsi_last_bb (incr_bb);
1538 g = gimple_build_assign (iter2, code, iter1, addcst);
1539 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1540 }
1541
1542 imm_use_iterator iter;
1543 use_operand_p use_p;
1544 gimple *use_stmt;
1545 if (TREE_ADDRESSABLE (orig_arg))
1546 {
1547 gsi = gsi_after_labels (body_bb);
1548 g = gimple_build_assign (orig_arg, iter1);
1549 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1550 }
1551 else
1552 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1553 if (use_stmt == phi)
1554 continue;
1555 else
1556 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1557 SET_USE (use_p, iter1);
1558 }
1559 }
1560 else if (node->simdclone->args[i].arg_type
1561 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1562 || (node->simdclone->args[i].arg_type
1563 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1564 {
1565 tree orig_arg = node->simdclone->args[i].orig_arg;
1566 tree def = ssa_default_def (cfun, orig_arg);
1567 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1568 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1569 if (def && !has_zero_uses (def))
1570 {
1571 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1572 iter1 = make_ssa_name (orig_arg);
1573 if (incr_bb)
1574 iter2 = make_ssa_name (orig_arg);
1575 tree iter3 = make_ssa_name (rtype);
1576 tree iter4 = make_ssa_name (rtype);
1577 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1578 gsi = gsi_after_labels (entry_bb);
1579 gimple *load
1580 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1581 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1582
1583 tree array = node->simdclone->args[i].simd_array;
1584 TREE_ADDRESSABLE (array) = 1;
1585 tree ptr = build_fold_addr_expr (array);
1586 phi = create_phi_node (iter1, body_bb);
1587 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1588 if (incr_bb)
1589 {
1590 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1591 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1592 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1593 gsi = gsi_last_bb (incr_bb);
1594 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1595 }
1596
1597 phi = create_phi_node (iter4, body_bb);
1598 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1599 if (incr_bb)
1600 {
1601 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1602 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1603 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1604 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1605 ? TREE_TYPE (iter3) : sizetype;
1606 tree addcst = simd_clone_linear_addend (node, i, addtype,
1607 entry_bb);
1608 g = gimple_build_assign (iter5, code, iter4, addcst);
1609 gsi = gsi_last_bb (incr_bb);
1610 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1611 }
1612
1613 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1614 gsi = gsi_after_labels (body_bb);
1615 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1616
1617 imm_use_iterator iter;
1618 use_operand_p use_p;
1619 gimple *use_stmt;
1620 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1621 if (use_stmt == load)
1622 continue;
1623 else
1624 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1625 SET_USE (use_p, iter1);
1626
1627 if (!TYPE_READONLY (rtype) && incr_bb)
1628 {
1629 tree v = make_ssa_name (rtype);
1630 tree aref = build4 (ARRAY_REF, rtype, array,
1631 size_zero_node, NULL_TREE,
1632 NULL_TREE);
1633 gsi = gsi_after_labels (new_exit_bb);
1634 g = gimple_build_assign (v, aref);
1635 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1636 g = gimple_build_assign (build_simple_mem_ref (def), v);
1637 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1638 }
1639 }
1640 }
1641
1642 calculate_dominance_info (CDI_DOMINATORS);
1643 if (loop)
1644 add_loop (loop, loop->header->loop_father);
1645 update_ssa (TODO_update_ssa);
1646
1647 pop_cfun ();
1648 }
1649
1650 /* If the function in NODE is tagged as an elemental SIMD function,
1651 create the appropriate SIMD clones. */
1652
1653 void
expand_simd_clones(struct cgraph_node * node)1654 expand_simd_clones (struct cgraph_node *node)
1655 {
1656 tree attr = lookup_attribute ("omp declare simd",
1657 DECL_ATTRIBUTES (node->decl));
1658 if (attr == NULL_TREE
1659 || node->inlined_to
1660 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1661 return;
1662
1663 /* Ignore
1664 #pragma omp declare simd
1665 extern int foo ();
1666 in C, there we don't know the argument types at all. */
1667 if (!node->definition
1668 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1669 return;
1670
1671 /* Call this before creating clone_info, as it might ggc_collect. */
1672 if (node->definition && node->has_gimple_body_p ())
1673 node->get_body ();
1674
1675 do
1676 {
1677 /* Start with parsing the "omp declare simd" attribute(s). */
1678 bool inbranch_clause_specified;
1679 struct cgraph_simd_clone *clone_info
1680 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1681 &inbranch_clause_specified);
1682 if (clone_info == NULL)
1683 continue;
1684
1685 int orig_simdlen = clone_info->simdlen;
1686 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1687 /* The target can return 0 (no simd clones should be created),
1688 1 (just one ISA of simd clones should be created) or higher
1689 count of ISA variants. In that case, clone_info is initialized
1690 for the first ISA variant. */
1691 int count
1692 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1693 base_type, 0);
1694 if (count == 0)
1695 continue;
1696
1697 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1698 also create one inbranch and one !inbranch clone of it. */
1699 for (int i = 0; i < count * 2; i++)
1700 {
1701 struct cgraph_simd_clone *clone = clone_info;
1702 if (inbranch_clause_specified && (i & 1) != 0)
1703 continue;
1704
1705 if (i != 0)
1706 {
1707 clone = simd_clone_struct_alloc (clone_info->nargs
1708 + ((i & 1) != 0));
1709 simd_clone_struct_copy (clone, clone_info);
1710 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1711 and simd_clone_adjust_argument_types did to the first
1712 clone's info. */
1713 clone->nargs -= clone_info->inbranch;
1714 clone->simdlen = orig_simdlen;
1715 /* And call the target hook again to get the right ISA. */
1716 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1717 base_type,
1718 i / 2);
1719 if ((i & 1) != 0)
1720 clone->inbranch = 1;
1721 }
1722
1723 /* simd_clone_mangle might fail if such a clone has been created
1724 already. */
1725 tree id = simd_clone_mangle (node, clone);
1726 if (id == NULL_TREE)
1727 {
1728 if (i == 0)
1729 clone->nargs += clone->inbranch;
1730 continue;
1731 }
1732
1733 /* Only when we are sure we want to create the clone actually
1734 clone the function (or definitions) or create another
1735 extern FUNCTION_DECL (for prototypes without definitions). */
1736 struct cgraph_node *n = simd_clone_create (node);
1737 if (n == NULL)
1738 {
1739 if (i == 0)
1740 clone->nargs += clone->inbranch;
1741 continue;
1742 }
1743
1744 n->simdclone = clone;
1745 clone->origin = node;
1746 clone->next_clone = NULL;
1747 if (node->simd_clones == NULL)
1748 {
1749 clone->prev_clone = n;
1750 node->simd_clones = n;
1751 }
1752 else
1753 {
1754 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1755 clone->prev_clone->simdclone->next_clone = n;
1756 node->simd_clones->simdclone->prev_clone = n;
1757 }
1758 symtab->change_decl_assembler_name (n->decl, id);
1759 /* And finally adjust the return type, parameters and for
1760 definitions also function body. */
1761 if (node->definition)
1762 simd_clone_adjust (n);
1763 else
1764 {
1765 TREE_TYPE (n->decl)
1766 = build_distinct_type_copy (TREE_TYPE (n->decl));
1767 targetm.simd_clone.adjust (n);
1768 simd_clone_adjust_return_type (n);
1769 simd_clone_adjust_argument_types (n);
1770 }
1771 }
1772 }
1773 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1774 }
1775
1776 /* Entry point for IPA simd clone creation pass. */
1777
1778 static unsigned int
ipa_omp_simd_clone(void)1779 ipa_omp_simd_clone (void)
1780 {
1781 struct cgraph_node *node;
1782 FOR_EACH_FUNCTION (node)
1783 expand_simd_clones (node);
1784 return 0;
1785 }
1786
1787 namespace {
1788
1789 const pass_data pass_data_omp_simd_clone =
1790 {
1791 SIMPLE_IPA_PASS, /* type */
1792 "simdclone", /* name */
1793 OPTGROUP_OMP, /* optinfo_flags */
1794 TV_NONE, /* tv_id */
1795 ( PROP_ssa | PROP_cfg ), /* properties_required */
1796 0, /* properties_provided */
1797 0, /* properties_destroyed */
1798 0, /* todo_flags_start */
1799 0, /* todo_flags_finish */
1800 };
1801
1802 class pass_omp_simd_clone : public simple_ipa_opt_pass
1803 {
1804 public:
pass_omp_simd_clone(gcc::context * ctxt)1805 pass_omp_simd_clone(gcc::context *ctxt)
1806 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1807 {}
1808
1809 /* opt_pass methods: */
1810 virtual bool gate (function *);
execute(function *)1811 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1812 };
1813
1814 bool
gate(function *)1815 pass_omp_simd_clone::gate (function *)
1816 {
1817 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1818 }
1819
1820 } // anon namespace
1821
1822 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1823 make_pass_omp_simd_clone (gcc::context *ctxt)
1824 {
1825 return new pass_omp_simd_clone (ctxt);
1826 }
1827