1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2021 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Fill an empty vector ARGS with parameter types of function FNDECL. This
90 uses TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static void
simd_clone_vector_of_formal_parm_types(vec<tree> * args,tree fndecl)94 simd_clone_vector_of_formal_parm_types (vec<tree> *args, tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 {
98 push_function_arg_types (args, TREE_TYPE (fndecl));
99 return;
100 }
101 push_function_arg_decls (args, fndecl);
102 unsigned int i;
103 tree arg;
104 FOR_EACH_VEC_ELT (*args, i, arg)
105 (*args)[i] = TREE_TYPE ((*args)[i]);
106 }
107
108 /* Given a simd function in NODE, extract the simd specific
109 information from the OMP clauses passed in CLAUSES, and return
110 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
111 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
112 otherwise set to FALSE. */
113
114 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)115 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
116 bool *inbranch_specified)
117 {
118 auto_vec<tree> args;
119 simd_clone_vector_of_formal_parm_types (&args, node->decl);
120 tree t;
121 int n;
122 *inbranch_specified = false;
123
124 n = args.length ();
125 if (n > 0 && args.last () == void_type_node)
126 n--;
127
128 /* Allocate one more than needed just in case this is an in-branch
129 clone which will require a mask argument. */
130 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
131 clone_info->nargs = n;
132
133 if (!clauses)
134 goto out;
135
136 clauses = TREE_VALUE (clauses);
137 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
138 goto out;
139
140 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
141 {
142 switch (OMP_CLAUSE_CODE (t))
143 {
144 case OMP_CLAUSE_INBRANCH:
145 clone_info->inbranch = 1;
146 *inbranch_specified = true;
147 break;
148 case OMP_CLAUSE_NOTINBRANCH:
149 clone_info->inbranch = 0;
150 *inbranch_specified = true;
151 break;
152 case OMP_CLAUSE_SIMDLEN:
153 clone_info->simdlen
154 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
155 break;
156 case OMP_CLAUSE_LINEAR:
157 {
158 tree decl = OMP_CLAUSE_DECL (t);
159 tree step = OMP_CLAUSE_LINEAR_STEP (t);
160 int argno = TREE_INT_CST_LOW (decl);
161 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
162 {
163 enum cgraph_simd_clone_arg_type arg_type;
164 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
165 switch (OMP_CLAUSE_LINEAR_KIND (t))
166 {
167 case OMP_CLAUSE_LINEAR_REF:
168 arg_type
169 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
170 break;
171 case OMP_CLAUSE_LINEAR_UVAL:
172 arg_type
173 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
174 break;
175 case OMP_CLAUSE_LINEAR_VAL:
176 case OMP_CLAUSE_LINEAR_DEFAULT:
177 arg_type
178 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
179 break;
180 default:
181 gcc_unreachable ();
182 }
183 else
184 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
185 clone_info->args[argno].arg_type = arg_type;
186 clone_info->args[argno].linear_step = tree_to_shwi (step);
187 gcc_assert (clone_info->args[argno].linear_step >= 0
188 && clone_info->args[argno].linear_step < n);
189 }
190 else
191 {
192 if (POINTER_TYPE_P (args[argno]))
193 step = fold_convert (ssizetype, step);
194 if (!tree_fits_shwi_p (step))
195 {
196 warning_at (OMP_CLAUSE_LOCATION (t), 0,
197 "ignoring large linear step");
198 return NULL;
199 }
200 else if (integer_zerop (step))
201 {
202 warning_at (OMP_CLAUSE_LOCATION (t), 0,
203 "ignoring zero linear step");
204 return NULL;
205 }
206 else
207 {
208 enum cgraph_simd_clone_arg_type arg_type;
209 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
210 switch (OMP_CLAUSE_LINEAR_KIND (t))
211 {
212 case OMP_CLAUSE_LINEAR_REF:
213 arg_type
214 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
215 break;
216 case OMP_CLAUSE_LINEAR_UVAL:
217 arg_type
218 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
219 break;
220 case OMP_CLAUSE_LINEAR_VAL:
221 case OMP_CLAUSE_LINEAR_DEFAULT:
222 arg_type
223 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
224 break;
225 default:
226 gcc_unreachable ();
227 }
228 else
229 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
230 clone_info->args[argno].arg_type = arg_type;
231 clone_info->args[argno].linear_step = tree_to_shwi (step);
232 }
233 }
234 break;
235 }
236 case OMP_CLAUSE_UNIFORM:
237 {
238 tree decl = OMP_CLAUSE_DECL (t);
239 int argno = tree_to_uhwi (decl);
240 clone_info->args[argno].arg_type
241 = SIMD_CLONE_ARG_TYPE_UNIFORM;
242 break;
243 }
244 case OMP_CLAUSE_ALIGNED:
245 {
246 /* Ignore aligned (x) for declare simd, for the ABI we really
247 need an alignment specified. */
248 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
249 break;
250 tree decl = OMP_CLAUSE_DECL (t);
251 int argno = tree_to_uhwi (decl);
252 clone_info->args[argno].alignment
253 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
254 break;
255 }
256 default:
257 break;
258 }
259 }
260
261 out:
262 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
263 {
264 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
265 "ignoring %<#pragma omp declare simd%> on function "
266 "with %<_Atomic%> qualified return type");
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 return clone_info;
282 }
283
284 /* Given a SIMD clone in NODE, calculate the characteristic data
285 type and return the coresponding type. The characteristic data
286 type is computed as described in the Intel Vector ABI. */
287
288 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)289 simd_clone_compute_base_data_type (struct cgraph_node *node,
290 struct cgraph_simd_clone *clone_info)
291 {
292 tree type = integer_type_node;
293 tree fndecl = node->decl;
294
295 /* a) For non-void function, the characteristic data type is the
296 return type. */
297 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
298 type = TREE_TYPE (TREE_TYPE (fndecl));
299
300 /* b) If the function has any non-uniform, non-linear parameters,
301 then the characteristic data type is the type of the first
302 such parameter. */
303 else
304 {
305 auto_vec<tree> map;
306 simd_clone_vector_of_formal_parm_types (&map, fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 }
314
315 /* c) If the characteristic data type determined by a) or b) above
316 is struct, union, or class type which is pass-by-value (except
317 for the type that maps to the built-in complex data type), the
318 characteristic data type is int. */
319 if (RECORD_OR_UNION_TYPE_P (type)
320 && !aggregate_value_p (type, NULL)
321 && TREE_CODE (type) != COMPLEX_TYPE)
322 return integer_type_node;
323
324 /* d) If none of the above three classes is applicable, the
325 characteristic data type is int. */
326
327 return type;
328
329 /* e) For Intel Xeon Phi native and offload compilation, if the
330 resulting characteristic data type is 8-bit or 16-bit integer
331 data type, the characteristic data type is int. */
332 /* Well, we don't handle Xeon Phi yet. */
333 }
334
335 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)336 simd_clone_mangle (struct cgraph_node *node,
337 struct cgraph_simd_clone *clone_info)
338 {
339 char vecsize_mangle = clone_info->vecsize_mangle;
340 char mask = clone_info->inbranch ? 'M' : 'N';
341 poly_uint64 simdlen = clone_info->simdlen;
342 unsigned int n;
343 pretty_printer pp;
344
345 gcc_assert (vecsize_mangle && maybe_ne (simdlen, 0U));
346
347 pp_string (&pp, "_ZGV");
348 pp_character (&pp, vecsize_mangle);
349 pp_character (&pp, mask);
350 /* For now, simdlen is always constant, while variable simdlen pp 'n'. */
351 unsigned int len = simdlen.to_constant ();
352 pp_decimal_int (&pp, (len));
353
354 for (n = 0; n < clone_info->nargs; ++n)
355 {
356 struct cgraph_simd_clone_arg arg = clone_info->args[n];
357
358 switch (arg.arg_type)
359 {
360 case SIMD_CLONE_ARG_TYPE_UNIFORM:
361 pp_character (&pp, 'u');
362 break;
363 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
364 pp_character (&pp, 'l');
365 goto mangle_linear;
366 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
367 pp_character (&pp, 'R');
368 goto mangle_linear;
369 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
370 pp_character (&pp, 'L');
371 goto mangle_linear;
372 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
373 pp_character (&pp, 'U');
374 goto mangle_linear;
375 mangle_linear:
376 gcc_assert (arg.linear_step != 0);
377 if (arg.linear_step > 1)
378 pp_unsigned_wide_integer (&pp, arg.linear_step);
379 else if (arg.linear_step < 0)
380 {
381 pp_character (&pp, 'n');
382 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
383 arg.linear_step));
384 }
385 break;
386 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
387 pp_string (&pp, "ls");
388 pp_unsigned_wide_integer (&pp, arg.linear_step);
389 break;
390 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
391 pp_string (&pp, "Rs");
392 pp_unsigned_wide_integer (&pp, arg.linear_step);
393 break;
394 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
395 pp_string (&pp, "Ls");
396 pp_unsigned_wide_integer (&pp, arg.linear_step);
397 break;
398 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
399 pp_string (&pp, "Us");
400 pp_unsigned_wide_integer (&pp, arg.linear_step);
401 break;
402 default:
403 pp_character (&pp, 'v');
404 }
405 if (arg.alignment)
406 {
407 pp_character (&pp, 'a');
408 pp_decimal_int (&pp, arg.alignment);
409 }
410 }
411
412 pp_underscore (&pp);
413 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
414 if (*str == '*')
415 ++str;
416 pp_string (&pp, str);
417 str = pp_formatted_text (&pp);
418
419 /* If there already is a SIMD clone with the same mangled name, don't
420 add another one. This can happen e.g. for
421 #pragma omp declare simd
422 #pragma omp declare simd simdlen(8)
423 int foo (int, int);
424 if the simdlen is assumed to be 8 for the first one, etc. */
425 for (struct cgraph_node *clone = node->simd_clones; clone;
426 clone = clone->simdclone->next_clone)
427 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
428 return NULL_TREE;
429
430 return get_identifier (str);
431 }
432
433 /* Create a simd clone of OLD_NODE and return it. */
434
435 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)436 simd_clone_create (struct cgraph_node *old_node)
437 {
438 struct cgraph_node *new_node;
439 if (old_node->definition)
440 {
441 if (!old_node->has_gimple_body_p ())
442 return NULL;
443 old_node->get_body ();
444 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
445 NULL, NULL,
446 "simdclone");
447 }
448 else
449 {
450 tree old_decl = old_node->decl;
451 tree new_decl = copy_node (old_node->decl);
452 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl,
453 "simdclone");
454 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
455 SET_DECL_RTL (new_decl, NULL);
456 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
457 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
458 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
459 if (old_node->in_other_partition)
460 new_node->in_other_partition = 1;
461 }
462 if (new_node == NULL)
463 return new_node;
464
465 set_decl_built_in_function (new_node->decl, NOT_BUILT_IN, 0);
466 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
467 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
468 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
469 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
470 DECL_VISIBILITY_SPECIFIED (new_node->decl)
471 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
472 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
473 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
474 if (DECL_ONE_ONLY (old_node->decl))
475 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
476
477 /* The method cgraph_version_clone_with_body () will force the new
478 symbol local. Undo this, and inherit external visibility from
479 the old node. */
480 new_node->local = old_node->local;
481 new_node->externally_visible = old_node->externally_visible;
482 new_node->calls_declare_variant_alt = old_node->calls_declare_variant_alt;
483
484 return new_node;
485 }
486
487 /* Adjust the return type of the given function to its appropriate
488 vector counterpart. Returns a simd array to be used throughout the
489 function as a return value. */
490
491 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)492 simd_clone_adjust_return_type (struct cgraph_node *node)
493 {
494 tree fndecl = node->decl;
495 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
496 poly_uint64 veclen;
497 tree t;
498
499 /* Adjust the function return type. */
500 if (orig_rettype == void_type_node)
501 return NULL_TREE;
502 t = TREE_TYPE (TREE_TYPE (fndecl));
503 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
504 veclen = node->simdclone->vecsize_int;
505 else
506 veclen = node->simdclone->vecsize_float;
507 veclen = exact_div (veclen, GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t)));
508 if (multiple_p (veclen, node->simdclone->simdlen))
509 veclen = node->simdclone->simdlen;
510 if (POINTER_TYPE_P (t))
511 t = pointer_sized_int_node;
512 if (known_eq (veclen, node->simdclone->simdlen))
513 t = build_vector_type (t, node->simdclone->simdlen);
514 else
515 {
516 t = build_vector_type (t, veclen);
517 t = build_array_type_nelts (t, exact_div (node->simdclone->simdlen,
518 veclen));
519 }
520 TREE_TYPE (TREE_TYPE (fndecl)) = t;
521 if (!node->definition)
522 return NULL_TREE;
523
524 t = DECL_RESULT (fndecl);
525 /* Adjust the DECL_RESULT. */
526 gcc_assert (TREE_TYPE (t) != void_type_node);
527 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
528 relayout_decl (t);
529
530 tree atype = build_array_type_nelts (orig_rettype,
531 node->simdclone->simdlen);
532 if (maybe_ne (veclen, node->simdclone->simdlen))
533 return build1 (VIEW_CONVERT_EXPR, atype, t);
534
535 /* Set up a SIMD array to use as the return value. */
536 tree retval = create_tmp_var_raw (atype, "retval");
537 gimple_add_tmp_var (retval);
538 return retval;
539 }
540
541 /* Each vector argument has a corresponding array to be used locally
542 as part of the eventual loop. Create such temporary array and
543 return it.
544
545 PREFIX is the prefix to be used for the temporary.
546
547 TYPE is the inner element type.
548
549 SIMDLEN is the number of elements. */
550
551 static tree
create_tmp_simd_array(const char * prefix,tree type,poly_uint64 simdlen)552 create_tmp_simd_array (const char *prefix, tree type, poly_uint64 simdlen)
553 {
554 tree atype = build_array_type_nelts (type, simdlen);
555 tree avar = create_tmp_var_raw (atype, prefix);
556 gimple_add_tmp_var (avar);
557 return avar;
558 }
559
560 /* Modify the function argument types to their corresponding vector
561 counterparts if appropriate. Also, create one array for each simd
562 argument to be used locally when using the function arguments as
563 part of the loop.
564
565 NODE is the function whose arguments are to be adjusted.
566
567 If NODE does not represent function definition, returns NULL. Otherwise
568 returns an adjustment class that will be filled describing how the argument
569 declarations will be remapped. New arguments which are not to be remapped
570 are marked with USER_FLAG. */
571
572 static ipa_param_body_adjustments *
simd_clone_adjust_argument_types(struct cgraph_node * node)573 simd_clone_adjust_argument_types (struct cgraph_node *node)
574 {
575 auto_vec<tree> args;
576
577 if (node->definition)
578 push_function_arg_decls (&args, node->decl);
579 else
580 simd_clone_vector_of_formal_parm_types (&args, node->decl);
581 struct cgraph_simd_clone *sc = node->simdclone;
582 vec<ipa_adjusted_param, va_gc> *new_params = NULL;
583 vec_safe_reserve (new_params, sc->nargs);
584 unsigned i, j, k;
585 poly_uint64 veclen;
586
587 for (i = 0; i < sc->nargs; ++i)
588 {
589 ipa_adjusted_param adj;
590 memset (&adj, 0, sizeof (adj));
591 tree parm = args[i];
592 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
593 adj.base_index = i;
594 adj.prev_clone_index = i;
595
596 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
597 sc->args[i].orig_type = parm_type;
598
599 switch (sc->args[i].arg_type)
600 {
601 default:
602 /* No adjustment necessary for scalar arguments. */
603 adj.op = IPA_PARAM_OP_COPY;
604 break;
605 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
606 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
607 if (node->definition)
608 sc->args[i].simd_array
609 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
610 TREE_TYPE (parm_type),
611 sc->simdlen);
612 adj.op = IPA_PARAM_OP_COPY;
613 break;
614 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
615 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
616 case SIMD_CLONE_ARG_TYPE_VECTOR:
617 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
618 veclen = sc->vecsize_int;
619 else
620 veclen = sc->vecsize_float;
621 veclen = exact_div (veclen,
622 GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type)));
623 if (multiple_p (veclen, sc->simdlen))
624 veclen = sc->simdlen;
625 adj.op = IPA_PARAM_OP_NEW;
626 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
627 if (POINTER_TYPE_P (parm_type))
628 adj.type = build_vector_type (pointer_sized_int_node, veclen);
629 else
630 adj.type = build_vector_type (parm_type, veclen);
631 sc->args[i].vector_type = adj.type;
632 k = vector_unroll_factor (sc->simdlen, veclen);
633 for (j = 1; j < k; j++)
634 {
635 vec_safe_push (new_params, adj);
636 if (j == 1)
637 {
638 memset (&adj, 0, sizeof (adj));
639 adj.op = IPA_PARAM_OP_NEW;
640 adj.user_flag = 1;
641 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
642 adj.base_index = i;
643 adj.prev_clone_index = i;
644 adj.type = sc->args[i].vector_type;
645 }
646 }
647
648 if (node->definition)
649 sc->args[i].simd_array
650 = create_tmp_simd_array (DECL_NAME (parm)
651 ? IDENTIFIER_POINTER (DECL_NAME (parm))
652 : NULL, parm_type, sc->simdlen);
653 }
654 vec_safe_push (new_params, adj);
655 }
656
657 if (sc->inbranch)
658 {
659 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
660 ipa_adjusted_param adj;
661 memset (&adj, 0, sizeof (adj));
662 adj.op = IPA_PARAM_OP_NEW;
663 adj.user_flag = 1;
664 adj.param_prefix_index = IPA_PARAM_PREFIX_MASK;
665
666 adj.base_index = i;
667 adj.prev_clone_index = i;
668 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
669 veclen = sc->vecsize_int;
670 else
671 veclen = sc->vecsize_float;
672 veclen = exact_div (veclen,
673 GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type)));
674 if (multiple_p (veclen, sc->simdlen))
675 veclen = sc->simdlen;
676 if (sc->mask_mode != VOIDmode)
677 adj.type
678 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
679 else if (POINTER_TYPE_P (base_type))
680 adj.type = build_vector_type (pointer_sized_int_node, veclen);
681 else
682 adj.type = build_vector_type (base_type, veclen);
683 vec_safe_push (new_params, adj);
684
685 k = vector_unroll_factor (sc->simdlen, veclen);
686 for (j = 1; j < k; j++)
687 vec_safe_push (new_params, adj);
688
689 /* We have previously allocated one extra entry for the mask. Use
690 it and fill it. */
691 sc->nargs++;
692 if (sc->mask_mode != VOIDmode)
693 base_type = boolean_type_node;
694 if (node->definition)
695 {
696 sc->args[i].orig_arg
697 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
698 if (sc->mask_mode == VOIDmode)
699 sc->args[i].simd_array
700 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
701 else if (k > 1)
702 sc->args[i].simd_array
703 = create_tmp_simd_array ("mask", adj.type, k);
704 else
705 sc->args[i].simd_array = NULL_TREE;
706 }
707 sc->args[i].orig_type = base_type;
708 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
709 }
710
711 if (node->definition)
712 {
713 ipa_param_body_adjustments *adjustments
714 = new ipa_param_body_adjustments (new_params, node->decl);
715
716 adjustments->modify_formal_parameters ();
717 return adjustments;
718 }
719 else
720 {
721 tree new_arg_types = NULL_TREE, new_reversed;
722 bool last_parm_void = false;
723 if (args.length () > 0 && args.last () == void_type_node)
724 last_parm_void = true;
725
726 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
727 j = vec_safe_length (new_params);
728 for (i = 0; i < j; i++)
729 {
730 struct ipa_adjusted_param *adj = &(*new_params)[i];
731 tree ptype;
732 if (adj->op == IPA_PARAM_OP_COPY)
733 ptype = args[adj->base_index];
734 else
735 ptype = adj->type;
736 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
737 }
738 new_reversed = nreverse (new_arg_types);
739 if (last_parm_void)
740 {
741 if (new_reversed)
742 TREE_CHAIN (new_arg_types) = void_list_node;
743 else
744 new_reversed = void_list_node;
745 }
746 TYPE_ARG_TYPES (TREE_TYPE (node->decl)) = new_reversed;
747 return NULL;
748 }
749 }
750
751 /* Initialize and copy the function arguments in NODE to their
752 corresponding local simd arrays. Returns a fresh gimple_seq with
753 the instruction sequence generated. */
754
755 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_param_body_adjustments * adjustments)756 simd_clone_init_simd_arrays (struct cgraph_node *node,
757 ipa_param_body_adjustments *adjustments)
758 {
759 gimple_seq seq = NULL;
760 unsigned i = 0, j = 0, k;
761
762 for (tree arg = DECL_ARGUMENTS (node->decl);
763 arg;
764 arg = DECL_CHAIN (arg), i++, j++)
765 {
766 if ((*adjustments->m_adj_params)[j].op == IPA_PARAM_OP_COPY
767 || POINTER_TYPE_P (TREE_TYPE (arg)))
768 continue;
769
770 node->simdclone->args[i].vector_arg = arg;
771
772 tree array = node->simdclone->args[i].simd_array;
773 if (node->simdclone->mask_mode != VOIDmode
774 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
775 {
776 if (array == NULL_TREE)
777 continue;
778 unsigned int l
779 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
780 for (k = 0; k <= l; k++)
781 {
782 if (k)
783 {
784 arg = DECL_CHAIN (arg);
785 j++;
786 }
787 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
788 array, size_int (k), NULL, NULL);
789 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
790 gimplify_and_add (t, &seq);
791 }
792 continue;
793 }
794 if (known_eq (simd_clone_subparts (TREE_TYPE (arg)),
795 node->simdclone->simdlen))
796 {
797 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
798 tree ptr = build_fold_addr_expr (array);
799 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
800 build_int_cst (ptype, 0));
801 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
802 gimplify_and_add (t, &seq);
803 }
804 else
805 {
806 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
807 unsigned int times = vector_unroll_factor (node->simdclone->simdlen,
808 simdlen);
809 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
810 for (k = 0; k < times; k++)
811 {
812 tree ptr = build_fold_addr_expr (array);
813 int elemsize;
814 if (k)
815 {
816 arg = DECL_CHAIN (arg);
817 j++;
818 }
819 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
820 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
821 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
822 build_int_cst (ptype, k * elemsize * simdlen));
823 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
824 gimplify_and_add (t, &seq);
825 }
826 }
827 }
828 return seq;
829 }
830
831 /* Callback info for ipa_simd_modify_stmt_ops below. */
832
833 struct modify_stmt_info {
834 ipa_param_body_adjustments *adjustments;
835 gimple *stmt;
836 gimple *after_stmt;
837 /* True if the parent statement was modified by
838 ipa_simd_modify_stmt_ops. */
839 bool modified;
840 };
841
842 /* Callback for walk_gimple_op.
843
844 Adjust operands from a given statement as specified in the
845 adjustments vector in the callback data. */
846
847 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)848 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
849 {
850 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
851 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
852 tree *orig_tp = tp;
853 if (TREE_CODE (*tp) == ADDR_EXPR)
854 tp = &TREE_OPERAND (*tp, 0);
855
856 if (TREE_CODE (*tp) == BIT_FIELD_REF
857 || TREE_CODE (*tp) == IMAGPART_EXPR
858 || TREE_CODE (*tp) == REALPART_EXPR)
859 tp = &TREE_OPERAND (*tp, 0);
860
861 tree repl = NULL_TREE;
862 ipa_param_body_replacement *pbr = NULL;
863
864 if (TREE_CODE (*tp) == PARM_DECL)
865 {
866 pbr = info->adjustments->get_expr_replacement (*tp, true);
867 if (pbr)
868 repl = pbr->repl;
869 }
870 else if (TYPE_P (*tp))
871 *walk_subtrees = 0;
872
873 if (repl)
874 repl = unshare_expr (repl);
875 else
876 {
877 if (tp != orig_tp)
878 {
879 *walk_subtrees = 0;
880 bool modified = info->modified;
881 info->modified = false;
882 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
883 if (!info->modified)
884 {
885 info->modified = modified;
886 return NULL_TREE;
887 }
888 info->modified = modified;
889 repl = *tp;
890 }
891 else
892 return NULL_TREE;
893 }
894
895 if (tp != orig_tp)
896 {
897 if (gimple_code (info->stmt) == GIMPLE_PHI
898 && pbr
899 && TREE_CODE (*orig_tp) == ADDR_EXPR
900 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
901 && pbr->dummy)
902 {
903 gcc_assert (TREE_CODE (pbr->dummy) == SSA_NAME);
904 *orig_tp = pbr->dummy;
905 info->modified = true;
906 return NULL_TREE;
907 }
908
909 repl = build_fold_addr_expr (repl);
910 gimple *stmt;
911 if (is_gimple_debug (info->stmt))
912 {
913 tree vexpr = make_node (DEBUG_EXPR_DECL);
914 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
915 DECL_ARTIFICIAL (vexpr) = 1;
916 TREE_TYPE (vexpr) = TREE_TYPE (repl);
917 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
918 repl = vexpr;
919 }
920 else
921 {
922 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
923 repl = gimple_assign_lhs (stmt);
924 }
925 gimple_stmt_iterator gsi;
926 if (gimple_code (info->stmt) == GIMPLE_PHI)
927 {
928 if (info->after_stmt)
929 gsi = gsi_for_stmt (info->after_stmt);
930 else
931 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
932 /* Cache SSA_NAME for next time. */
933 if (pbr
934 && TREE_CODE (*orig_tp) == ADDR_EXPR
935 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
936 {
937 gcc_assert (!pbr->dummy);
938 pbr->dummy = repl;
939 }
940 }
941 else
942 gsi = gsi_for_stmt (info->stmt);
943 if (info->after_stmt)
944 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
945 else
946 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
947 if (gimple_code (info->stmt) == GIMPLE_PHI)
948 info->after_stmt = stmt;
949 *orig_tp = repl;
950 }
951 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
952 {
953 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
954 *tp = vce;
955 }
956 else
957 *tp = repl;
958
959 info->modified = true;
960 return NULL_TREE;
961 }
962
963 /* Traverse the function body and perform all modifications as
964 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
965 modified such that the replacement/reduction value will now be an
966 offset into the corresponding simd_array.
967
968 This function will replace all function argument uses with their
969 corresponding simd array elements, and ajust the return values
970 accordingly. */
971
972 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_param_body_adjustments * adjustments,tree retval_array,tree iter)973 ipa_simd_modify_function_body (struct cgraph_node *node,
974 ipa_param_body_adjustments *adjustments,
975 tree retval_array, tree iter)
976 {
977 basic_block bb;
978 unsigned int i, j;
979
980
981 /* Register replacements for every function argument use to an offset into
982 the corresponding simd_array. */
983 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
984 {
985 if (!node->simdclone->args[i].vector_arg
986 || (*adjustments->m_adj_params)[j].user_flag)
987 continue;
988
989 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
990 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
991 tree r = build4 (ARRAY_REF, basetype, node->simdclone->args[i].simd_array,
992 iter, NULL_TREE, NULL_TREE);
993 adjustments->register_replacement (&(*adjustments->m_adj_params)[j], r);
994
995 if (multiple_p (node->simdclone->simdlen, simd_clone_subparts (vectype)))
996 j += vector_unroll_factor (node->simdclone->simdlen,
997 simd_clone_subparts (vectype)) - 1;
998 }
999
1000 tree name;
1001 FOR_EACH_SSA_NAME (i, name, cfun)
1002 {
1003 tree base_var;
1004 if (SSA_NAME_VAR (name)
1005 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
1006 && (base_var
1007 = adjustments->get_replacement_ssa_base (SSA_NAME_VAR (name))))
1008 {
1009 if (SSA_NAME_IS_DEFAULT_DEF (name))
1010 {
1011 tree old_decl = SSA_NAME_VAR (name);
1012 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1013 gimple_stmt_iterator gsi = gsi_after_labels (bb);
1014 tree repl = adjustments->lookup_replacement (old_decl, 0);
1015 gcc_checking_assert (repl);
1016 repl = unshare_expr (repl);
1017 set_ssa_default_def (cfun, old_decl, NULL_TREE);
1018 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1019 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
1020 gimple *stmt = gimple_build_assign (name, repl);
1021 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1022 }
1023 else
1024 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1025 }
1026 }
1027
1028 struct modify_stmt_info info;
1029 info.adjustments = adjustments;
1030
1031 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1032 {
1033 gimple_stmt_iterator gsi;
1034
1035 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1036 {
1037 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1038 int i, n = gimple_phi_num_args (phi);
1039 info.stmt = phi;
1040 info.after_stmt = NULL;
1041 struct walk_stmt_info wi;
1042 memset (&wi, 0, sizeof (wi));
1043 info.modified = false;
1044 wi.info = &info;
1045 for (i = 0; i < n; ++i)
1046 {
1047 int walk_subtrees = 1;
1048 tree arg = gimple_phi_arg_def (phi, i);
1049 tree op = arg;
1050 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1051 if (op != arg)
1052 {
1053 SET_PHI_ARG_DEF (phi, i, op);
1054 gcc_assert (TREE_CODE (op) == SSA_NAME);
1055 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1056 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1057 }
1058 }
1059 }
1060
1061 gsi = gsi_start_bb (bb);
1062 while (!gsi_end_p (gsi))
1063 {
1064 gimple *stmt = gsi_stmt (gsi);
1065 info.stmt = stmt;
1066 info.after_stmt = NULL;
1067 struct walk_stmt_info wi;
1068
1069 memset (&wi, 0, sizeof (wi));
1070 info.modified = false;
1071 wi.info = &info;
1072 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1073
1074 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1075 {
1076 tree retval = gimple_return_retval (return_stmt);
1077 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1078 e->flags |= EDGE_FALLTHRU;
1079 if (!retval)
1080 {
1081 gsi_remove (&gsi, true);
1082 continue;
1083 }
1084
1085 /* Replace `return foo' with `retval_array[iter] = foo'. */
1086 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1087 retval_array, iter, NULL, NULL);
1088 stmt = gimple_build_assign (ref, retval);
1089 gsi_replace (&gsi, stmt, true);
1090 info.modified = true;
1091 }
1092
1093 if (info.modified)
1094 {
1095 update_stmt (stmt);
1096 /* If the above changed the var of a debug bind into something
1097 different, remove the debug stmt. We could also for all the
1098 replaced parameters add VAR_DECLs for debug info purposes,
1099 add debug stmts for those to be the simd array accesses and
1100 replace debug stmt var operand with that var. Debugging of
1101 vectorized loops doesn't work too well, so don't bother for
1102 now. */
1103 if ((gimple_debug_bind_p (stmt)
1104 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1105 || (gimple_debug_source_bind_p (stmt)
1106 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1107 {
1108 gsi_remove (&gsi, true);
1109 continue;
1110 }
1111 if (maybe_clean_eh_stmt (stmt))
1112 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1113 }
1114 gsi_next (&gsi);
1115 }
1116 }
1117 }
1118
1119 /* Helper function of simd_clone_adjust, return linear step addend
1120 of Ith argument. */
1121
1122 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1123 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1124 tree addtype, basic_block entry_bb)
1125 {
1126 tree ptype = NULL_TREE;
1127 switch (node->simdclone->args[i].arg_type)
1128 {
1129 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1130 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1131 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1132 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1133 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1134 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1135 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1136 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1137 break;
1138 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1139 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1140 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1141 break;
1142 default:
1143 gcc_unreachable ();
1144 }
1145
1146 unsigned int idx = node->simdclone->args[i].linear_step;
1147 tree arg = node->simdclone->args[idx].orig_arg;
1148 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1149 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1150 gimple *g;
1151 tree ret;
1152 if (is_gimple_reg (arg))
1153 ret = get_or_create_ssa_default_def (cfun, arg);
1154 else
1155 {
1156 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1157 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1158 ret = gimple_assign_lhs (g);
1159 }
1160 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1161 {
1162 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1163 build_simple_mem_ref (ret));
1164 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1165 ret = gimple_assign_lhs (g);
1166 }
1167 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1168 {
1169 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1170 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1171 ret = gimple_assign_lhs (g);
1172 }
1173 if (POINTER_TYPE_P (ptype))
1174 {
1175 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1176 if (size && TREE_CODE (size) == INTEGER_CST)
1177 {
1178 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1179 ret, fold_convert (addtype, size));
1180 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1181 ret = gimple_assign_lhs (g);
1182 }
1183 }
1184 return ret;
1185 }
1186
1187 /* Adjust the argument types in NODE to their appropriate vector
1188 counterparts. */
1189
1190 static void
simd_clone_adjust(struct cgraph_node * node)1191 simd_clone_adjust (struct cgraph_node *node)
1192 {
1193 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1194
1195 TREE_TYPE (node->decl) = build_distinct_type_copy (TREE_TYPE (node->decl));
1196 targetm.simd_clone.adjust (node);
1197
1198 tree retval = simd_clone_adjust_return_type (node);
1199 ipa_param_body_adjustments *adjustments
1200 = simd_clone_adjust_argument_types (node);
1201 gcc_assert (adjustments);
1202
1203 push_gimplify_context ();
1204
1205 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1206
1207 /* Adjust all uses of vector arguments accordingly. Adjust all
1208 return values accordingly. */
1209 tree iter = create_tmp_var (unsigned_type_node, "iter");
1210 tree iter1 = make_ssa_name (iter);
1211 tree iter2 = NULL_TREE;
1212 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1213 delete adjustments;
1214
1215 /* Initialize the iteration variable. */
1216 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1217 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1218 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1219 /* Insert the SIMD array and iv initialization at function
1220 entry. */
1221 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1222
1223 pop_gimplify_context (NULL);
1224
1225 gimple *g;
1226 basic_block incr_bb = NULL;
1227 class loop *loop = NULL;
1228
1229 /* Create a new BB right before the original exit BB, to hold the
1230 iteration increment and the condition/branch. */
1231 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1232 {
1233 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1234 incr_bb = create_empty_bb (orig_exit);
1235 incr_bb->count = profile_count::zero ();
1236 add_bb_to_loop (incr_bb, body_bb->loop_father);
1237 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1238 {
1239 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1240 redirect_edge_succ (e, incr_bb);
1241 incr_bb->count += e->count ();
1242 }
1243 }
1244 else if (node->simdclone->inbranch)
1245 {
1246 incr_bb = create_empty_bb (entry_bb);
1247 incr_bb->count = profile_count::zero ();
1248 add_bb_to_loop (incr_bb, body_bb->loop_father);
1249 }
1250
1251 if (incr_bb)
1252 {
1253 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1254 gsi = gsi_last_bb (incr_bb);
1255 iter2 = make_ssa_name (iter);
1256 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1257 build_int_cst (unsigned_type_node, 1));
1258 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1259
1260 /* Mostly annotate the loop for the vectorizer (the rest is done
1261 below). */
1262 loop = alloc_loop ();
1263 cfun->has_force_vectorize_loops = true;
1264 /* For now, simlen is always constant. */
1265 loop->safelen = node->simdclone->simdlen.to_constant ();
1266 loop->force_vectorize = true;
1267 loop->header = body_bb;
1268 }
1269
1270 /* Branch around the body if the mask applies. */
1271 if (node->simdclone->inbranch)
1272 {
1273 gsi = gsi_last_bb (loop->header);
1274 tree mask_array
1275 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1276 tree mask;
1277 if (node->simdclone->mask_mode != VOIDmode)
1278 {
1279 tree shift_cnt;
1280 if (mask_array == NULL_TREE)
1281 {
1282 tree arg = node->simdclone->args[node->simdclone->nargs
1283 - 1].vector_arg;
1284 mask = get_or_create_ssa_default_def (cfun, arg);
1285 shift_cnt = iter1;
1286 }
1287 else
1288 {
1289 tree maskt = TREE_TYPE (mask_array);
1290 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1291 /* For now, c must be constant here. */
1292 c = exact_div (node->simdclone->simdlen, c + 1).to_constant ();
1293 int s = exact_log2 (c);
1294 gcc_assert (s > 0);
1295 c--;
1296 tree idx = make_ssa_name (TREE_TYPE (iter1));
1297 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1298 build_int_cst (NULL_TREE, s));
1299 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1300 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1301 tree aref = build4 (ARRAY_REF,
1302 TREE_TYPE (TREE_TYPE (mask_array)),
1303 mask_array, idx, NULL, NULL);
1304 g = gimple_build_assign (mask, aref);
1305 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1306 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1307 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1308 build_int_cst (TREE_TYPE (iter1), c));
1309 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1310 }
1311 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1312 RSHIFT_EXPR, mask, shift_cnt);
1313 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1314 mask = gimple_assign_lhs (g);
1315 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1316 BIT_AND_EXPR, mask,
1317 build_int_cst (TREE_TYPE (mask), 1));
1318 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1319 mask = gimple_assign_lhs (g);
1320 }
1321 else
1322 {
1323 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1324 tree aref = build4 (ARRAY_REF,
1325 TREE_TYPE (TREE_TYPE (mask_array)),
1326 mask_array, iter1, NULL, NULL);
1327 g = gimple_build_assign (mask, aref);
1328 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1329 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1330 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1331 {
1332 aref = build1 (VIEW_CONVERT_EXPR,
1333 build_nonstandard_integer_type (bitsize, 0),
1334 mask);
1335 mask = make_ssa_name (TREE_TYPE (aref));
1336 g = gimple_build_assign (mask, aref);
1337 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1338 }
1339 }
1340
1341 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1342 NULL, NULL);
1343 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1344 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1345 e->probability = profile_probability::unlikely ().guessed ();
1346 incr_bb->count += e->count ();
1347 edge fallthru = FALLTHRU_EDGE (loop->header);
1348 fallthru->flags = EDGE_FALSE_VALUE;
1349 fallthru->probability = profile_probability::likely ().guessed ();
1350 }
1351
1352 basic_block latch_bb = NULL;
1353 basic_block new_exit_bb = NULL;
1354
1355 /* Generate the condition. */
1356 if (incr_bb)
1357 {
1358 gsi = gsi_last_bb (incr_bb);
1359 g = gimple_build_cond (LT_EXPR, iter2,
1360 build_int_cst (unsigned_type_node,
1361 node->simdclone->simdlen),
1362 NULL, NULL);
1363 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1364 edge e = split_block (incr_bb, gsi_stmt (gsi));
1365 latch_bb = e->dest;
1366 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1367 loop->latch = latch_bb;
1368
1369 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1370
1371 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1372
1373 /* FIXME: Do we need to distribute probabilities for the conditional? */
1374 new_e->probability = profile_probability::guessed_never ();
1375 /* The successor of incr_bb is already pointing to latch_bb; just
1376 change the flags.
1377 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1378 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1379 }
1380
1381 gphi *phi = create_phi_node (iter1, body_bb);
1382 edge preheader_edge = find_edge (entry_bb, body_bb);
1383 edge latch_edge = NULL;
1384 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1385 UNKNOWN_LOCATION);
1386 if (incr_bb)
1387 {
1388 latch_edge = single_succ_edge (latch_bb);
1389 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1390
1391 /* Generate the new return. */
1392 gsi = gsi_last_bb (new_exit_bb);
1393 if (retval
1394 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1395 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1396 retval = TREE_OPERAND (retval, 0);
1397 else if (retval)
1398 {
1399 retval = build1 (VIEW_CONVERT_EXPR,
1400 TREE_TYPE (TREE_TYPE (node->decl)),
1401 retval);
1402 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1403 false, GSI_CONTINUE_LINKING);
1404 }
1405 g = gimple_build_return (retval);
1406 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1407 }
1408
1409 /* Handle aligned clauses by replacing default defs of the aligned
1410 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1411 lhs. Handle linear by adding PHIs. */
1412 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1413 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1414 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1415 || !is_gimple_reg_type
1416 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1417 {
1418 tree orig_arg = node->simdclone->args[i].orig_arg;
1419 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1420 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1421 else
1422 {
1423 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1424 gimple_add_tmp_var (iter1);
1425 }
1426 gsi = gsi_after_labels (entry_bb);
1427 g = gimple_build_assign (iter1, orig_arg);
1428 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1429 gsi = gsi_after_labels (body_bb);
1430 g = gimple_build_assign (orig_arg, iter1);
1431 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1432 }
1433 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1434 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1435 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1436 == REFERENCE_TYPE
1437 && TREE_ADDRESSABLE
1438 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1439 {
1440 tree orig_arg = node->simdclone->args[i].orig_arg;
1441 tree def = ssa_default_def (cfun, orig_arg);
1442 if (def && !has_zero_uses (def))
1443 {
1444 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1445 gimple_add_tmp_var (iter1);
1446 gsi = gsi_after_labels (entry_bb);
1447 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1448 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1449 gsi = gsi_after_labels (body_bb);
1450 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1451 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1452 }
1453 }
1454 else if (node->simdclone->args[i].alignment
1455 && node->simdclone->args[i].arg_type
1456 == SIMD_CLONE_ARG_TYPE_UNIFORM
1457 && (node->simdclone->args[i].alignment
1458 & (node->simdclone->args[i].alignment - 1)) == 0
1459 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1460 == POINTER_TYPE)
1461 {
1462 unsigned int alignment = node->simdclone->args[i].alignment;
1463 tree orig_arg = node->simdclone->args[i].orig_arg;
1464 tree def = ssa_default_def (cfun, orig_arg);
1465 if (def && !has_zero_uses (def))
1466 {
1467 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1468 gimple_seq seq = NULL;
1469 bool need_cvt = false;
1470 gcall *call
1471 = gimple_build_call (fn, 2, def, size_int (alignment));
1472 g = call;
1473 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1474 ptr_type_node))
1475 need_cvt = true;
1476 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1477 gimple_call_set_lhs (g, t);
1478 gimple_seq_add_stmt_without_update (&seq, g);
1479 if (need_cvt)
1480 {
1481 t = make_ssa_name (orig_arg);
1482 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1483 gimple_seq_add_stmt_without_update (&seq, g);
1484 }
1485 gsi_insert_seq_on_edge_immediate
1486 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1487
1488 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1489 node->create_edge (cgraph_node::get_create (fn),
1490 call, entry_bb->count);
1491
1492 imm_use_iterator iter;
1493 use_operand_p use_p;
1494 gimple *use_stmt;
1495 tree repl = gimple_get_lhs (g);
1496 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1497 if (is_gimple_debug (use_stmt) || use_stmt == call)
1498 continue;
1499 else
1500 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1501 SET_USE (use_p, repl);
1502 }
1503 }
1504 else if ((node->simdclone->args[i].arg_type
1505 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1506 || (node->simdclone->args[i].arg_type
1507 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1508 || (node->simdclone->args[i].arg_type
1509 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1510 || (node->simdclone->args[i].arg_type
1511 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1512 {
1513 tree orig_arg = node->simdclone->args[i].orig_arg;
1514 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1515 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1516 tree def = NULL_TREE;
1517 if (TREE_ADDRESSABLE (orig_arg))
1518 {
1519 def = make_ssa_name (TREE_TYPE (orig_arg));
1520 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1521 if (incr_bb)
1522 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1523 gsi = gsi_after_labels (entry_bb);
1524 g = gimple_build_assign (def, orig_arg);
1525 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1526 }
1527 else
1528 {
1529 def = ssa_default_def (cfun, orig_arg);
1530 if (!def || has_zero_uses (def))
1531 def = NULL_TREE;
1532 else
1533 {
1534 iter1 = make_ssa_name (orig_arg);
1535 if (incr_bb)
1536 iter2 = make_ssa_name (orig_arg);
1537 }
1538 }
1539 if (def)
1540 {
1541 phi = create_phi_node (iter1, body_bb);
1542 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1543 if (incr_bb)
1544 {
1545 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1546 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1547 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1548 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1549 ? TREE_TYPE (orig_arg) : sizetype;
1550 tree addcst = simd_clone_linear_addend (node, i, addtype,
1551 entry_bb);
1552 gsi = gsi_last_bb (incr_bb);
1553 g = gimple_build_assign (iter2, code, iter1, addcst);
1554 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1555 }
1556
1557 imm_use_iterator iter;
1558 use_operand_p use_p;
1559 gimple *use_stmt;
1560 if (TREE_ADDRESSABLE (orig_arg))
1561 {
1562 gsi = gsi_after_labels (body_bb);
1563 g = gimple_build_assign (orig_arg, iter1);
1564 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1565 }
1566 else
1567 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1568 if (use_stmt == phi)
1569 continue;
1570 else
1571 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1572 SET_USE (use_p, iter1);
1573 }
1574 }
1575 else if (node->simdclone->args[i].arg_type
1576 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1577 || (node->simdclone->args[i].arg_type
1578 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1579 {
1580 tree orig_arg = node->simdclone->args[i].orig_arg;
1581 tree def = ssa_default_def (cfun, orig_arg);
1582 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1583 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1584 if (def && !has_zero_uses (def))
1585 {
1586 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1587 iter1 = make_ssa_name (orig_arg);
1588 if (incr_bb)
1589 iter2 = make_ssa_name (orig_arg);
1590 tree iter3 = make_ssa_name (rtype);
1591 tree iter4 = make_ssa_name (rtype);
1592 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1593 gsi = gsi_after_labels (entry_bb);
1594 gimple *load
1595 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1596 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1597
1598 tree array = node->simdclone->args[i].simd_array;
1599 TREE_ADDRESSABLE (array) = 1;
1600 tree ptr = build_fold_addr_expr (array);
1601 phi = create_phi_node (iter1, body_bb);
1602 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1603 if (incr_bb)
1604 {
1605 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1606 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1607 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1608 gsi = gsi_last_bb (incr_bb);
1609 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1610 }
1611
1612 phi = create_phi_node (iter4, body_bb);
1613 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1614 if (incr_bb)
1615 {
1616 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1617 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1618 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1619 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1620 ? TREE_TYPE (iter3) : sizetype;
1621 tree addcst = simd_clone_linear_addend (node, i, addtype,
1622 entry_bb);
1623 g = gimple_build_assign (iter5, code, iter4, addcst);
1624 gsi = gsi_last_bb (incr_bb);
1625 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1626 }
1627
1628 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1629 gsi = gsi_after_labels (body_bb);
1630 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1631
1632 imm_use_iterator iter;
1633 use_operand_p use_p;
1634 gimple *use_stmt;
1635 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1636 if (use_stmt == load)
1637 continue;
1638 else
1639 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1640 SET_USE (use_p, iter1);
1641
1642 if (!TYPE_READONLY (rtype) && incr_bb)
1643 {
1644 tree v = make_ssa_name (rtype);
1645 tree aref = build4 (ARRAY_REF, rtype, array,
1646 size_zero_node, NULL_TREE,
1647 NULL_TREE);
1648 gsi = gsi_after_labels (new_exit_bb);
1649 g = gimple_build_assign (v, aref);
1650 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1651 g = gimple_build_assign (build_simple_mem_ref (def), v);
1652 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1653 }
1654 }
1655 }
1656
1657 calculate_dominance_info (CDI_DOMINATORS);
1658 if (loop)
1659 add_loop (loop, loop->header->loop_father);
1660 update_ssa (TODO_update_ssa);
1661
1662 pop_cfun ();
1663 }
1664
1665 /* If the function in NODE is tagged as an elemental SIMD function,
1666 create the appropriate SIMD clones. */
1667
1668 void
expand_simd_clones(struct cgraph_node * node)1669 expand_simd_clones (struct cgraph_node *node)
1670 {
1671 tree attr = lookup_attribute ("omp declare simd",
1672 DECL_ATTRIBUTES (node->decl));
1673 if (attr == NULL_TREE
1674 || node->inlined_to
1675 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1676 return;
1677
1678 /* Ignore
1679 #pragma omp declare simd
1680 extern int foo ();
1681 in C, there we don't know the argument types at all. */
1682 if (!node->definition
1683 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1684 return;
1685
1686 /* Call this before creating clone_info, as it might ggc_collect. */
1687 if (node->definition && node->has_gimple_body_p ())
1688 node->get_body ();
1689
1690 do
1691 {
1692 /* Start with parsing the "omp declare simd" attribute(s). */
1693 bool inbranch_clause_specified;
1694 struct cgraph_simd_clone *clone_info
1695 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1696 &inbranch_clause_specified);
1697 if (clone_info == NULL)
1698 continue;
1699
1700 poly_uint64 orig_simdlen = clone_info->simdlen;
1701 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1702 /* The target can return 0 (no simd clones should be created),
1703 1 (just one ISA of simd clones should be created) or higher
1704 count of ISA variants. In that case, clone_info is initialized
1705 for the first ISA variant. */
1706 int count
1707 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1708 base_type, 0);
1709 if (count == 0)
1710 continue;
1711
1712 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1713 also create one inbranch and one !inbranch clone of it. */
1714 for (int i = 0; i < count * 2; i++)
1715 {
1716 struct cgraph_simd_clone *clone = clone_info;
1717 if (inbranch_clause_specified && (i & 1) != 0)
1718 continue;
1719
1720 if (i != 0)
1721 {
1722 clone = simd_clone_struct_alloc (clone_info->nargs
1723 + ((i & 1) != 0));
1724 simd_clone_struct_copy (clone, clone_info);
1725 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1726 and simd_clone_adjust_argument_types did to the first
1727 clone's info. */
1728 clone->nargs -= clone_info->inbranch;
1729 clone->simdlen = orig_simdlen;
1730 /* And call the target hook again to get the right ISA. */
1731 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1732 base_type,
1733 i / 2);
1734 if ((i & 1) != 0)
1735 clone->inbranch = 1;
1736 }
1737
1738 /* simd_clone_mangle might fail if such a clone has been created
1739 already. */
1740 tree id = simd_clone_mangle (node, clone);
1741 if (id == NULL_TREE)
1742 {
1743 if (i == 0)
1744 clone->nargs += clone->inbranch;
1745 continue;
1746 }
1747
1748 /* Only when we are sure we want to create the clone actually
1749 clone the function (or definitions) or create another
1750 extern FUNCTION_DECL (for prototypes without definitions). */
1751 struct cgraph_node *n = simd_clone_create (node);
1752 if (n == NULL)
1753 {
1754 if (i == 0)
1755 clone->nargs += clone->inbranch;
1756 continue;
1757 }
1758
1759 n->simdclone = clone;
1760 clone->origin = node;
1761 clone->next_clone = NULL;
1762 if (node->simd_clones == NULL)
1763 {
1764 clone->prev_clone = n;
1765 node->simd_clones = n;
1766 }
1767 else
1768 {
1769 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1770 clone->prev_clone->simdclone->next_clone = n;
1771 node->simd_clones->simdclone->prev_clone = n;
1772 }
1773 symtab->change_decl_assembler_name (n->decl, id);
1774 /* And finally adjust the return type, parameters and for
1775 definitions also function body. */
1776 if (node->definition)
1777 simd_clone_adjust (n);
1778 else
1779 {
1780 TREE_TYPE (n->decl)
1781 = build_distinct_type_copy (TREE_TYPE (n->decl));
1782 targetm.simd_clone.adjust (n);
1783 simd_clone_adjust_return_type (n);
1784 simd_clone_adjust_argument_types (n);
1785 }
1786 }
1787 }
1788 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1789 }
1790
1791 /* Entry point for IPA simd clone creation pass. */
1792
1793 static unsigned int
ipa_omp_simd_clone(void)1794 ipa_omp_simd_clone (void)
1795 {
1796 struct cgraph_node *node;
1797 FOR_EACH_FUNCTION (node)
1798 expand_simd_clones (node);
1799 return 0;
1800 }
1801
1802 namespace {
1803
1804 const pass_data pass_data_omp_simd_clone =
1805 {
1806 SIMPLE_IPA_PASS, /* type */
1807 "simdclone", /* name */
1808 OPTGROUP_OMP, /* optinfo_flags */
1809 TV_NONE, /* tv_id */
1810 ( PROP_ssa | PROP_cfg ), /* properties_required */
1811 0, /* properties_provided */
1812 0, /* properties_destroyed */
1813 0, /* todo_flags_start */
1814 0, /* todo_flags_finish */
1815 };
1816
1817 class pass_omp_simd_clone : public simple_ipa_opt_pass
1818 {
1819 public:
pass_omp_simd_clone(gcc::context * ctxt)1820 pass_omp_simd_clone(gcc::context *ctxt)
1821 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1822 {}
1823
1824 /* opt_pass methods: */
1825 virtual bool gate (function *);
execute(function *)1826 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1827 };
1828
1829 bool
gate(function *)1830 pass_omp_simd_clone::gate (function *)
1831 {
1832 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1833 }
1834
1835 } // anon namespace
1836
1837 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1838 make_pass_omp_simd_clone (gcc::context *ctxt)
1839 {
1840 return new pass_omp_simd_clone (ctxt);
1841 }
1842