1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2016 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-prop.h"
49 #include "tree-eh.h"
50
51
52 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
53 of arguments to reserve space for. */
54
55 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)56 simd_clone_struct_alloc (int nargs)
57 {
58 struct cgraph_simd_clone *clone_info;
59 size_t len = (sizeof (struct cgraph_simd_clone)
60 + nargs * sizeof (struct cgraph_simd_clone_arg));
61 clone_info = (struct cgraph_simd_clone *)
62 ggc_internal_cleared_alloc (len);
63 return clone_info;
64 }
65
66 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
67
68 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)69 simd_clone_struct_copy (struct cgraph_simd_clone *to,
70 struct cgraph_simd_clone *from)
71 {
72 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
73 + ((from->nargs - from->inbranch)
74 * sizeof (struct cgraph_simd_clone_arg))));
75 }
76
77 /* Return vector of parameter types of function FNDECL. This uses
78 TYPE_ARG_TYPES if available, otherwise falls back to types of
79 DECL_ARGUMENTS types. */
80
81 static vec<tree>
simd_clone_vector_of_formal_parm_types(tree fndecl)82 simd_clone_vector_of_formal_parm_types (tree fndecl)
83 {
84 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
85 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
86 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
87 unsigned int i;
88 tree arg;
89 FOR_EACH_VEC_ELT (args, i, arg)
90 args[i] = TREE_TYPE (args[i]);
91 return args;
92 }
93
94 /* Given a simd function in NODE, extract the simd specific
95 information from the OMP clauses passed in CLAUSES, and return
96 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
97 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
98 otherwise set to FALSE. */
99
100 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)101 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
102 bool *inbranch_specified)
103 {
104 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
105 tree t;
106 int n;
107 *inbranch_specified = false;
108
109 n = args.length ();
110 if (n > 0 && args.last () == void_type_node)
111 n--;
112
113 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
114 be cloned have a distinctive artificial label in addition to "omp
115 declare simd". */
116 bool cilk_clone
117 = (flag_cilkplus
118 && lookup_attribute ("cilk simd function",
119 DECL_ATTRIBUTES (node->decl)));
120
121 /* Allocate one more than needed just in case this is an in-branch
122 clone which will require a mask argument. */
123 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
124 clone_info->nargs = n;
125 clone_info->cilk_elemental = cilk_clone;
126
127 if (!clauses)
128 {
129 args.release ();
130 return clone_info;
131 }
132 clauses = TREE_VALUE (clauses);
133 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
134 return clone_info;
135
136 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
137 {
138 switch (OMP_CLAUSE_CODE (t))
139 {
140 case OMP_CLAUSE_INBRANCH:
141 clone_info->inbranch = 1;
142 *inbranch_specified = true;
143 break;
144 case OMP_CLAUSE_NOTINBRANCH:
145 clone_info->inbranch = 0;
146 *inbranch_specified = true;
147 break;
148 case OMP_CLAUSE_SIMDLEN:
149 clone_info->simdlen
150 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
151 break;
152 case OMP_CLAUSE_LINEAR:
153 {
154 tree decl = OMP_CLAUSE_DECL (t);
155 tree step = OMP_CLAUSE_LINEAR_STEP (t);
156 int argno = TREE_INT_CST_LOW (decl);
157 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
158 {
159 enum cgraph_simd_clone_arg_type arg_type;
160 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
161 switch (OMP_CLAUSE_LINEAR_KIND (t))
162 {
163 case OMP_CLAUSE_LINEAR_REF:
164 arg_type
165 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
166 break;
167 case OMP_CLAUSE_LINEAR_UVAL:
168 arg_type
169 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
170 break;
171 case OMP_CLAUSE_LINEAR_VAL:
172 case OMP_CLAUSE_LINEAR_DEFAULT:
173 arg_type
174 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
175 break;
176 default:
177 gcc_unreachable ();
178 }
179 else
180 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
181 clone_info->args[argno].arg_type = arg_type;
182 clone_info->args[argno].linear_step = tree_to_shwi (step);
183 gcc_assert (clone_info->args[argno].linear_step >= 0
184 && clone_info->args[argno].linear_step < n);
185 }
186 else
187 {
188 if (POINTER_TYPE_P (args[argno]))
189 step = fold_convert (ssizetype, step);
190 if (!tree_fits_shwi_p (step))
191 {
192 warning_at (OMP_CLAUSE_LOCATION (t), 0,
193 "ignoring large linear step");
194 args.release ();
195 return NULL;
196 }
197 else if (integer_zerop (step))
198 {
199 warning_at (OMP_CLAUSE_LOCATION (t), 0,
200 "ignoring zero linear step");
201 args.release ();
202 return NULL;
203 }
204 else
205 {
206 enum cgraph_simd_clone_arg_type arg_type;
207 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
208 switch (OMP_CLAUSE_LINEAR_KIND (t))
209 {
210 case OMP_CLAUSE_LINEAR_REF:
211 arg_type
212 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
213 break;
214 case OMP_CLAUSE_LINEAR_UVAL:
215 arg_type
216 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
217 break;
218 case OMP_CLAUSE_LINEAR_VAL:
219 case OMP_CLAUSE_LINEAR_DEFAULT:
220 arg_type
221 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
222 break;
223 default:
224 gcc_unreachable ();
225 }
226 else
227 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
228 clone_info->args[argno].arg_type = arg_type;
229 clone_info->args[argno].linear_step = tree_to_shwi (step);
230 }
231 }
232 break;
233 }
234 case OMP_CLAUSE_UNIFORM:
235 {
236 tree decl = OMP_CLAUSE_DECL (t);
237 int argno = tree_to_uhwi (decl);
238 clone_info->args[argno].arg_type
239 = SIMD_CLONE_ARG_TYPE_UNIFORM;
240 break;
241 }
242 case OMP_CLAUSE_ALIGNED:
243 {
244 tree decl = OMP_CLAUSE_DECL (t);
245 int argno = tree_to_uhwi (decl);
246 clone_info->args[argno].alignment
247 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
248 break;
249 }
250 default:
251 break;
252 }
253 }
254 args.release ();
255 return clone_info;
256 }
257
258 /* Given a SIMD clone in NODE, calculate the characteristic data
259 type and return the coresponding type. The characteristic data
260 type is computed as described in the Intel Vector ABI. */
261
262 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)263 simd_clone_compute_base_data_type (struct cgraph_node *node,
264 struct cgraph_simd_clone *clone_info)
265 {
266 tree type = integer_type_node;
267 tree fndecl = node->decl;
268
269 /* a) For non-void function, the characteristic data type is the
270 return type. */
271 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
272 type = TREE_TYPE (TREE_TYPE (fndecl));
273
274 /* b) If the function has any non-uniform, non-linear parameters,
275 then the characteristic data type is the type of the first
276 such parameter. */
277 else
278 {
279 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
280 for (unsigned int i = 0; i < clone_info->nargs; ++i)
281 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
282 {
283 type = map[i];
284 break;
285 }
286 map.release ();
287 }
288
289 /* c) If the characteristic data type determined by a) or b) above
290 is struct, union, or class type which is pass-by-value (except
291 for the type that maps to the built-in complex data type), the
292 characteristic data type is int. */
293 if (RECORD_OR_UNION_TYPE_P (type)
294 && !aggregate_value_p (type, NULL)
295 && TREE_CODE (type) != COMPLEX_TYPE)
296 return integer_type_node;
297
298 /* d) If none of the above three classes is applicable, the
299 characteristic data type is int. */
300
301 return type;
302
303 /* e) For Intel Xeon Phi native and offload compilation, if the
304 resulting characteristic data type is 8-bit or 16-bit integer
305 data type, the characteristic data type is int. */
306 /* Well, we don't handle Xeon Phi yet. */
307 }
308
309 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)310 simd_clone_mangle (struct cgraph_node *node,
311 struct cgraph_simd_clone *clone_info)
312 {
313 char vecsize_mangle = clone_info->vecsize_mangle;
314 char mask = clone_info->inbranch ? 'M' : 'N';
315 unsigned int simdlen = clone_info->simdlen;
316 unsigned int n;
317 pretty_printer pp;
318
319 gcc_assert (vecsize_mangle && simdlen);
320
321 pp_string (&pp, "_ZGV");
322 pp_character (&pp, vecsize_mangle);
323 pp_character (&pp, mask);
324 pp_decimal_int (&pp, simdlen);
325
326 for (n = 0; n < clone_info->nargs; ++n)
327 {
328 struct cgraph_simd_clone_arg arg = clone_info->args[n];
329
330 switch (arg.arg_type)
331 {
332 case SIMD_CLONE_ARG_TYPE_UNIFORM:
333 pp_character (&pp, 'u');
334 break;
335 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
336 pp_character (&pp, 'l');
337 goto mangle_linear;
338 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
339 pp_character (&pp, 'R');
340 goto mangle_linear;
341 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
342 pp_character (&pp, 'L');
343 goto mangle_linear;
344 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
345 pp_character (&pp, 'U');
346 goto mangle_linear;
347 mangle_linear:
348 gcc_assert (arg.linear_step != 0);
349 if (arg.linear_step > 1)
350 pp_unsigned_wide_integer (&pp, arg.linear_step);
351 else if (arg.linear_step < 0)
352 {
353 pp_character (&pp, 'n');
354 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
355 arg.linear_step));
356 }
357 break;
358 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
359 pp_string (&pp, "ls");
360 pp_unsigned_wide_integer (&pp, arg.linear_step);
361 break;
362 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
363 pp_string (&pp, "Rs");
364 pp_unsigned_wide_integer (&pp, arg.linear_step);
365 break;
366 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
367 pp_string (&pp, "Ls");
368 pp_unsigned_wide_integer (&pp, arg.linear_step);
369 break;
370 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
371 pp_string (&pp, "Us");
372 pp_unsigned_wide_integer (&pp, arg.linear_step);
373 break;
374 default:
375 pp_character (&pp, 'v');
376 }
377 if (arg.alignment)
378 {
379 pp_character (&pp, 'a');
380 pp_decimal_int (&pp, arg.alignment);
381 }
382 }
383
384 pp_underscore (&pp);
385 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
386 if (*str == '*')
387 ++str;
388 pp_string (&pp, str);
389 str = pp_formatted_text (&pp);
390
391 /* If there already is a SIMD clone with the same mangled name, don't
392 add another one. This can happen e.g. for
393 #pragma omp declare simd
394 #pragma omp declare simd simdlen(8)
395 int foo (int, int);
396 if the simdlen is assumed to be 8 for the first one, etc. */
397 for (struct cgraph_node *clone = node->simd_clones; clone;
398 clone = clone->simdclone->next_clone)
399 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
400 str) == 0)
401 return NULL_TREE;
402
403 return get_identifier (str);
404 }
405
406 /* Create a simd clone of OLD_NODE and return it. */
407
408 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)409 simd_clone_create (struct cgraph_node *old_node)
410 {
411 struct cgraph_node *new_node;
412 if (old_node->definition)
413 {
414 if (!old_node->has_gimple_body_p ())
415 return NULL;
416 old_node->get_body ();
417 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
418 false, NULL, NULL,
419 "simdclone");
420 }
421 else
422 {
423 tree old_decl = old_node->decl;
424 tree new_decl = copy_node (old_node->decl);
425 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
426 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
427 SET_DECL_RTL (new_decl, NULL);
428 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
429 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
430 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
431 if (old_node->in_other_partition)
432 new_node->in_other_partition = 1;
433 }
434 if (new_node == NULL)
435 return new_node;
436
437 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
438
439 /* The function cgraph_function_versioning () will force the new
440 symbol local. Undo this, and inherit external visability from
441 the old node. */
442 new_node->local.local = old_node->local.local;
443 new_node->externally_visible = old_node->externally_visible;
444
445 return new_node;
446 }
447
448 /* Adjust the return type of the given function to its appropriate
449 vector counterpart. Returns a simd array to be used throughout the
450 function as a return value. */
451
452 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)453 simd_clone_adjust_return_type (struct cgraph_node *node)
454 {
455 tree fndecl = node->decl;
456 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
457 unsigned int veclen;
458 tree t;
459
460 /* Adjust the function return type. */
461 if (orig_rettype == void_type_node)
462 return NULL_TREE;
463 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
464 t = TREE_TYPE (TREE_TYPE (fndecl));
465 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
466 veclen = node->simdclone->vecsize_int;
467 else
468 veclen = node->simdclone->vecsize_float;
469 veclen /= GET_MODE_BITSIZE (TYPE_MODE (t));
470 if (veclen > node->simdclone->simdlen)
471 veclen = node->simdclone->simdlen;
472 if (POINTER_TYPE_P (t))
473 t = pointer_sized_int_node;
474 if (veclen == node->simdclone->simdlen)
475 t = build_vector_type (t, node->simdclone->simdlen);
476 else
477 {
478 t = build_vector_type (t, veclen);
479 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
480 }
481 TREE_TYPE (TREE_TYPE (fndecl)) = t;
482 if (!node->definition)
483 return NULL_TREE;
484
485 t = DECL_RESULT (fndecl);
486 /* Adjust the DECL_RESULT. */
487 gcc_assert (TREE_TYPE (t) != void_type_node);
488 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
489 relayout_decl (t);
490
491 tree atype = build_array_type_nelts (orig_rettype,
492 node->simdclone->simdlen);
493 if (veclen != node->simdclone->simdlen)
494 return build1 (VIEW_CONVERT_EXPR, atype, t);
495
496 /* Set up a SIMD array to use as the return value. */
497 tree retval = create_tmp_var_raw (atype, "retval");
498 gimple_add_tmp_var (retval);
499 return retval;
500 }
501
502 /* Each vector argument has a corresponding array to be used locally
503 as part of the eventual loop. Create such temporary array and
504 return it.
505
506 PREFIX is the prefix to be used for the temporary.
507
508 TYPE is the inner element type.
509
510 SIMDLEN is the number of elements. */
511
512 static tree
create_tmp_simd_array(const char * prefix,tree type,int simdlen)513 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
514 {
515 tree atype = build_array_type_nelts (type, simdlen);
516 tree avar = create_tmp_var_raw (atype, prefix);
517 gimple_add_tmp_var (avar);
518 return avar;
519 }
520
521 /* Modify the function argument types to their corresponding vector
522 counterparts if appropriate. Also, create one array for each simd
523 argument to be used locally when using the function arguments as
524 part of the loop.
525
526 NODE is the function whose arguments are to be adjusted.
527
528 Returns an adjustment vector that will be filled describing how the
529 argument types will be adjusted. */
530
531 static ipa_parm_adjustment_vec
simd_clone_adjust_argument_types(struct cgraph_node * node)532 simd_clone_adjust_argument_types (struct cgraph_node *node)
533 {
534 vec<tree> args;
535 ipa_parm_adjustment_vec adjustments;
536
537 if (node->definition)
538 args = ipa_get_vector_of_formal_parms (node->decl);
539 else
540 args = simd_clone_vector_of_formal_parm_types (node->decl);
541 adjustments.create (args.length ());
542 unsigned i, j, veclen;
543 struct ipa_parm_adjustment adj;
544 struct cgraph_simd_clone *sc = node->simdclone;
545
546 for (i = 0; i < sc->nargs; ++i)
547 {
548 memset (&adj, 0, sizeof (adj));
549 tree parm = args[i];
550 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
551 adj.base_index = i;
552 adj.base = parm;
553
554 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
555 sc->args[i].orig_type = parm_type;
556
557 switch (sc->args[i].arg_type)
558 {
559 default:
560 /* No adjustment necessary for scalar arguments. */
561 adj.op = IPA_PARM_OP_COPY;
562 break;
563 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
564 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
565 if (node->definition)
566 sc->args[i].simd_array
567 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
568 TREE_TYPE (parm_type),
569 sc->simdlen);
570 adj.op = IPA_PARM_OP_COPY;
571 break;
572 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
573 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
574 case SIMD_CLONE_ARG_TYPE_VECTOR:
575 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
576 veclen = sc->vecsize_int;
577 else
578 veclen = sc->vecsize_float;
579 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
580 if (veclen > sc->simdlen)
581 veclen = sc->simdlen;
582 adj.arg_prefix = "simd";
583 if (POINTER_TYPE_P (parm_type))
584 adj.type = build_vector_type (pointer_sized_int_node, veclen);
585 else
586 adj.type = build_vector_type (parm_type, veclen);
587 sc->args[i].vector_type = adj.type;
588 for (j = veclen; j < sc->simdlen; j += veclen)
589 {
590 adjustments.safe_push (adj);
591 if (j == veclen)
592 {
593 memset (&adj, 0, sizeof (adj));
594 adj.op = IPA_PARM_OP_NEW;
595 adj.arg_prefix = "simd";
596 adj.base_index = i;
597 adj.type = sc->args[i].vector_type;
598 }
599 }
600
601 if (node->definition)
602 sc->args[i].simd_array
603 = create_tmp_simd_array (DECL_NAME (parm)
604 ? IDENTIFIER_POINTER (DECL_NAME (parm))
605 : NULL, parm_type, sc->simdlen);
606 }
607 adjustments.safe_push (adj);
608 }
609
610 if (sc->inbranch)
611 {
612 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
613
614 memset (&adj, 0, sizeof (adj));
615 adj.op = IPA_PARM_OP_NEW;
616 adj.arg_prefix = "mask";
617
618 adj.base_index = i;
619 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
620 veclen = sc->vecsize_int;
621 else
622 veclen = sc->vecsize_float;
623 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
624 if (veclen > sc->simdlen)
625 veclen = sc->simdlen;
626 if (sc->mask_mode != VOIDmode)
627 adj.type
628 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
629 else if (POINTER_TYPE_P (base_type))
630 adj.type = build_vector_type (pointer_sized_int_node, veclen);
631 else
632 adj.type = build_vector_type (base_type, veclen);
633 adjustments.safe_push (adj);
634
635 for (j = veclen; j < sc->simdlen; j += veclen)
636 adjustments.safe_push (adj);
637
638 /* We have previously allocated one extra entry for the mask. Use
639 it and fill it. */
640 sc->nargs++;
641 if (sc->mask_mode != VOIDmode)
642 base_type = boolean_type_node;
643 if (node->definition)
644 {
645 sc->args[i].orig_arg
646 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
647 if (sc->mask_mode == VOIDmode)
648 sc->args[i].simd_array
649 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
650 else if (veclen < sc->simdlen)
651 sc->args[i].simd_array
652 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
653 else
654 sc->args[i].simd_array = NULL_TREE;
655 }
656 sc->args[i].orig_type = base_type;
657 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
658 }
659
660 if (node->definition)
661 ipa_modify_formal_parameters (node->decl, adjustments);
662 else
663 {
664 tree new_arg_types = NULL_TREE, new_reversed;
665 bool last_parm_void = false;
666 if (args.length () > 0 && args.last () == void_type_node)
667 last_parm_void = true;
668
669 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
670 j = adjustments.length ();
671 for (i = 0; i < j; i++)
672 {
673 struct ipa_parm_adjustment *adj = &adjustments[i];
674 tree ptype;
675 if (adj->op == IPA_PARM_OP_COPY)
676 ptype = args[adj->base_index];
677 else
678 ptype = adj->type;
679 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
680 }
681 new_reversed = nreverse (new_arg_types);
682 if (last_parm_void)
683 {
684 if (new_reversed)
685 TREE_CHAIN (new_arg_types) = void_list_node;
686 else
687 new_reversed = void_list_node;
688 }
689
690 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
691 TYPE_ARG_TYPES (new_type) = new_reversed;
692 TREE_TYPE (node->decl) = new_type;
693
694 adjustments.release ();
695 }
696 args.release ();
697 return adjustments;
698 }
699
700 /* Initialize and copy the function arguments in NODE to their
701 corresponding local simd arrays. Returns a fresh gimple_seq with
702 the instruction sequence generated. */
703
704 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments)705 simd_clone_init_simd_arrays (struct cgraph_node *node,
706 ipa_parm_adjustment_vec adjustments)
707 {
708 gimple_seq seq = NULL;
709 unsigned i = 0, j = 0, k;
710
711 for (tree arg = DECL_ARGUMENTS (node->decl);
712 arg;
713 arg = DECL_CHAIN (arg), i++, j++)
714 {
715 if (adjustments[j].op == IPA_PARM_OP_COPY
716 || POINTER_TYPE_P (TREE_TYPE (arg)))
717 continue;
718
719 node->simdclone->args[i].vector_arg = arg;
720
721 tree array = node->simdclone->args[i].simd_array;
722 if (node->simdclone->mask_mode != VOIDmode
723 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
724 {
725 if (array == NULL_TREE)
726 continue;
727 unsigned int l
728 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
729 for (k = 0; k <= l; k++)
730 {
731 if (k)
732 {
733 arg = DECL_CHAIN (arg);
734 j++;
735 }
736 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
737 array, size_int (k), NULL, NULL);
738 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
739 gimplify_and_add (t, &seq);
740 }
741 continue;
742 }
743 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
744 {
745 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
746 tree ptr = build_fold_addr_expr (array);
747 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
748 build_int_cst (ptype, 0));
749 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
750 gimplify_and_add (t, &seq);
751 }
752 else
753 {
754 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
755 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
756 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
757 {
758 tree ptr = build_fold_addr_expr (array);
759 int elemsize;
760 if (k)
761 {
762 arg = DECL_CHAIN (arg);
763 j++;
764 }
765 elemsize
766 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
767 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
768 build_int_cst (ptype, k * elemsize));
769 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
770 gimplify_and_add (t, &seq);
771 }
772 }
773 }
774 return seq;
775 }
776
777 /* Callback info for ipa_simd_modify_stmt_ops below. */
778
779 struct modify_stmt_info {
780 ipa_parm_adjustment_vec adjustments;
781 gimple *stmt;
782 /* True if the parent statement was modified by
783 ipa_simd_modify_stmt_ops. */
784 bool modified;
785 };
786
787 /* Callback for walk_gimple_op.
788
789 Adjust operands from a given statement as specified in the
790 adjustments vector in the callback data. */
791
792 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)793 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
794 {
795 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
796 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
797 tree *orig_tp = tp;
798 if (TREE_CODE (*tp) == ADDR_EXPR)
799 tp = &TREE_OPERAND (*tp, 0);
800 struct ipa_parm_adjustment *cand = NULL;
801 if (TREE_CODE (*tp) == PARM_DECL)
802 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
803 else
804 {
805 if (TYPE_P (*tp))
806 *walk_subtrees = 0;
807 }
808
809 tree repl = NULL_TREE;
810 if (cand)
811 repl = unshare_expr (cand->new_decl);
812 else
813 {
814 if (tp != orig_tp)
815 {
816 *walk_subtrees = 0;
817 bool modified = info->modified;
818 info->modified = false;
819 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
820 if (!info->modified)
821 {
822 info->modified = modified;
823 return NULL_TREE;
824 }
825 info->modified = modified;
826 repl = *tp;
827 }
828 else
829 return NULL_TREE;
830 }
831
832 if (tp != orig_tp)
833 {
834 repl = build_fold_addr_expr (repl);
835 gimple *stmt;
836 if (is_gimple_debug (info->stmt))
837 {
838 tree vexpr = make_node (DEBUG_EXPR_DECL);
839 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
840 DECL_ARTIFICIAL (vexpr) = 1;
841 TREE_TYPE (vexpr) = TREE_TYPE (repl);
842 DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl));
843 repl = vexpr;
844 }
845 else
846 {
847 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
848 repl = gimple_assign_lhs (stmt);
849 }
850 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
851 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
852 *orig_tp = repl;
853 }
854 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
855 {
856 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
857 *tp = vce;
858 }
859 else
860 *tp = repl;
861
862 info->modified = true;
863 return NULL_TREE;
864 }
865
866 /* Traverse the function body and perform all modifications as
867 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
868 modified such that the replacement/reduction value will now be an
869 offset into the corresponding simd_array.
870
871 This function will replace all function argument uses with their
872 corresponding simd array elements, and ajust the return values
873 accordingly. */
874
875 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments,tree retval_array,tree iter)876 ipa_simd_modify_function_body (struct cgraph_node *node,
877 ipa_parm_adjustment_vec adjustments,
878 tree retval_array, tree iter)
879 {
880 basic_block bb;
881 unsigned int i, j, l;
882
883 /* Re-use the adjustments array, but this time use it to replace
884 every function argument use to an offset into the corresponding
885 simd_array. */
886 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
887 {
888 if (!node->simdclone->args[i].vector_arg)
889 continue;
890
891 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
892 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
893 adjustments[j].new_decl
894 = build4 (ARRAY_REF,
895 basetype,
896 node->simdclone->args[i].simd_array,
897 iter,
898 NULL_TREE, NULL_TREE);
899 if (adjustments[j].op == IPA_PARM_OP_NONE
900 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
901 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
902 }
903
904 l = adjustments.length ();
905 for (i = 1; i < num_ssa_names; i++)
906 {
907 tree name = ssa_name (i);
908 if (name
909 && SSA_NAME_VAR (name)
910 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
911 {
912 for (j = 0; j < l; j++)
913 if (SSA_NAME_VAR (name) == adjustments[j].base
914 && adjustments[j].new_decl)
915 {
916 tree base_var;
917 if (adjustments[j].new_ssa_base == NULL_TREE)
918 {
919 base_var
920 = copy_var_decl (adjustments[j].base,
921 DECL_NAME (adjustments[j].base),
922 TREE_TYPE (adjustments[j].base));
923 adjustments[j].new_ssa_base = base_var;
924 }
925 else
926 base_var = adjustments[j].new_ssa_base;
927 if (SSA_NAME_IS_DEFAULT_DEF (name))
928 {
929 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
930 gimple_stmt_iterator gsi = gsi_after_labels (bb);
931 tree new_decl = unshare_expr (adjustments[j].new_decl);
932 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
933 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
934 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
935 gimple *stmt = gimple_build_assign (name, new_decl);
936 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
937 }
938 else
939 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
940 }
941 }
942 }
943
944 struct modify_stmt_info info;
945 info.adjustments = adjustments;
946
947 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
948 {
949 gimple_stmt_iterator gsi;
950
951 gsi = gsi_start_bb (bb);
952 while (!gsi_end_p (gsi))
953 {
954 gimple *stmt = gsi_stmt (gsi);
955 info.stmt = stmt;
956 struct walk_stmt_info wi;
957
958 memset (&wi, 0, sizeof (wi));
959 info.modified = false;
960 wi.info = &info;
961 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
962
963 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
964 {
965 tree retval = gimple_return_retval (return_stmt);
966 if (!retval)
967 {
968 gsi_remove (&gsi, true);
969 continue;
970 }
971
972 /* Replace `return foo' with `retval_array[iter] = foo'. */
973 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
974 retval_array, iter, NULL, NULL);
975 stmt = gimple_build_assign (ref, retval);
976 gsi_replace (&gsi, stmt, true);
977 info.modified = true;
978 }
979
980 if (info.modified)
981 {
982 update_stmt (stmt);
983 if (maybe_clean_eh_stmt (stmt))
984 gimple_purge_dead_eh_edges (gimple_bb (stmt));
985 }
986 gsi_next (&gsi);
987 }
988 }
989 }
990
991 /* Helper function of simd_clone_adjust, return linear step addend
992 of Ith argument. */
993
994 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)995 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
996 tree addtype, basic_block entry_bb)
997 {
998 tree ptype = NULL_TREE;
999 switch (node->simdclone->args[i].arg_type)
1000 {
1001 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1002 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1003 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1004 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1005 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1006 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1007 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1008 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1009 break;
1010 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1011 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1012 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1013 break;
1014 default:
1015 gcc_unreachable ();
1016 }
1017
1018 unsigned int idx = node->simdclone->args[i].linear_step;
1019 tree arg = node->simdclone->args[idx].orig_arg;
1020 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1021 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1022 gimple *g;
1023 tree ret;
1024 if (is_gimple_reg (arg))
1025 ret = get_or_create_ssa_default_def (cfun, arg);
1026 else
1027 {
1028 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1029 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1030 ret = gimple_assign_lhs (g);
1031 }
1032 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1033 {
1034 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1035 build_simple_mem_ref (ret));
1036 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1037 ret = gimple_assign_lhs (g);
1038 }
1039 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1040 {
1041 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1042 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1043 ret = gimple_assign_lhs (g);
1044 }
1045 if (POINTER_TYPE_P (ptype))
1046 {
1047 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1048 if (size && TREE_CODE (size) == INTEGER_CST)
1049 {
1050 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1051 ret, fold_convert (addtype, size));
1052 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1053 ret = gimple_assign_lhs (g);
1054 }
1055 }
1056 return ret;
1057 }
1058
1059 /* Adjust the argument types in NODE to their appropriate vector
1060 counterparts. */
1061
1062 static void
simd_clone_adjust(struct cgraph_node * node)1063 simd_clone_adjust (struct cgraph_node *node)
1064 {
1065 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1066
1067 targetm.simd_clone.adjust (node);
1068
1069 tree retval = simd_clone_adjust_return_type (node);
1070 ipa_parm_adjustment_vec adjustments
1071 = simd_clone_adjust_argument_types (node);
1072
1073 push_gimplify_context ();
1074
1075 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1076
1077 /* Adjust all uses of vector arguments accordingly. Adjust all
1078 return values accordingly. */
1079 tree iter = create_tmp_var (unsigned_type_node, "iter");
1080 tree iter1 = make_ssa_name (iter);
1081 tree iter2 = NULL_TREE;
1082 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1083
1084 /* Initialize the iteration variable. */
1085 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1086 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1087 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1088 /* Insert the SIMD array and iv initialization at function
1089 entry. */
1090 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1091
1092 pop_gimplify_context (NULL);
1093
1094 gimple *g;
1095 basic_block incr_bb = NULL;
1096 struct loop *loop = NULL;
1097
1098 /* Create a new BB right before the original exit BB, to hold the
1099 iteration increment and the condition/branch. */
1100 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1101 {
1102 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1103 incr_bb = create_empty_bb (orig_exit);
1104 add_bb_to_loop (incr_bb, body_bb->loop_father);
1105 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1106 flag. Set it now to be a FALLTHRU_EDGE. */
1107 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
1108 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
1109 for (unsigned i = 0;
1110 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
1111 {
1112 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
1113 redirect_edge_succ (e, incr_bb);
1114 }
1115 }
1116 else if (node->simdclone->inbranch)
1117 {
1118 incr_bb = create_empty_bb (entry_bb);
1119 add_bb_to_loop (incr_bb, body_bb->loop_father);
1120 }
1121
1122 if (incr_bb)
1123 {
1124 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1125 e->probability = REG_BR_PROB_BASE;
1126 gsi = gsi_last_bb (incr_bb);
1127 iter2 = make_ssa_name (iter);
1128 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1129 build_int_cst (unsigned_type_node, 1));
1130 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1131
1132 /* Mostly annotate the loop for the vectorizer (the rest is done
1133 below). */
1134 loop = alloc_loop ();
1135 cfun->has_force_vectorize_loops = true;
1136 loop->safelen = node->simdclone->simdlen;
1137 loop->force_vectorize = true;
1138 loop->header = body_bb;
1139 }
1140
1141 /* Branch around the body if the mask applies. */
1142 if (node->simdclone->inbranch)
1143 {
1144 gsi = gsi_last_bb (loop->header);
1145 tree mask_array
1146 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1147 tree mask;
1148 if (node->simdclone->mask_mode != VOIDmode)
1149 {
1150 tree shift_cnt;
1151 if (mask_array == NULL_TREE)
1152 {
1153 tree arg = node->simdclone->args[node->simdclone->nargs
1154 - 1].vector_arg;
1155 mask = get_or_create_ssa_default_def (cfun, arg);
1156 shift_cnt = iter1;
1157 }
1158 else
1159 {
1160 tree maskt = TREE_TYPE (mask_array);
1161 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1162 c = node->simdclone->simdlen / (c + 1);
1163 int s = exact_log2 (c);
1164 gcc_assert (s > 0);
1165 c--;
1166 tree idx = make_ssa_name (TREE_TYPE (iter1));
1167 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1168 build_int_cst (NULL_TREE, s));
1169 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1170 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1171 tree aref = build4 (ARRAY_REF,
1172 TREE_TYPE (TREE_TYPE (mask_array)),
1173 mask_array, idx, NULL, NULL);
1174 g = gimple_build_assign (mask, aref);
1175 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1176 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1177 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1178 build_int_cst (TREE_TYPE (iter1), c));
1179 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1180 }
1181 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1182 RSHIFT_EXPR, mask, shift_cnt);
1183 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1184 mask = gimple_assign_lhs (g);
1185 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1186 BIT_AND_EXPR, mask,
1187 build_int_cst (TREE_TYPE (mask), 1));
1188 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1189 mask = gimple_assign_lhs (g);
1190 }
1191 else
1192 {
1193 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1194 tree aref = build4 (ARRAY_REF,
1195 TREE_TYPE (TREE_TYPE (mask_array)),
1196 mask_array, iter1, NULL, NULL);
1197 g = gimple_build_assign (mask, aref);
1198 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1199 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
1200 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1201 {
1202 aref = build1 (VIEW_CONVERT_EXPR,
1203 build_nonstandard_integer_type (bitsize, 0),
1204 mask);
1205 mask = make_ssa_name (TREE_TYPE (aref));
1206 g = gimple_build_assign (mask, aref);
1207 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1208 }
1209 }
1210
1211 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1212 NULL, NULL);
1213 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1214 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1215 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
1216 }
1217
1218 basic_block latch_bb = NULL;
1219 basic_block new_exit_bb = NULL;
1220
1221 /* Generate the condition. */
1222 if (incr_bb)
1223 {
1224 gsi = gsi_last_bb (incr_bb);
1225 g = gimple_build_cond (LT_EXPR, iter2,
1226 build_int_cst (unsigned_type_node,
1227 node->simdclone->simdlen),
1228 NULL, NULL);
1229 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1230 edge e = split_block (incr_bb, gsi_stmt (gsi));
1231 latch_bb = e->dest;
1232 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1233 loop->latch = latch_bb;
1234
1235 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1236
1237 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1238 /* The successor of incr_bb is already pointing to latch_bb; just
1239 change the flags.
1240 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1241 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1242 }
1243
1244 gphi *phi = create_phi_node (iter1, body_bb);
1245 edge preheader_edge = find_edge (entry_bb, body_bb);
1246 edge latch_edge = NULL;
1247 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1248 UNKNOWN_LOCATION);
1249 if (incr_bb)
1250 {
1251 latch_edge = single_succ_edge (latch_bb);
1252 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1253
1254 /* Generate the new return. */
1255 gsi = gsi_last_bb (new_exit_bb);
1256 if (retval
1257 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1258 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1259 retval = TREE_OPERAND (retval, 0);
1260 else if (retval)
1261 {
1262 retval = build1 (VIEW_CONVERT_EXPR,
1263 TREE_TYPE (TREE_TYPE (node->decl)),
1264 retval);
1265 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1266 false, GSI_CONTINUE_LINKING);
1267 }
1268 g = gimple_build_return (retval);
1269 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1270 }
1271
1272 /* Handle aligned clauses by replacing default defs of the aligned
1273 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1274 lhs. Handle linear by adding PHIs. */
1275 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1276 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1277 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1278 || !is_gimple_reg_type
1279 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1280 {
1281 tree orig_arg = node->simdclone->args[i].orig_arg;
1282 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1283 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1284 else
1285 {
1286 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1287 gimple_add_tmp_var (iter1);
1288 }
1289 gsi = gsi_after_labels (entry_bb);
1290 g = gimple_build_assign (iter1, orig_arg);
1291 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1292 gsi = gsi_after_labels (body_bb);
1293 g = gimple_build_assign (orig_arg, iter1);
1294 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1295 }
1296 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1297 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1298 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1299 == REFERENCE_TYPE
1300 && TREE_ADDRESSABLE
1301 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1302 {
1303 tree orig_arg = node->simdclone->args[i].orig_arg;
1304 tree def = ssa_default_def (cfun, orig_arg);
1305 if (def && !has_zero_uses (def))
1306 {
1307 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1308 gimple_add_tmp_var (iter1);
1309 gsi = gsi_after_labels (entry_bb);
1310 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1311 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1312 gsi = gsi_after_labels (body_bb);
1313 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1314 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1315 }
1316 }
1317 else if (node->simdclone->args[i].alignment
1318 && node->simdclone->args[i].arg_type
1319 == SIMD_CLONE_ARG_TYPE_UNIFORM
1320 && (node->simdclone->args[i].alignment
1321 & (node->simdclone->args[i].alignment - 1)) == 0
1322 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1323 == POINTER_TYPE)
1324 {
1325 unsigned int alignment = node->simdclone->args[i].alignment;
1326 tree orig_arg = node->simdclone->args[i].orig_arg;
1327 tree def = ssa_default_def (cfun, orig_arg);
1328 if (def && !has_zero_uses (def))
1329 {
1330 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1331 gimple_seq seq = NULL;
1332 bool need_cvt = false;
1333 gcall *call
1334 = gimple_build_call (fn, 2, def, size_int (alignment));
1335 g = call;
1336 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1337 ptr_type_node))
1338 need_cvt = true;
1339 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1340 gimple_call_set_lhs (g, t);
1341 gimple_seq_add_stmt_without_update (&seq, g);
1342 if (need_cvt)
1343 {
1344 t = make_ssa_name (orig_arg);
1345 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1346 gimple_seq_add_stmt_without_update (&seq, g);
1347 }
1348 gsi_insert_seq_on_edge_immediate
1349 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1350
1351 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1352 int freq = compute_call_stmt_bb_frequency (current_function_decl,
1353 entry_bb);
1354 node->create_edge (cgraph_node::get_create (fn),
1355 call, entry_bb->count, freq);
1356
1357 imm_use_iterator iter;
1358 use_operand_p use_p;
1359 gimple *use_stmt;
1360 tree repl = gimple_get_lhs (g);
1361 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1362 if (is_gimple_debug (use_stmt) || use_stmt == call)
1363 continue;
1364 else
1365 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1366 SET_USE (use_p, repl);
1367 }
1368 }
1369 else if ((node->simdclone->args[i].arg_type
1370 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1371 || (node->simdclone->args[i].arg_type
1372 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1373 || (node->simdclone->args[i].arg_type
1374 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1375 || (node->simdclone->args[i].arg_type
1376 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1377 {
1378 tree orig_arg = node->simdclone->args[i].orig_arg;
1379 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1380 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1381 tree def = NULL_TREE;
1382 if (TREE_ADDRESSABLE (orig_arg))
1383 {
1384 def = make_ssa_name (TREE_TYPE (orig_arg));
1385 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1386 if (incr_bb)
1387 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1388 gsi = gsi_after_labels (entry_bb);
1389 g = gimple_build_assign (def, orig_arg);
1390 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1391 }
1392 else
1393 {
1394 def = ssa_default_def (cfun, orig_arg);
1395 if (!def || has_zero_uses (def))
1396 def = NULL_TREE;
1397 else
1398 {
1399 iter1 = make_ssa_name (orig_arg);
1400 if (incr_bb)
1401 iter2 = make_ssa_name (orig_arg);
1402 }
1403 }
1404 if (def)
1405 {
1406 phi = create_phi_node (iter1, body_bb);
1407 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1408 if (incr_bb)
1409 {
1410 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1411 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1412 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1413 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1414 ? TREE_TYPE (orig_arg) : sizetype;
1415 tree addcst = simd_clone_linear_addend (node, i, addtype,
1416 entry_bb);
1417 gsi = gsi_last_bb (incr_bb);
1418 g = gimple_build_assign (iter2, code, iter1, addcst);
1419 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1420 }
1421
1422 imm_use_iterator iter;
1423 use_operand_p use_p;
1424 gimple *use_stmt;
1425 if (TREE_ADDRESSABLE (orig_arg))
1426 {
1427 gsi = gsi_after_labels (body_bb);
1428 g = gimple_build_assign (orig_arg, iter1);
1429 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1430 }
1431 else
1432 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1433 if (use_stmt == phi)
1434 continue;
1435 else
1436 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1437 SET_USE (use_p, iter1);
1438 }
1439 }
1440 else if (node->simdclone->args[i].arg_type
1441 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1442 || (node->simdclone->args[i].arg_type
1443 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1444 {
1445 tree orig_arg = node->simdclone->args[i].orig_arg;
1446 tree def = ssa_default_def (cfun, orig_arg);
1447 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1448 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1449 if (def && !has_zero_uses (def))
1450 {
1451 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1452 iter1 = make_ssa_name (orig_arg);
1453 if (incr_bb)
1454 iter2 = make_ssa_name (orig_arg);
1455 tree iter3 = make_ssa_name (rtype);
1456 tree iter4 = make_ssa_name (rtype);
1457 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1458 gsi = gsi_after_labels (entry_bb);
1459 gimple *load
1460 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1461 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1462
1463 tree array = node->simdclone->args[i].simd_array;
1464 TREE_ADDRESSABLE (array) = 1;
1465 tree ptr = build_fold_addr_expr (array);
1466 phi = create_phi_node (iter1, body_bb);
1467 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1468 if (incr_bb)
1469 {
1470 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1471 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1472 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1473 gsi = gsi_last_bb (incr_bb);
1474 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1475 }
1476
1477 phi = create_phi_node (iter4, body_bb);
1478 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1479 if (incr_bb)
1480 {
1481 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1482 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1483 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1484 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1485 ? TREE_TYPE (iter3) : sizetype;
1486 tree addcst = simd_clone_linear_addend (node, i, addtype,
1487 entry_bb);
1488 g = gimple_build_assign (iter5, code, iter4, addcst);
1489 gsi = gsi_last_bb (incr_bb);
1490 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1491 }
1492
1493 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1494 gsi = gsi_after_labels (body_bb);
1495 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1496
1497 imm_use_iterator iter;
1498 use_operand_p use_p;
1499 gimple *use_stmt;
1500 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1501 if (use_stmt == load)
1502 continue;
1503 else
1504 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1505 SET_USE (use_p, iter1);
1506
1507 if (!TYPE_READONLY (rtype) && incr_bb)
1508 {
1509 tree v = make_ssa_name (rtype);
1510 tree aref = build4 (ARRAY_REF, rtype, array,
1511 size_zero_node, NULL_TREE,
1512 NULL_TREE);
1513 gsi = gsi_after_labels (new_exit_bb);
1514 g = gimple_build_assign (v, aref);
1515 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1516 g = gimple_build_assign (build_simple_mem_ref (def), v);
1517 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1518 }
1519 }
1520 }
1521
1522 calculate_dominance_info (CDI_DOMINATORS);
1523 if (loop)
1524 add_loop (loop, loop->header->loop_father);
1525 update_ssa (TODO_update_ssa);
1526
1527 pop_cfun ();
1528 }
1529
1530 /* If the function in NODE is tagged as an elemental SIMD function,
1531 create the appropriate SIMD clones. */
1532
1533 static void
expand_simd_clones(struct cgraph_node * node)1534 expand_simd_clones (struct cgraph_node *node)
1535 {
1536 tree attr = lookup_attribute ("omp declare simd",
1537 DECL_ATTRIBUTES (node->decl));
1538 if (attr == NULL_TREE
1539 || node->global.inlined_to
1540 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1541 return;
1542
1543 /* Ignore
1544 #pragma omp declare simd
1545 extern int foo ();
1546 in C, there we don't know the argument types at all. */
1547 if (!node->definition
1548 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1549 return;
1550
1551 /* Call this before creating clone_info, as it might ggc_collect. */
1552 if (node->definition && node->has_gimple_body_p ())
1553 node->get_body ();
1554
1555 do
1556 {
1557 /* Start with parsing the "omp declare simd" attribute(s). */
1558 bool inbranch_clause_specified;
1559 struct cgraph_simd_clone *clone_info
1560 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1561 &inbranch_clause_specified);
1562 if (clone_info == NULL)
1563 continue;
1564
1565 int orig_simdlen = clone_info->simdlen;
1566 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1567 /* The target can return 0 (no simd clones should be created),
1568 1 (just one ISA of simd clones should be created) or higher
1569 count of ISA variants. In that case, clone_info is initialized
1570 for the first ISA variant. */
1571 int count
1572 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1573 base_type, 0);
1574 if (count == 0)
1575 continue;
1576
1577 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1578 also create one inbranch and one !inbranch clone of it. */
1579 for (int i = 0; i < count * 2; i++)
1580 {
1581 struct cgraph_simd_clone *clone = clone_info;
1582 if (inbranch_clause_specified && (i & 1) != 0)
1583 continue;
1584
1585 if (i != 0)
1586 {
1587 clone = simd_clone_struct_alloc (clone_info->nargs
1588 + ((i & 1) != 0));
1589 simd_clone_struct_copy (clone, clone_info);
1590 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1591 and simd_clone_adjust_argument_types did to the first
1592 clone's info. */
1593 clone->nargs -= clone_info->inbranch;
1594 clone->simdlen = orig_simdlen;
1595 /* And call the target hook again to get the right ISA. */
1596 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1597 base_type,
1598 i / 2);
1599 if ((i & 1) != 0)
1600 clone->inbranch = 1;
1601 }
1602
1603 /* simd_clone_mangle might fail if such a clone has been created
1604 already. */
1605 tree id = simd_clone_mangle (node, clone);
1606 if (id == NULL_TREE)
1607 continue;
1608
1609 /* Only when we are sure we want to create the clone actually
1610 clone the function (or definitions) or create another
1611 extern FUNCTION_DECL (for prototypes without definitions). */
1612 struct cgraph_node *n = simd_clone_create (node);
1613 if (n == NULL)
1614 continue;
1615
1616 n->simdclone = clone;
1617 clone->origin = node;
1618 clone->next_clone = NULL;
1619 if (node->simd_clones == NULL)
1620 {
1621 clone->prev_clone = n;
1622 node->simd_clones = n;
1623 }
1624 else
1625 {
1626 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1627 clone->prev_clone->simdclone->next_clone = n;
1628 node->simd_clones->simdclone->prev_clone = n;
1629 }
1630 symtab->change_decl_assembler_name (n->decl, id);
1631 /* And finally adjust the return type, parameters and for
1632 definitions also function body. */
1633 if (node->definition)
1634 simd_clone_adjust (n);
1635 else
1636 {
1637 simd_clone_adjust_return_type (n);
1638 simd_clone_adjust_argument_types (n);
1639 }
1640 }
1641 }
1642 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1643 }
1644
1645 /* Entry point for IPA simd clone creation pass. */
1646
1647 static unsigned int
ipa_omp_simd_clone(void)1648 ipa_omp_simd_clone (void)
1649 {
1650 struct cgraph_node *node;
1651 FOR_EACH_FUNCTION (node)
1652 expand_simd_clones (node);
1653 return 0;
1654 }
1655
1656 namespace {
1657
1658 const pass_data pass_data_omp_simd_clone =
1659 {
1660 SIMPLE_IPA_PASS, /* type */
1661 "simdclone", /* name */
1662 OPTGROUP_NONE, /* optinfo_flags */
1663 TV_NONE, /* tv_id */
1664 ( PROP_ssa | PROP_cfg ), /* properties_required */
1665 0, /* properties_provided */
1666 0, /* properties_destroyed */
1667 0, /* todo_flags_start */
1668 0, /* todo_flags_finish */
1669 };
1670
1671 class pass_omp_simd_clone : public simple_ipa_opt_pass
1672 {
1673 public:
pass_omp_simd_clone(gcc::context * ctxt)1674 pass_omp_simd_clone(gcc::context *ctxt)
1675 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1676 {}
1677
1678 /* opt_pass methods: */
1679 virtual bool gate (function *);
execute(function *)1680 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1681 };
1682
1683 bool
gate(function *)1684 pass_omp_simd_clone::gate (function *)
1685 {
1686 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1687 }
1688
1689 } // anon namespace
1690
1691 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1692 make_pass_omp_simd_clone (gcc::context *ctxt)
1693 {
1694 return new pass_omp_simd_clone (ctxt);
1695 }
1696