1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2018 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Return vector of parameter types of function FNDECL. This uses
90 TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static vec<tree>
simd_clone_vector_of_formal_parm_types(tree fndecl)94 simd_clone_vector_of_formal_parm_types (tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
98 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
99 unsigned int i;
100 tree arg;
101 FOR_EACH_VEC_ELT (args, i, arg)
102 args[i] = TREE_TYPE (args[i]);
103 return args;
104 }
105
106 /* Given a simd function in NODE, extract the simd specific
107 information from the OMP clauses passed in CLAUSES, and return
108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
109 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
110 otherwise set to FALSE. */
111
112 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)113 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
114 bool *inbranch_specified)
115 {
116 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
117 tree t;
118 int n;
119 *inbranch_specified = false;
120
121 n = args.length ();
122 if (n > 0 && args.last () == void_type_node)
123 n--;
124
125 /* Allocate one more than needed just in case this is an in-branch
126 clone which will require a mask argument. */
127 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
128 clone_info->nargs = n;
129
130 if (!clauses)
131 goto out;
132
133 clauses = TREE_VALUE (clauses);
134 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
135 goto out;
136
137 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
138 {
139 switch (OMP_CLAUSE_CODE (t))
140 {
141 case OMP_CLAUSE_INBRANCH:
142 clone_info->inbranch = 1;
143 *inbranch_specified = true;
144 break;
145 case OMP_CLAUSE_NOTINBRANCH:
146 clone_info->inbranch = 0;
147 *inbranch_specified = true;
148 break;
149 case OMP_CLAUSE_SIMDLEN:
150 clone_info->simdlen
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
152 break;
153 case OMP_CLAUSE_LINEAR:
154 {
155 tree decl = OMP_CLAUSE_DECL (t);
156 tree step = OMP_CLAUSE_LINEAR_STEP (t);
157 int argno = TREE_INT_CST_LOW (decl);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
159 {
160 enum cgraph_simd_clone_arg_type arg_type;
161 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
162 switch (OMP_CLAUSE_LINEAR_KIND (t))
163 {
164 case OMP_CLAUSE_LINEAR_REF:
165 arg_type
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
167 break;
168 case OMP_CLAUSE_LINEAR_UVAL:
169 arg_type
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
171 break;
172 case OMP_CLAUSE_LINEAR_VAL:
173 case OMP_CLAUSE_LINEAR_DEFAULT:
174 arg_type
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
176 break;
177 default:
178 gcc_unreachable ();
179 }
180 else
181 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
182 clone_info->args[argno].arg_type = arg_type;
183 clone_info->args[argno].linear_step = tree_to_shwi (step);
184 gcc_assert (clone_info->args[argno].linear_step >= 0
185 && clone_info->args[argno].linear_step < n);
186 }
187 else
188 {
189 if (POINTER_TYPE_P (args[argno]))
190 step = fold_convert (ssizetype, step);
191 if (!tree_fits_shwi_p (step))
192 {
193 warning_at (OMP_CLAUSE_LOCATION (t), 0,
194 "ignoring large linear step");
195 args.release ();
196 return NULL;
197 }
198 else if (integer_zerop (step))
199 {
200 warning_at (OMP_CLAUSE_LOCATION (t), 0,
201 "ignoring zero linear step");
202 args.release ();
203 return NULL;
204 }
205 else
206 {
207 enum cgraph_simd_clone_arg_type arg_type;
208 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
209 switch (OMP_CLAUSE_LINEAR_KIND (t))
210 {
211 case OMP_CLAUSE_LINEAR_REF:
212 arg_type
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
214 break;
215 case OMP_CLAUSE_LINEAR_UVAL:
216 arg_type
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
218 break;
219 case OMP_CLAUSE_LINEAR_VAL:
220 case OMP_CLAUSE_LINEAR_DEFAULT:
221 arg_type
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
223 break;
224 default:
225 gcc_unreachable ();
226 }
227 else
228 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
229 clone_info->args[argno].arg_type = arg_type;
230 clone_info->args[argno].linear_step = tree_to_shwi (step);
231 }
232 }
233 break;
234 }
235 case OMP_CLAUSE_UNIFORM:
236 {
237 tree decl = OMP_CLAUSE_DECL (t);
238 int argno = tree_to_uhwi (decl);
239 clone_info->args[argno].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM;
241 break;
242 }
243 case OMP_CLAUSE_ALIGNED:
244 {
245 /* Ignore aligned (x) for declare simd, for the ABI we really
246 need an alignment specified. */
247 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
248 break;
249 tree decl = OMP_CLAUSE_DECL (t);
250 int argno = tree_to_uhwi (decl);
251 clone_info->args[argno].alignment
252 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
253 break;
254 }
255 default:
256 break;
257 }
258 }
259
260 out:
261 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
262 {
263 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
264 "ignoring %<#pragma omp declare simd%> on function "
265 "with %<_Atomic%> qualified return type");
266 args.release ();
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 args.release ();
282 return clone_info;
283 }
284
285 /* Given a SIMD clone in NODE, calculate the characteristic data
286 type and return the coresponding type. The characteristic data
287 type is computed as described in the Intel Vector ABI. */
288
289 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)290 simd_clone_compute_base_data_type (struct cgraph_node *node,
291 struct cgraph_simd_clone *clone_info)
292 {
293 tree type = integer_type_node;
294 tree fndecl = node->decl;
295
296 /* a) For non-void function, the characteristic data type is the
297 return type. */
298 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
299 type = TREE_TYPE (TREE_TYPE (fndecl));
300
301 /* b) If the function has any non-uniform, non-linear parameters,
302 then the characteristic data type is the type of the first
303 such parameter. */
304 else
305 {
306 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 map.release ();
314 }
315
316 /* c) If the characteristic data type determined by a) or b) above
317 is struct, union, or class type which is pass-by-value (except
318 for the type that maps to the built-in complex data type), the
319 characteristic data type is int. */
320 if (RECORD_OR_UNION_TYPE_P (type)
321 && !aggregate_value_p (type, NULL)
322 && TREE_CODE (type) != COMPLEX_TYPE)
323 return integer_type_node;
324
325 /* d) If none of the above three classes is applicable, the
326 characteristic data type is int. */
327
328 return type;
329
330 /* e) For Intel Xeon Phi native and offload compilation, if the
331 resulting characteristic data type is 8-bit or 16-bit integer
332 data type, the characteristic data type is int. */
333 /* Well, we don't handle Xeon Phi yet. */
334 }
335
336 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)337 simd_clone_mangle (struct cgraph_node *node,
338 struct cgraph_simd_clone *clone_info)
339 {
340 char vecsize_mangle = clone_info->vecsize_mangle;
341 char mask = clone_info->inbranch ? 'M' : 'N';
342 unsigned int simdlen = clone_info->simdlen;
343 unsigned int n;
344 pretty_printer pp;
345
346 gcc_assert (vecsize_mangle && simdlen);
347
348 pp_string (&pp, "_ZGV");
349 pp_character (&pp, vecsize_mangle);
350 pp_character (&pp, mask);
351 pp_decimal_int (&pp, simdlen);
352
353 for (n = 0; n < clone_info->nargs; ++n)
354 {
355 struct cgraph_simd_clone_arg arg = clone_info->args[n];
356
357 switch (arg.arg_type)
358 {
359 case SIMD_CLONE_ARG_TYPE_UNIFORM:
360 pp_character (&pp, 'u');
361 break;
362 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
363 pp_character (&pp, 'l');
364 goto mangle_linear;
365 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
366 pp_character (&pp, 'R');
367 goto mangle_linear;
368 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
369 pp_character (&pp, 'L');
370 goto mangle_linear;
371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
372 pp_character (&pp, 'U');
373 goto mangle_linear;
374 mangle_linear:
375 gcc_assert (arg.linear_step != 0);
376 if (arg.linear_step > 1)
377 pp_unsigned_wide_integer (&pp, arg.linear_step);
378 else if (arg.linear_step < 0)
379 {
380 pp_character (&pp, 'n');
381 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
382 arg.linear_step));
383 }
384 break;
385 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
386 pp_string (&pp, "ls");
387 pp_unsigned_wide_integer (&pp, arg.linear_step);
388 break;
389 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
390 pp_string (&pp, "Rs");
391 pp_unsigned_wide_integer (&pp, arg.linear_step);
392 break;
393 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
394 pp_string (&pp, "Ls");
395 pp_unsigned_wide_integer (&pp, arg.linear_step);
396 break;
397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
398 pp_string (&pp, "Us");
399 pp_unsigned_wide_integer (&pp, arg.linear_step);
400 break;
401 default:
402 pp_character (&pp, 'v');
403 }
404 if (arg.alignment)
405 {
406 pp_character (&pp, 'a');
407 pp_decimal_int (&pp, arg.alignment);
408 }
409 }
410
411 pp_underscore (&pp);
412 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
413 if (*str == '*')
414 ++str;
415 pp_string (&pp, str);
416 str = pp_formatted_text (&pp);
417
418 /* If there already is a SIMD clone with the same mangled name, don't
419 add another one. This can happen e.g. for
420 #pragma omp declare simd
421 #pragma omp declare simd simdlen(8)
422 int foo (int, int);
423 if the simdlen is assumed to be 8 for the first one, etc. */
424 for (struct cgraph_node *clone = node->simd_clones; clone;
425 clone = clone->simdclone->next_clone)
426 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
427 return NULL_TREE;
428
429 return get_identifier (str);
430 }
431
432 /* Create a simd clone of OLD_NODE and return it. */
433
434 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)435 simd_clone_create (struct cgraph_node *old_node)
436 {
437 struct cgraph_node *new_node;
438 if (old_node->definition)
439 {
440 if (!old_node->has_gimple_body_p ())
441 return NULL;
442 old_node->get_body ();
443 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
444 false, NULL, NULL,
445 "simdclone");
446 }
447 else
448 {
449 tree old_decl = old_node->decl;
450 tree new_decl = copy_node (old_node->decl);
451 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
452 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
453 SET_DECL_RTL (new_decl, NULL);
454 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
455 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
456 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
457 if (old_node->in_other_partition)
458 new_node->in_other_partition = 1;
459 }
460 if (new_node == NULL)
461 return new_node;
462
463 DECL_BUILT_IN_CLASS (new_node->decl) = NOT_BUILT_IN;
464 DECL_FUNCTION_CODE (new_node->decl) = (enum built_in_function) 0;
465 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
466 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
467 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
468 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
469 DECL_VISIBILITY_SPECIFIED (new_node->decl)
470 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
471 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
472 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
473 if (DECL_ONE_ONLY (old_node->decl))
474 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
475
476 /* The method cgraph_version_clone_with_body () will force the new
477 symbol local. Undo this, and inherit external visibility from
478 the old node. */
479 new_node->local.local = old_node->local.local;
480 new_node->externally_visible = old_node->externally_visible;
481
482 return new_node;
483 }
484
485 /* Adjust the return type of the given function to its appropriate
486 vector counterpart. Returns a simd array to be used throughout the
487 function as a return value. */
488
489 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)490 simd_clone_adjust_return_type (struct cgraph_node *node)
491 {
492 tree fndecl = node->decl;
493 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
494 unsigned int veclen;
495 tree t;
496
497 /* Adjust the function return type. */
498 if (orig_rettype == void_type_node)
499 return NULL_TREE;
500 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
501 t = TREE_TYPE (TREE_TYPE (fndecl));
502 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
503 veclen = node->simdclone->vecsize_int;
504 else
505 veclen = node->simdclone->vecsize_float;
506 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
507 if (veclen > node->simdclone->simdlen)
508 veclen = node->simdclone->simdlen;
509 if (POINTER_TYPE_P (t))
510 t = pointer_sized_int_node;
511 if (veclen == node->simdclone->simdlen)
512 t = build_vector_type (t, node->simdclone->simdlen);
513 else
514 {
515 t = build_vector_type (t, veclen);
516 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
517 }
518 TREE_TYPE (TREE_TYPE (fndecl)) = t;
519 if (!node->definition)
520 return NULL_TREE;
521
522 t = DECL_RESULT (fndecl);
523 /* Adjust the DECL_RESULT. */
524 gcc_assert (TREE_TYPE (t) != void_type_node);
525 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
526 relayout_decl (t);
527
528 tree atype = build_array_type_nelts (orig_rettype,
529 node->simdclone->simdlen);
530 if (veclen != node->simdclone->simdlen)
531 return build1 (VIEW_CONVERT_EXPR, atype, t);
532
533 /* Set up a SIMD array to use as the return value. */
534 tree retval = create_tmp_var_raw (atype, "retval");
535 gimple_add_tmp_var (retval);
536 return retval;
537 }
538
539 /* Each vector argument has a corresponding array to be used locally
540 as part of the eventual loop. Create such temporary array and
541 return it.
542
543 PREFIX is the prefix to be used for the temporary.
544
545 TYPE is the inner element type.
546
547 SIMDLEN is the number of elements. */
548
549 static tree
create_tmp_simd_array(const char * prefix,tree type,int simdlen)550 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
551 {
552 tree atype = build_array_type_nelts (type, simdlen);
553 tree avar = create_tmp_var_raw (atype, prefix);
554 gimple_add_tmp_var (avar);
555 return avar;
556 }
557
558 /* Modify the function argument types to their corresponding vector
559 counterparts if appropriate. Also, create one array for each simd
560 argument to be used locally when using the function arguments as
561 part of the loop.
562
563 NODE is the function whose arguments are to be adjusted.
564
565 Returns an adjustment vector that will be filled describing how the
566 argument types will be adjusted. */
567
568 static ipa_parm_adjustment_vec
simd_clone_adjust_argument_types(struct cgraph_node * node)569 simd_clone_adjust_argument_types (struct cgraph_node *node)
570 {
571 vec<tree> args;
572 ipa_parm_adjustment_vec adjustments;
573
574 if (node->definition)
575 args = ipa_get_vector_of_formal_parms (node->decl);
576 else
577 args = simd_clone_vector_of_formal_parm_types (node->decl);
578 adjustments.create (args.length ());
579 unsigned i, j, veclen;
580 struct ipa_parm_adjustment adj;
581 struct cgraph_simd_clone *sc = node->simdclone;
582
583 for (i = 0; i < sc->nargs; ++i)
584 {
585 memset (&adj, 0, sizeof (adj));
586 tree parm = args[i];
587 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
588 adj.base_index = i;
589 adj.base = parm;
590
591 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
592 sc->args[i].orig_type = parm_type;
593
594 switch (sc->args[i].arg_type)
595 {
596 default:
597 /* No adjustment necessary for scalar arguments. */
598 adj.op = IPA_PARM_OP_COPY;
599 break;
600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
602 if (node->definition)
603 sc->args[i].simd_array
604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
605 TREE_TYPE (parm_type),
606 sc->simdlen);
607 adj.op = IPA_PARM_OP_COPY;
608 break;
609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
611 case SIMD_CLONE_ARG_TYPE_VECTOR:
612 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
613 veclen = sc->vecsize_int;
614 else
615 veclen = sc->vecsize_float;
616 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
617 if (veclen > sc->simdlen)
618 veclen = sc->simdlen;
619 adj.arg_prefix = "simd";
620 if (POINTER_TYPE_P (parm_type))
621 adj.type = build_vector_type (pointer_sized_int_node, veclen);
622 else
623 adj.type = build_vector_type (parm_type, veclen);
624 sc->args[i].vector_type = adj.type;
625 for (j = veclen; j < sc->simdlen; j += veclen)
626 {
627 adjustments.safe_push (adj);
628 if (j == veclen)
629 {
630 memset (&adj, 0, sizeof (adj));
631 adj.op = IPA_PARM_OP_NEW;
632 adj.arg_prefix = "simd";
633 adj.base_index = i;
634 adj.type = sc->args[i].vector_type;
635 }
636 }
637
638 if (node->definition)
639 sc->args[i].simd_array
640 = create_tmp_simd_array (DECL_NAME (parm)
641 ? IDENTIFIER_POINTER (DECL_NAME (parm))
642 : NULL, parm_type, sc->simdlen);
643 }
644 adjustments.safe_push (adj);
645 }
646
647 if (sc->inbranch)
648 {
649 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
650
651 memset (&adj, 0, sizeof (adj));
652 adj.op = IPA_PARM_OP_NEW;
653 adj.arg_prefix = "mask";
654
655 adj.base_index = i;
656 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
657 veclen = sc->vecsize_int;
658 else
659 veclen = sc->vecsize_float;
660 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
661 if (veclen > sc->simdlen)
662 veclen = sc->simdlen;
663 if (sc->mask_mode != VOIDmode)
664 adj.type
665 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
666 else if (POINTER_TYPE_P (base_type))
667 adj.type = build_vector_type (pointer_sized_int_node, veclen);
668 else
669 adj.type = build_vector_type (base_type, veclen);
670 adjustments.safe_push (adj);
671
672 for (j = veclen; j < sc->simdlen; j += veclen)
673 adjustments.safe_push (adj);
674
675 /* We have previously allocated one extra entry for the mask. Use
676 it and fill it. */
677 sc->nargs++;
678 if (sc->mask_mode != VOIDmode)
679 base_type = boolean_type_node;
680 if (node->definition)
681 {
682 sc->args[i].orig_arg
683 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
684 if (sc->mask_mode == VOIDmode)
685 sc->args[i].simd_array
686 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
687 else if (veclen < sc->simdlen)
688 sc->args[i].simd_array
689 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
690 else
691 sc->args[i].simd_array = NULL_TREE;
692 }
693 sc->args[i].orig_type = base_type;
694 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
695 }
696
697 if (node->definition)
698 ipa_modify_formal_parameters (node->decl, adjustments);
699 else
700 {
701 tree new_arg_types = NULL_TREE, new_reversed;
702 bool last_parm_void = false;
703 if (args.length () > 0 && args.last () == void_type_node)
704 last_parm_void = true;
705
706 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
707 j = adjustments.length ();
708 for (i = 0; i < j; i++)
709 {
710 struct ipa_parm_adjustment *adj = &adjustments[i];
711 tree ptype;
712 if (adj->op == IPA_PARM_OP_COPY)
713 ptype = args[adj->base_index];
714 else
715 ptype = adj->type;
716 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
717 }
718 new_reversed = nreverse (new_arg_types);
719 if (last_parm_void)
720 {
721 if (new_reversed)
722 TREE_CHAIN (new_arg_types) = void_list_node;
723 else
724 new_reversed = void_list_node;
725 }
726
727 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
728 TYPE_ARG_TYPES (new_type) = new_reversed;
729 TREE_TYPE (node->decl) = new_type;
730
731 adjustments.release ();
732 }
733 args.release ();
734 return adjustments;
735 }
736
737 /* Initialize and copy the function arguments in NODE to their
738 corresponding local simd arrays. Returns a fresh gimple_seq with
739 the instruction sequence generated. */
740
741 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments)742 simd_clone_init_simd_arrays (struct cgraph_node *node,
743 ipa_parm_adjustment_vec adjustments)
744 {
745 gimple_seq seq = NULL;
746 unsigned i = 0, j = 0, k;
747
748 for (tree arg = DECL_ARGUMENTS (node->decl);
749 arg;
750 arg = DECL_CHAIN (arg), i++, j++)
751 {
752 if (adjustments[j].op == IPA_PARM_OP_COPY
753 || POINTER_TYPE_P (TREE_TYPE (arg)))
754 continue;
755
756 node->simdclone->args[i].vector_arg = arg;
757
758 tree array = node->simdclone->args[i].simd_array;
759 if (node->simdclone->mask_mode != VOIDmode
760 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
761 {
762 if (array == NULL_TREE)
763 continue;
764 unsigned int l
765 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
766 for (k = 0; k <= l; k++)
767 {
768 if (k)
769 {
770 arg = DECL_CHAIN (arg);
771 j++;
772 }
773 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
774 array, size_int (k), NULL, NULL);
775 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
776 gimplify_and_add (t, &seq);
777 }
778 continue;
779 }
780 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen)
781 {
782 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
783 tree ptr = build_fold_addr_expr (array);
784 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
785 build_int_cst (ptype, 0));
786 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
787 gimplify_and_add (t, &seq);
788 }
789 else
790 {
791 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
792 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
793 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
794 {
795 tree ptr = build_fold_addr_expr (array);
796 int elemsize;
797 if (k)
798 {
799 arg = DECL_CHAIN (arg);
800 j++;
801 }
802 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
803 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
804 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
805 build_int_cst (ptype, k * elemsize));
806 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
807 gimplify_and_add (t, &seq);
808 }
809 }
810 }
811 return seq;
812 }
813
814 /* Callback info for ipa_simd_modify_stmt_ops below. */
815
816 struct modify_stmt_info {
817 ipa_parm_adjustment_vec adjustments;
818 gimple *stmt;
819 /* True if the parent statement was modified by
820 ipa_simd_modify_stmt_ops. */
821 bool modified;
822 };
823
824 /* Callback for walk_gimple_op.
825
826 Adjust operands from a given statement as specified in the
827 adjustments vector in the callback data. */
828
829 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)830 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
831 {
832 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
833 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
834 tree *orig_tp = tp;
835 if (TREE_CODE (*tp) == ADDR_EXPR)
836 tp = &TREE_OPERAND (*tp, 0);
837 struct ipa_parm_adjustment *cand = NULL;
838 if (TREE_CODE (*tp) == PARM_DECL)
839 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
840 else
841 {
842 if (TYPE_P (*tp))
843 *walk_subtrees = 0;
844 }
845
846 tree repl = NULL_TREE;
847 if (cand)
848 repl = unshare_expr (cand->new_decl);
849 else
850 {
851 if (tp != orig_tp)
852 {
853 *walk_subtrees = 0;
854 bool modified = info->modified;
855 info->modified = false;
856 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
857 if (!info->modified)
858 {
859 info->modified = modified;
860 return NULL_TREE;
861 }
862 info->modified = modified;
863 repl = *tp;
864 }
865 else
866 return NULL_TREE;
867 }
868
869 if (tp != orig_tp)
870 {
871 if (gimple_code (info->stmt) == GIMPLE_PHI
872 && cand
873 && TREE_CODE (*orig_tp) == ADDR_EXPR
874 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
875 && cand->alias_ptr_type)
876 {
877 gcc_assert (TREE_CODE (cand->alias_ptr_type) == SSA_NAME);
878 *orig_tp = cand->alias_ptr_type;
879 info->modified = true;
880 return NULL_TREE;
881 }
882
883 repl = build_fold_addr_expr (repl);
884 gimple *stmt;
885 if (is_gimple_debug (info->stmt))
886 {
887 tree vexpr = make_node (DEBUG_EXPR_DECL);
888 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
889 DECL_ARTIFICIAL (vexpr) = 1;
890 TREE_TYPE (vexpr) = TREE_TYPE (repl);
891 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
892 repl = vexpr;
893 }
894 else
895 {
896 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
897 repl = gimple_assign_lhs (stmt);
898 }
899 gimple_stmt_iterator gsi;
900 if (gimple_code (info->stmt) == GIMPLE_PHI)
901 {
902 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
903 /* Cache SSA_NAME for next time. */
904 if (cand
905 && TREE_CODE (*orig_tp) == ADDR_EXPR
906 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
907 cand->alias_ptr_type = repl;
908 }
909 else
910 gsi = gsi_for_stmt (info->stmt);
911 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
912 *orig_tp = repl;
913 }
914 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
915 {
916 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
917 *tp = vce;
918 }
919 else
920 *tp = repl;
921
922 info->modified = true;
923 return NULL_TREE;
924 }
925
926 /* Traverse the function body and perform all modifications as
927 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
928 modified such that the replacement/reduction value will now be an
929 offset into the corresponding simd_array.
930
931 This function will replace all function argument uses with their
932 corresponding simd array elements, and ajust the return values
933 accordingly. */
934
935 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments,tree retval_array,tree iter)936 ipa_simd_modify_function_body (struct cgraph_node *node,
937 ipa_parm_adjustment_vec adjustments,
938 tree retval_array, tree iter)
939 {
940 basic_block bb;
941 unsigned int i, j, l;
942
943 /* Re-use the adjustments array, but this time use it to replace
944 every function argument use to an offset into the corresponding
945 simd_array. */
946 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
947 {
948 if (!node->simdclone->args[i].vector_arg)
949 continue;
950
951 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
952 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
953 adjustments[j].new_decl
954 = build4 (ARRAY_REF,
955 basetype,
956 node->simdclone->args[i].simd_array,
957 iter,
958 NULL_TREE, NULL_TREE);
959 if (adjustments[j].op == IPA_PARM_OP_NONE
960 && simd_clone_subparts (vectype) < node->simdclone->simdlen)
961 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1;
962 }
963
964 l = adjustments.length ();
965 tree name;
966
967 FOR_EACH_SSA_NAME (i, name, cfun)
968 {
969 if (SSA_NAME_VAR (name)
970 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
971 {
972 for (j = 0; j < l; j++)
973 if (SSA_NAME_VAR (name) == adjustments[j].base
974 && adjustments[j].new_decl)
975 {
976 tree base_var;
977 if (adjustments[j].new_ssa_base == NULL_TREE)
978 {
979 base_var
980 = copy_var_decl (adjustments[j].base,
981 DECL_NAME (adjustments[j].base),
982 TREE_TYPE (adjustments[j].base));
983 adjustments[j].new_ssa_base = base_var;
984 }
985 else
986 base_var = adjustments[j].new_ssa_base;
987 if (SSA_NAME_IS_DEFAULT_DEF (name))
988 {
989 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
990 gimple_stmt_iterator gsi = gsi_after_labels (bb);
991 tree new_decl = unshare_expr (adjustments[j].new_decl);
992 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
993 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
994 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
995 gimple *stmt = gimple_build_assign (name, new_decl);
996 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
997 }
998 else
999 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1000 }
1001 }
1002 }
1003
1004 struct modify_stmt_info info;
1005 info.adjustments = adjustments;
1006
1007 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1008 {
1009 gimple_stmt_iterator gsi;
1010
1011 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1012 {
1013 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1014 int i, n = gimple_phi_num_args (phi);
1015 info.stmt = phi;
1016 struct walk_stmt_info wi;
1017 memset (&wi, 0, sizeof (wi));
1018 info.modified = false;
1019 wi.info = &info;
1020 for (i = 0; i < n; ++i)
1021 {
1022 int walk_subtrees = 1;
1023 tree arg = gimple_phi_arg_def (phi, i);
1024 tree op = arg;
1025 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1026 if (op != arg)
1027 {
1028 SET_PHI_ARG_DEF (phi, i, op);
1029 gcc_assert (TREE_CODE (op) == SSA_NAME);
1030 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1031 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1032 }
1033 }
1034 }
1035
1036 gsi = gsi_start_bb (bb);
1037 while (!gsi_end_p (gsi))
1038 {
1039 gimple *stmt = gsi_stmt (gsi);
1040 info.stmt = stmt;
1041 struct walk_stmt_info wi;
1042
1043 memset (&wi, 0, sizeof (wi));
1044 info.modified = false;
1045 wi.info = &info;
1046 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1047
1048 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1049 {
1050 tree retval = gimple_return_retval (return_stmt);
1051 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1052 e->flags |= EDGE_FALLTHRU;
1053 if (!retval)
1054 {
1055 gsi_remove (&gsi, true);
1056 continue;
1057 }
1058
1059 /* Replace `return foo' with `retval_array[iter] = foo'. */
1060 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1061 retval_array, iter, NULL, NULL);
1062 stmt = gimple_build_assign (ref, retval);
1063 gsi_replace (&gsi, stmt, true);
1064 info.modified = true;
1065 }
1066
1067 if (info.modified)
1068 {
1069 update_stmt (stmt);
1070 /* If the above changed the var of a debug bind into something
1071 different, remove the debug stmt. We could also for all the
1072 replaced parameters add VAR_DECLs for debug info purposes,
1073 add debug stmts for those to be the simd array accesses and
1074 replace debug stmt var operand with that var. Debugging of
1075 vectorized loops doesn't work too well, so don't bother for
1076 now. */
1077 if ((gimple_debug_bind_p (stmt)
1078 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1079 || (gimple_debug_source_bind_p (stmt)
1080 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1081 {
1082 gsi_remove (&gsi, true);
1083 continue;
1084 }
1085 if (maybe_clean_eh_stmt (stmt))
1086 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1087 }
1088 gsi_next (&gsi);
1089 }
1090 }
1091 }
1092
1093 /* Helper function of simd_clone_adjust, return linear step addend
1094 of Ith argument. */
1095
1096 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1097 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1098 tree addtype, basic_block entry_bb)
1099 {
1100 tree ptype = NULL_TREE;
1101 switch (node->simdclone->args[i].arg_type)
1102 {
1103 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1104 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1105 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1106 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1107 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1108 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1109 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1110 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1111 break;
1112 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1113 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1114 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1115 break;
1116 default:
1117 gcc_unreachable ();
1118 }
1119
1120 unsigned int idx = node->simdclone->args[i].linear_step;
1121 tree arg = node->simdclone->args[idx].orig_arg;
1122 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1123 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1124 gimple *g;
1125 tree ret;
1126 if (is_gimple_reg (arg))
1127 ret = get_or_create_ssa_default_def (cfun, arg);
1128 else
1129 {
1130 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1131 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1132 ret = gimple_assign_lhs (g);
1133 }
1134 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1135 {
1136 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1137 build_simple_mem_ref (ret));
1138 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1139 ret = gimple_assign_lhs (g);
1140 }
1141 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1142 {
1143 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1144 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1145 ret = gimple_assign_lhs (g);
1146 }
1147 if (POINTER_TYPE_P (ptype))
1148 {
1149 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1150 if (size && TREE_CODE (size) == INTEGER_CST)
1151 {
1152 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1153 ret, fold_convert (addtype, size));
1154 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1155 ret = gimple_assign_lhs (g);
1156 }
1157 }
1158 return ret;
1159 }
1160
1161 /* Adjust the argument types in NODE to their appropriate vector
1162 counterparts. */
1163
1164 static void
simd_clone_adjust(struct cgraph_node * node)1165 simd_clone_adjust (struct cgraph_node *node)
1166 {
1167 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1168
1169 targetm.simd_clone.adjust (node);
1170
1171 tree retval = simd_clone_adjust_return_type (node);
1172 ipa_parm_adjustment_vec adjustments
1173 = simd_clone_adjust_argument_types (node);
1174
1175 push_gimplify_context ();
1176
1177 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1178
1179 /* Adjust all uses of vector arguments accordingly. Adjust all
1180 return values accordingly. */
1181 tree iter = create_tmp_var (unsigned_type_node, "iter");
1182 tree iter1 = make_ssa_name (iter);
1183 tree iter2 = NULL_TREE;
1184 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1185 adjustments.release ();
1186
1187 /* Initialize the iteration variable. */
1188 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1189 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1190 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1191 /* Insert the SIMD array and iv initialization at function
1192 entry. */
1193 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1194
1195 pop_gimplify_context (NULL);
1196
1197 gimple *g;
1198 basic_block incr_bb = NULL;
1199 struct loop *loop = NULL;
1200
1201 /* Create a new BB right before the original exit BB, to hold the
1202 iteration increment and the condition/branch. */
1203 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1204 {
1205 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1206 incr_bb = create_empty_bb (orig_exit);
1207 incr_bb->count = profile_count::zero ();
1208 add_bb_to_loop (incr_bb, body_bb->loop_father);
1209 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1210 {
1211 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1212 redirect_edge_succ (e, incr_bb);
1213 incr_bb->count += e->count ();
1214 }
1215 }
1216 else if (node->simdclone->inbranch)
1217 {
1218 incr_bb = create_empty_bb (entry_bb);
1219 incr_bb->count = profile_count::zero ();
1220 add_bb_to_loop (incr_bb, body_bb->loop_father);
1221 }
1222
1223 if (incr_bb)
1224 {
1225 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1226 gsi = gsi_last_bb (incr_bb);
1227 iter2 = make_ssa_name (iter);
1228 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1229 build_int_cst (unsigned_type_node, 1));
1230 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1231
1232 /* Mostly annotate the loop for the vectorizer (the rest is done
1233 below). */
1234 loop = alloc_loop ();
1235 cfun->has_force_vectorize_loops = true;
1236 loop->safelen = node->simdclone->simdlen;
1237 loop->force_vectorize = true;
1238 loop->header = body_bb;
1239 }
1240
1241 /* Branch around the body if the mask applies. */
1242 if (node->simdclone->inbranch)
1243 {
1244 gsi = gsi_last_bb (loop->header);
1245 tree mask_array
1246 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1247 tree mask;
1248 if (node->simdclone->mask_mode != VOIDmode)
1249 {
1250 tree shift_cnt;
1251 if (mask_array == NULL_TREE)
1252 {
1253 tree arg = node->simdclone->args[node->simdclone->nargs
1254 - 1].vector_arg;
1255 mask = get_or_create_ssa_default_def (cfun, arg);
1256 shift_cnt = iter1;
1257 }
1258 else
1259 {
1260 tree maskt = TREE_TYPE (mask_array);
1261 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1262 c = node->simdclone->simdlen / (c + 1);
1263 int s = exact_log2 (c);
1264 gcc_assert (s > 0);
1265 c--;
1266 tree idx = make_ssa_name (TREE_TYPE (iter1));
1267 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1268 build_int_cst (NULL_TREE, s));
1269 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1270 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1271 tree aref = build4 (ARRAY_REF,
1272 TREE_TYPE (TREE_TYPE (mask_array)),
1273 mask_array, idx, NULL, NULL);
1274 g = gimple_build_assign (mask, aref);
1275 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1276 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1277 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1278 build_int_cst (TREE_TYPE (iter1), c));
1279 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1280 }
1281 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1282 RSHIFT_EXPR, mask, shift_cnt);
1283 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1284 mask = gimple_assign_lhs (g);
1285 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1286 BIT_AND_EXPR, mask,
1287 build_int_cst (TREE_TYPE (mask), 1));
1288 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1289 mask = gimple_assign_lhs (g);
1290 }
1291 else
1292 {
1293 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1294 tree aref = build4 (ARRAY_REF,
1295 TREE_TYPE (TREE_TYPE (mask_array)),
1296 mask_array, iter1, NULL, NULL);
1297 g = gimple_build_assign (mask, aref);
1298 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1299 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1300 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1301 {
1302 aref = build1 (VIEW_CONVERT_EXPR,
1303 build_nonstandard_integer_type (bitsize, 0),
1304 mask);
1305 mask = make_ssa_name (TREE_TYPE (aref));
1306 g = gimple_build_assign (mask, aref);
1307 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1308 }
1309 }
1310
1311 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1312 NULL, NULL);
1313 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1314 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1315 e->probability = profile_probability::unlikely ().guessed ();
1316 incr_bb->count += e->count ();
1317 edge fallthru = FALLTHRU_EDGE (loop->header);
1318 fallthru->flags = EDGE_FALSE_VALUE;
1319 fallthru->probability = profile_probability::likely ().guessed ();
1320 }
1321
1322 basic_block latch_bb = NULL;
1323 basic_block new_exit_bb = NULL;
1324
1325 /* Generate the condition. */
1326 if (incr_bb)
1327 {
1328 gsi = gsi_last_bb (incr_bb);
1329 g = gimple_build_cond (LT_EXPR, iter2,
1330 build_int_cst (unsigned_type_node,
1331 node->simdclone->simdlen),
1332 NULL, NULL);
1333 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1334 edge e = split_block (incr_bb, gsi_stmt (gsi));
1335 latch_bb = e->dest;
1336 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1337 loop->latch = latch_bb;
1338
1339 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1340
1341 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1342
1343 /* FIXME: Do we need to distribute probabilities for the conditional? */
1344 new_e->probability = profile_probability::guessed_never ();
1345 /* The successor of incr_bb is already pointing to latch_bb; just
1346 change the flags.
1347 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1348 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1349 }
1350
1351 gphi *phi = create_phi_node (iter1, body_bb);
1352 edge preheader_edge = find_edge (entry_bb, body_bb);
1353 edge latch_edge = NULL;
1354 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1355 UNKNOWN_LOCATION);
1356 if (incr_bb)
1357 {
1358 latch_edge = single_succ_edge (latch_bb);
1359 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1360
1361 /* Generate the new return. */
1362 gsi = gsi_last_bb (new_exit_bb);
1363 if (retval
1364 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1365 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1366 retval = TREE_OPERAND (retval, 0);
1367 else if (retval)
1368 {
1369 retval = build1 (VIEW_CONVERT_EXPR,
1370 TREE_TYPE (TREE_TYPE (node->decl)),
1371 retval);
1372 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1373 false, GSI_CONTINUE_LINKING);
1374 }
1375 g = gimple_build_return (retval);
1376 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1377 }
1378
1379 /* Handle aligned clauses by replacing default defs of the aligned
1380 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1381 lhs. Handle linear by adding PHIs. */
1382 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1383 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1384 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1385 || !is_gimple_reg_type
1386 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1387 {
1388 tree orig_arg = node->simdclone->args[i].orig_arg;
1389 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1390 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1391 else
1392 {
1393 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1394 gimple_add_tmp_var (iter1);
1395 }
1396 gsi = gsi_after_labels (entry_bb);
1397 g = gimple_build_assign (iter1, orig_arg);
1398 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1399 gsi = gsi_after_labels (body_bb);
1400 g = gimple_build_assign (orig_arg, iter1);
1401 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1402 }
1403 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1404 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1405 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1406 == REFERENCE_TYPE
1407 && TREE_ADDRESSABLE
1408 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1409 {
1410 tree orig_arg = node->simdclone->args[i].orig_arg;
1411 tree def = ssa_default_def (cfun, orig_arg);
1412 if (def && !has_zero_uses (def))
1413 {
1414 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1415 gimple_add_tmp_var (iter1);
1416 gsi = gsi_after_labels (entry_bb);
1417 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1418 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1419 gsi = gsi_after_labels (body_bb);
1420 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1421 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1422 }
1423 }
1424 else if (node->simdclone->args[i].alignment
1425 && node->simdclone->args[i].arg_type
1426 == SIMD_CLONE_ARG_TYPE_UNIFORM
1427 && (node->simdclone->args[i].alignment
1428 & (node->simdclone->args[i].alignment - 1)) == 0
1429 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1430 == POINTER_TYPE)
1431 {
1432 unsigned int alignment = node->simdclone->args[i].alignment;
1433 tree orig_arg = node->simdclone->args[i].orig_arg;
1434 tree def = ssa_default_def (cfun, orig_arg);
1435 if (def && !has_zero_uses (def))
1436 {
1437 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1438 gimple_seq seq = NULL;
1439 bool need_cvt = false;
1440 gcall *call
1441 = gimple_build_call (fn, 2, def, size_int (alignment));
1442 g = call;
1443 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1444 ptr_type_node))
1445 need_cvt = true;
1446 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1447 gimple_call_set_lhs (g, t);
1448 gimple_seq_add_stmt_without_update (&seq, g);
1449 if (need_cvt)
1450 {
1451 t = make_ssa_name (orig_arg);
1452 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1453 gimple_seq_add_stmt_without_update (&seq, g);
1454 }
1455 gsi_insert_seq_on_edge_immediate
1456 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1457
1458 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1459 node->create_edge (cgraph_node::get_create (fn),
1460 call, entry_bb->count);
1461
1462 imm_use_iterator iter;
1463 use_operand_p use_p;
1464 gimple *use_stmt;
1465 tree repl = gimple_get_lhs (g);
1466 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1467 if (is_gimple_debug (use_stmt) || use_stmt == call)
1468 continue;
1469 else
1470 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1471 SET_USE (use_p, repl);
1472 }
1473 }
1474 else if ((node->simdclone->args[i].arg_type
1475 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1476 || (node->simdclone->args[i].arg_type
1477 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1478 || (node->simdclone->args[i].arg_type
1479 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1480 || (node->simdclone->args[i].arg_type
1481 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1482 {
1483 tree orig_arg = node->simdclone->args[i].orig_arg;
1484 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1485 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1486 tree def = NULL_TREE;
1487 if (TREE_ADDRESSABLE (orig_arg))
1488 {
1489 def = make_ssa_name (TREE_TYPE (orig_arg));
1490 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1491 if (incr_bb)
1492 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1493 gsi = gsi_after_labels (entry_bb);
1494 g = gimple_build_assign (def, orig_arg);
1495 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1496 }
1497 else
1498 {
1499 def = ssa_default_def (cfun, orig_arg);
1500 if (!def || has_zero_uses (def))
1501 def = NULL_TREE;
1502 else
1503 {
1504 iter1 = make_ssa_name (orig_arg);
1505 if (incr_bb)
1506 iter2 = make_ssa_name (orig_arg);
1507 }
1508 }
1509 if (def)
1510 {
1511 phi = create_phi_node (iter1, body_bb);
1512 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1513 if (incr_bb)
1514 {
1515 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1516 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1517 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1518 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1519 ? TREE_TYPE (orig_arg) : sizetype;
1520 tree addcst = simd_clone_linear_addend (node, i, addtype,
1521 entry_bb);
1522 gsi = gsi_last_bb (incr_bb);
1523 g = gimple_build_assign (iter2, code, iter1, addcst);
1524 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1525 }
1526
1527 imm_use_iterator iter;
1528 use_operand_p use_p;
1529 gimple *use_stmt;
1530 if (TREE_ADDRESSABLE (orig_arg))
1531 {
1532 gsi = gsi_after_labels (body_bb);
1533 g = gimple_build_assign (orig_arg, iter1);
1534 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1535 }
1536 else
1537 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1538 if (use_stmt == phi)
1539 continue;
1540 else
1541 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1542 SET_USE (use_p, iter1);
1543 }
1544 }
1545 else if (node->simdclone->args[i].arg_type
1546 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1547 || (node->simdclone->args[i].arg_type
1548 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1549 {
1550 tree orig_arg = node->simdclone->args[i].orig_arg;
1551 tree def = ssa_default_def (cfun, orig_arg);
1552 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1553 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1554 if (def && !has_zero_uses (def))
1555 {
1556 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1557 iter1 = make_ssa_name (orig_arg);
1558 if (incr_bb)
1559 iter2 = make_ssa_name (orig_arg);
1560 tree iter3 = make_ssa_name (rtype);
1561 tree iter4 = make_ssa_name (rtype);
1562 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1563 gsi = gsi_after_labels (entry_bb);
1564 gimple *load
1565 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1566 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1567
1568 tree array = node->simdclone->args[i].simd_array;
1569 TREE_ADDRESSABLE (array) = 1;
1570 tree ptr = build_fold_addr_expr (array);
1571 phi = create_phi_node (iter1, body_bb);
1572 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1573 if (incr_bb)
1574 {
1575 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1576 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1577 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1578 gsi = gsi_last_bb (incr_bb);
1579 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1580 }
1581
1582 phi = create_phi_node (iter4, body_bb);
1583 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1584 if (incr_bb)
1585 {
1586 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1587 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1588 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1589 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1590 ? TREE_TYPE (iter3) : sizetype;
1591 tree addcst = simd_clone_linear_addend (node, i, addtype,
1592 entry_bb);
1593 g = gimple_build_assign (iter5, code, iter4, addcst);
1594 gsi = gsi_last_bb (incr_bb);
1595 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1596 }
1597
1598 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1599 gsi = gsi_after_labels (body_bb);
1600 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1601
1602 imm_use_iterator iter;
1603 use_operand_p use_p;
1604 gimple *use_stmt;
1605 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1606 if (use_stmt == load)
1607 continue;
1608 else
1609 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1610 SET_USE (use_p, iter1);
1611
1612 if (!TYPE_READONLY (rtype) && incr_bb)
1613 {
1614 tree v = make_ssa_name (rtype);
1615 tree aref = build4 (ARRAY_REF, rtype, array,
1616 size_zero_node, NULL_TREE,
1617 NULL_TREE);
1618 gsi = gsi_after_labels (new_exit_bb);
1619 g = gimple_build_assign (v, aref);
1620 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1621 g = gimple_build_assign (build_simple_mem_ref (def), v);
1622 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1623 }
1624 }
1625 }
1626
1627 calculate_dominance_info (CDI_DOMINATORS);
1628 if (loop)
1629 add_loop (loop, loop->header->loop_father);
1630 update_ssa (TODO_update_ssa);
1631
1632 pop_cfun ();
1633 }
1634
1635 /* If the function in NODE is tagged as an elemental SIMD function,
1636 create the appropriate SIMD clones. */
1637
1638 void
expand_simd_clones(struct cgraph_node * node)1639 expand_simd_clones (struct cgraph_node *node)
1640 {
1641 tree attr = lookup_attribute ("omp declare simd",
1642 DECL_ATTRIBUTES (node->decl));
1643 if (attr == NULL_TREE
1644 || node->global.inlined_to
1645 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1646 return;
1647
1648 /* Ignore
1649 #pragma omp declare simd
1650 extern int foo ();
1651 in C, there we don't know the argument types at all. */
1652 if (!node->definition
1653 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1654 return;
1655
1656 /* Call this before creating clone_info, as it might ggc_collect. */
1657 if (node->definition && node->has_gimple_body_p ())
1658 node->get_body ();
1659
1660 do
1661 {
1662 /* Start with parsing the "omp declare simd" attribute(s). */
1663 bool inbranch_clause_specified;
1664 struct cgraph_simd_clone *clone_info
1665 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1666 &inbranch_clause_specified);
1667 if (clone_info == NULL)
1668 continue;
1669
1670 int orig_simdlen = clone_info->simdlen;
1671 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1672 /* The target can return 0 (no simd clones should be created),
1673 1 (just one ISA of simd clones should be created) or higher
1674 count of ISA variants. In that case, clone_info is initialized
1675 for the first ISA variant. */
1676 int count
1677 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1678 base_type, 0);
1679 if (count == 0)
1680 continue;
1681
1682 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1683 also create one inbranch and one !inbranch clone of it. */
1684 for (int i = 0; i < count * 2; i++)
1685 {
1686 struct cgraph_simd_clone *clone = clone_info;
1687 if (inbranch_clause_specified && (i & 1) != 0)
1688 continue;
1689
1690 if (i != 0)
1691 {
1692 clone = simd_clone_struct_alloc (clone_info->nargs
1693 + ((i & 1) != 0));
1694 simd_clone_struct_copy (clone, clone_info);
1695 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1696 and simd_clone_adjust_argument_types did to the first
1697 clone's info. */
1698 clone->nargs -= clone_info->inbranch;
1699 clone->simdlen = orig_simdlen;
1700 /* And call the target hook again to get the right ISA. */
1701 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1702 base_type,
1703 i / 2);
1704 if ((i & 1) != 0)
1705 clone->inbranch = 1;
1706 }
1707
1708 /* simd_clone_mangle might fail if such a clone has been created
1709 already. */
1710 tree id = simd_clone_mangle (node, clone);
1711 if (id == NULL_TREE)
1712 {
1713 if (i == 0)
1714 clone->nargs += clone->inbranch;
1715 continue;
1716 }
1717
1718 /* Only when we are sure we want to create the clone actually
1719 clone the function (or definitions) or create another
1720 extern FUNCTION_DECL (for prototypes without definitions). */
1721 struct cgraph_node *n = simd_clone_create (node);
1722 if (n == NULL)
1723 {
1724 if (i == 0)
1725 clone->nargs += clone->inbranch;
1726 continue;
1727 }
1728
1729 n->simdclone = clone;
1730 clone->origin = node;
1731 clone->next_clone = NULL;
1732 if (node->simd_clones == NULL)
1733 {
1734 clone->prev_clone = n;
1735 node->simd_clones = n;
1736 }
1737 else
1738 {
1739 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1740 clone->prev_clone->simdclone->next_clone = n;
1741 node->simd_clones->simdclone->prev_clone = n;
1742 }
1743 symtab->change_decl_assembler_name (n->decl, id);
1744 /* And finally adjust the return type, parameters and for
1745 definitions also function body. */
1746 if (node->definition)
1747 simd_clone_adjust (n);
1748 else
1749 {
1750 simd_clone_adjust_return_type (n);
1751 simd_clone_adjust_argument_types (n);
1752 }
1753 }
1754 }
1755 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1756 }
1757
1758 /* Entry point for IPA simd clone creation pass. */
1759
1760 static unsigned int
ipa_omp_simd_clone(void)1761 ipa_omp_simd_clone (void)
1762 {
1763 struct cgraph_node *node;
1764 FOR_EACH_FUNCTION (node)
1765 expand_simd_clones (node);
1766 return 0;
1767 }
1768
1769 namespace {
1770
1771 const pass_data pass_data_omp_simd_clone =
1772 {
1773 SIMPLE_IPA_PASS, /* type */
1774 "simdclone", /* name */
1775 OPTGROUP_OMP, /* optinfo_flags */
1776 TV_NONE, /* tv_id */
1777 ( PROP_ssa | PROP_cfg ), /* properties_required */
1778 0, /* properties_provided */
1779 0, /* properties_destroyed */
1780 0, /* todo_flags_start */
1781 0, /* todo_flags_finish */
1782 };
1783
1784 class pass_omp_simd_clone : public simple_ipa_opt_pass
1785 {
1786 public:
pass_omp_simd_clone(gcc::context * ctxt)1787 pass_omp_simd_clone(gcc::context *ctxt)
1788 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1789 {}
1790
1791 /* opt_pass methods: */
1792 virtual bool gate (function *);
execute(function *)1793 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1794 };
1795
1796 bool
gate(function *)1797 pass_omp_simd_clone::gate (function *)
1798 {
1799 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1800 }
1801
1802 } // anon namespace
1803
1804 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1805 make_pass_omp_simd_clone (gcc::context *ctxt)
1806 {
1807 return new pass_omp_simd_clone (ctxt);
1808 }
1809