1 /* Lowering pass for OMP directives. Converts OMP directives into explicit 2 calls to the runtime library (libgomp), data marshalling to implement data 3 sharing and copying clauses, offloading to accelerators, and more. 4 5 Contributed by Diego Novillo <dnovillo@redhat.com> 6 7 Copyright (C) 2005-2015 Free Software Foundation, Inc. 8 9 This file is part of GCC. 10 11 GCC is free software; you can redistribute it and/or modify it under 12 the terms of the GNU General Public License as published by the Free 13 Software Foundation; either version 3, or (at your option) any later 14 version. 15 16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 17 WARRANTY; without even the implied warranty of MERCHANTABILITY or 18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 19 for more details. 20 21 You should have received a copy of the GNU General Public License 22 along with GCC; see the file COPYING3. If not see 23 <http://www.gnu.org/licenses/>. */ 24 25 #include "config.h" 26 #include "system.h" 27 #include "coretypes.h" 28 #include "tm.h" 29 #include "hash-set.h" 30 #include "machmode.h" 31 #include "vec.h" 32 #include "double-int.h" 33 #include "input.h" 34 #include "alias.h" 35 #include "symtab.h" 36 #include "wide-int.h" 37 #include "inchash.h" 38 #include "tree.h" 39 #include "fold-const.h" 40 #include "stringpool.h" 41 #include "stor-layout.h" 42 #include "rtl.h" 43 #include "predict.h" 44 #include "hard-reg-set.h" 45 #include "function.h" 46 #include "dominance.h" 47 #include "cfg.h" 48 #include "cfganal.h" 49 #include "basic-block.h" 50 #include "tree-ssa-alias.h" 51 #include "internal-fn.h" 52 #include "gimple-fold.h" 53 #include "gimple-expr.h" 54 #include "is-a.h" 55 #include "gimple.h" 56 #include "gimplify.h" 57 #include "gimple-iterator.h" 58 #include "gimplify-me.h" 59 #include "gimple-walk.h" 60 #include "tree-iterator.h" 61 #include "tree-inline.h" 62 #include "langhooks.h" 63 #include "diagnostic-core.h" 64 #include "gimple-ssa.h" 65 #include "hash-map.h" 66 #include "plugin-api.h" 67 #include "ipa-ref.h" 68 #include "cgraph.h" 69 #include "tree-cfg.h" 70 #include "tree-phinodes.h" 71 #include "ssa-iterators.h" 72 #include "tree-ssanames.h" 73 #include "tree-into-ssa.h" 74 #include "hashtab.h" 75 #include "flags.h" 76 #include "statistics.h" 77 #include "real.h" 78 #include "fixed-value.h" 79 #include "insn-config.h" 80 #include "expmed.h" 81 #include "dojump.h" 82 #include "explow.h" 83 #include "calls.h" 84 #include "emit-rtl.h" 85 #include "varasm.h" 86 #include "stmt.h" 87 #include "expr.h" 88 #include "tree-dfa.h" 89 #include "tree-ssa.h" 90 #include "tree-pass.h" 91 #include "except.h" 92 #include "splay-tree.h" 93 #include "insn-codes.h" 94 #include "optabs.h" 95 #include "cfgloop.h" 96 #include "target.h" 97 #include "common/common-target.h" 98 #include "omp-low.h" 99 #include "gimple-low.h" 100 #include "tree-cfgcleanup.h" 101 #include "pretty-print.h" 102 #include "alloc-pool.h" 103 #include "symbol-summary.h" 104 #include "ipa-prop.h" 105 #include "tree-nested.h" 106 #include "tree-eh.h" 107 #include "cilk.h" 108 #include "context.h" 109 #include "lto-section-names.h" 110 #include "gomp-constants.h" 111 112 113 /* Lowering of OMP parallel and workshare constructs proceeds in two 114 phases. The first phase scans the function looking for OMP statements 115 and then for variables that must be replaced to satisfy data sharing 116 clauses. The second phase expands code for the constructs, as well as 117 re-gimplifying things when variables have been replaced with complex 118 expressions. 119 120 Final code generation is done by pass_expand_omp. The flowgraph is 121 scanned for regions which are then moved to a new 122 function, to be invoked by the thread library, or offloaded. */ 123 124 /* OMP region information. Every parallel and workshare 125 directive is enclosed between two markers, the OMP_* directive 126 and a corresponding OMP_RETURN statement. */ 127 128 struct omp_region 129 { 130 /* The enclosing region. */ 131 struct omp_region *outer; 132 133 /* First child region. */ 134 struct omp_region *inner; 135 136 /* Next peer region. */ 137 struct omp_region *next; 138 139 /* Block containing the omp directive as its last stmt. */ 140 basic_block entry; 141 142 /* Block containing the OMP_RETURN as its last stmt. */ 143 basic_block exit; 144 145 /* Block containing the OMP_CONTINUE as its last stmt. */ 146 basic_block cont; 147 148 /* If this is a combined parallel+workshare region, this is a list 149 of additional arguments needed by the combined parallel+workshare 150 library call. */ 151 vec<tree, va_gc> *ws_args; 152 153 /* The code for the omp directive of this region. */ 154 enum gimple_code type; 155 156 /* Schedule kind, only used for OMP_FOR type regions. */ 157 enum omp_clause_schedule_kind sched_kind; 158 159 /* True if this is a combined parallel+workshare region. */ 160 bool is_combined_parallel; 161 }; 162 163 /* Levels of parallelism as defined by OpenACC. Increasing numbers 164 correspond to deeper loop nesting levels. */ 165 #define MASK_GANG 1 166 #define MASK_WORKER 2 167 #define MASK_VECTOR 4 168 169 /* Context structure. Used to store information about each parallel 170 directive in the code. */ 171 172 typedef struct omp_context 173 { 174 /* This field must be at the beginning, as we do "inheritance": Some 175 callback functions for tree-inline.c (e.g., omp_copy_decl) 176 receive a copy_body_data pointer that is up-casted to an 177 omp_context pointer. */ 178 copy_body_data cb; 179 180 /* The tree of contexts corresponding to the encountered constructs. */ 181 struct omp_context *outer; 182 gimple stmt; 183 184 /* Map variables to fields in a structure that allows communication 185 between sending and receiving threads. */ 186 splay_tree field_map; 187 tree record_type; 188 tree sender_decl; 189 tree receiver_decl; 190 191 /* These are used just by task contexts, if task firstprivate fn is 192 needed. srecord_type is used to communicate from the thread 193 that encountered the task construct to task firstprivate fn, 194 record_type is allocated by GOMP_task, initialized by task firstprivate 195 fn and passed to the task body fn. */ 196 splay_tree sfield_map; 197 tree srecord_type; 198 199 /* A chain of variables to add to the top-level block surrounding the 200 construct. In the case of a parallel, this is in the child function. */ 201 tree block_vars; 202 203 /* A map of reduction pointer variables. For accelerators, each 204 reduction variable is replaced with an array. Each thread, in turn, 205 is assigned to a slot on that array. */ 206 splay_tree reduction_map; 207 208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit 209 barriers should jump to during omplower pass. */ 210 tree cancel_label; 211 212 /* What to do with variables with implicitly determined sharing 213 attributes. */ 214 enum omp_clause_default_kind default_kind; 215 216 /* Nesting depth of this context. Used to beautify error messages re 217 invalid gotos. The outermost ctx is depth 1, with depth 0 being 218 reserved for the main body of the function. */ 219 int depth; 220 221 /* True if this parallel directive is nested within another. */ 222 bool is_nested; 223 224 /* True if this construct can be cancelled. */ 225 bool cancellable; 226 227 /* For OpenACC loops, a mask of gang, worker and vector used at 228 levels below this one. */ 229 int gwv_below; 230 /* For OpenACC loops, a mask of gang, worker and vector used at 231 this level and above. For parallel and kernels clauses, a mask 232 indicating which of num_gangs/num_workers/num_vectors was used. */ 233 int gwv_this; 234 } omp_context; 235 236 /* A structure holding the elements of: 237 for (V = N1; V cond N2; V += STEP) [...] */ 238 239 struct omp_for_data_loop 240 { 241 tree v, n1, n2, step; 242 enum tree_code cond_code; 243 }; 244 245 /* A structure describing the main elements of a parallel loop. */ 246 247 struct omp_for_data 248 { 249 struct omp_for_data_loop loop; 250 tree chunk_size; 251 gomp_for *for_stmt; 252 tree pre, iter_type; 253 int collapse; 254 bool have_nowait, have_ordered; 255 enum omp_clause_schedule_kind sched_kind; 256 struct omp_for_data_loop *loops; 257 }; 258 259 260 static splay_tree all_contexts; 261 static int taskreg_nesting_level; 262 static int target_nesting_level; 263 static struct omp_region *root_omp_region; 264 static bitmap task_shared_vars; 265 static vec<omp_context *> taskreg_contexts; 266 267 static void scan_omp (gimple_seq *, omp_context *); 268 static tree scan_omp_1_op (tree *, int *, void *); 269 270 #define WALK_SUBSTMTS \ 271 case GIMPLE_BIND: \ 272 case GIMPLE_TRY: \ 273 case GIMPLE_CATCH: \ 274 case GIMPLE_EH_FILTER: \ 275 case GIMPLE_TRANSACTION: \ 276 /* The sub-statements for these should be walked. */ \ 277 *handled_ops_p = false; \ 278 break; 279 280 /* Helper function to get the name of the array containing the partial 281 reductions for OpenACC reductions. */ 282 static const char * 283 oacc_get_reduction_array_id (tree node) 284 { 285 const char *id = IDENTIFIER_POINTER (DECL_NAME (node)); 286 int len = strlen ("OACC") + strlen (id); 287 char *temp_name = XALLOCAVEC (char, len + 1); 288 snprintf (temp_name, len + 1, "OACC%s", id); 289 return IDENTIFIER_POINTER (get_identifier (temp_name)); 290 } 291 292 /* Determine the number of threads OpenACC threads used to determine the 293 size of the array of partial reductions. Currently, this is num_gangs 294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS, 295 because it is independed of the device used. */ 296 297 static tree 298 oacc_max_threads (omp_context *ctx) 299 { 300 tree nthreads, vector_length, gangs, clauses; 301 302 gangs = fold_convert (sizetype, integer_one_node); 303 vector_length = gangs; 304 305 /* The reduction clause may be nested inside a loop directive. 306 Scan for the innermost vector_length clause. */ 307 for (omp_context *oc = ctx; oc; oc = oc->outer) 308 { 309 if (gimple_code (oc->stmt) != GIMPLE_OMP_TARGET 310 || (gimple_omp_target_kind (oc->stmt) 311 != GF_OMP_TARGET_KIND_OACC_PARALLEL)) 312 continue; 313 314 clauses = gimple_omp_target_clauses (oc->stmt); 315 316 vector_length = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH); 317 if (vector_length) 318 vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length), 319 sizetype, 320 OMP_CLAUSE_VECTOR_LENGTH_EXPR 321 (vector_length)); 322 else 323 vector_length = fold_convert (sizetype, integer_one_node); 324 325 gangs = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS); 326 if (gangs) 327 gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (gangs), sizetype, 328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs)); 329 else 330 gangs = fold_convert (sizetype, integer_one_node); 331 332 break; 333 } 334 335 nthreads = fold_build2 (MULT_EXPR, sizetype, gangs, vector_length); 336 337 return nthreads; 338 } 339 340 /* Holds offload tables with decls. */ 341 vec<tree, va_gc> *offload_funcs, *offload_vars; 342 343 /* Convenience function for calling scan_omp_1_op on tree operands. */ 344 345 static inline tree 346 scan_omp_op (tree *tp, omp_context *ctx) 347 { 348 struct walk_stmt_info wi; 349 350 memset (&wi, 0, sizeof (wi)); 351 wi.info = ctx; 352 wi.want_locations = true; 353 354 return walk_tree (tp, scan_omp_1_op, &wi, NULL); 355 } 356 357 static void lower_omp (gimple_seq *, omp_context *); 358 static tree lookup_decl_in_outer_ctx (tree, omp_context *); 359 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *); 360 361 /* Find an OMP clause of type KIND within CLAUSES. */ 362 363 tree 364 find_omp_clause (tree clauses, enum omp_clause_code kind) 365 { 366 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) 367 if (OMP_CLAUSE_CODE (clauses) == kind) 368 return clauses; 369 370 return NULL_TREE; 371 } 372 373 /* Return true if CTX is for an omp parallel. */ 374 375 static inline bool 376 is_parallel_ctx (omp_context *ctx) 377 { 378 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL; 379 } 380 381 382 /* Return true if CTX is for an omp task. */ 383 384 static inline bool 385 is_task_ctx (omp_context *ctx) 386 { 387 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK; 388 } 389 390 391 /* Return true if CTX is for an omp parallel or omp task. */ 392 393 static inline bool 394 is_taskreg_ctx (omp_context *ctx) 395 { 396 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL 397 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK; 398 } 399 400 401 /* Return true if REGION is a combined parallel+workshare region. */ 402 403 static inline bool 404 is_combined_parallel (struct omp_region *region) 405 { 406 return region->is_combined_parallel; 407 } 408 409 410 /* Extract the header elements of parallel loop FOR_STMT and store 411 them into *FD. */ 412 413 static void 414 extract_omp_for_data (gomp_for *for_stmt, struct omp_for_data *fd, 415 struct omp_for_data_loop *loops) 416 { 417 tree t, var, *collapse_iter, *collapse_count; 418 tree count = NULL_TREE, iter_type = long_integer_type_node; 419 struct omp_for_data_loop *loop; 420 int i; 421 struct omp_for_data_loop dummy_loop; 422 location_t loc = gimple_location (for_stmt); 423 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD; 424 bool distribute = gimple_omp_for_kind (for_stmt) 425 == GF_OMP_FOR_KIND_DISTRIBUTE; 426 427 fd->for_stmt = for_stmt; 428 fd->pre = NULL; 429 fd->collapse = gimple_omp_for_collapse (for_stmt); 430 if (fd->collapse > 1) 431 fd->loops = loops; 432 else 433 fd->loops = &fd->loop; 434 435 fd->have_nowait = distribute || simd; 436 fd->have_ordered = false; 437 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC; 438 fd->chunk_size = NULL_TREE; 439 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR) 440 fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR; 441 collapse_iter = NULL; 442 collapse_count = NULL; 443 444 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t)) 445 switch (OMP_CLAUSE_CODE (t)) 446 { 447 case OMP_CLAUSE_NOWAIT: 448 fd->have_nowait = true; 449 break; 450 case OMP_CLAUSE_ORDERED: 451 fd->have_ordered = true; 452 break; 453 case OMP_CLAUSE_SCHEDULE: 454 gcc_assert (!distribute); 455 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t); 456 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t); 457 break; 458 case OMP_CLAUSE_DIST_SCHEDULE: 459 gcc_assert (distribute); 460 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t); 461 break; 462 case OMP_CLAUSE_COLLAPSE: 463 if (fd->collapse > 1) 464 { 465 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t); 466 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t); 467 } 468 break; 469 default: 470 break; 471 } 472 473 /* FIXME: for now map schedule(auto) to schedule(static). 474 There should be analysis to determine whether all iterations 475 are approximately the same amount of work (then schedule(static) 476 is best) or if it varies (then schedule(dynamic,N) is better). */ 477 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO) 478 { 479 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC; 480 gcc_assert (fd->chunk_size == NULL); 481 } 482 gcc_assert (fd->collapse == 1 || collapse_iter != NULL); 483 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME) 484 gcc_assert (fd->chunk_size == NULL); 485 else if (fd->chunk_size == NULL) 486 { 487 /* We only need to compute a default chunk size for ordered 488 static loops and dynamic loops. */ 489 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC 490 || fd->have_ordered) 491 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC) 492 ? integer_zero_node : integer_one_node; 493 } 494 495 for (i = 0; i < fd->collapse; i++) 496 { 497 if (fd->collapse == 1) 498 loop = &fd->loop; 499 else if (loops != NULL) 500 loop = loops + i; 501 else 502 loop = &dummy_loop; 503 504 loop->v = gimple_omp_for_index (for_stmt, i); 505 gcc_assert (SSA_VAR_P (loop->v)); 506 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE 507 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE); 508 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v; 509 loop->n1 = gimple_omp_for_initial (for_stmt, i); 510 511 loop->cond_code = gimple_omp_for_cond (for_stmt, i); 512 loop->n2 = gimple_omp_for_final (for_stmt, i); 513 switch (loop->cond_code) 514 { 515 case LT_EXPR: 516 case GT_EXPR: 517 break; 518 case NE_EXPR: 519 gcc_assert (gimple_omp_for_kind (for_stmt) 520 == GF_OMP_FOR_KIND_CILKSIMD 521 || (gimple_omp_for_kind (for_stmt) 522 == GF_OMP_FOR_KIND_CILKFOR)); 523 break; 524 case LE_EXPR: 525 if (POINTER_TYPE_P (TREE_TYPE (loop->n2))) 526 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1); 527 else 528 loop->n2 = fold_build2_loc (loc, 529 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2, 530 build_int_cst (TREE_TYPE (loop->n2), 1)); 531 loop->cond_code = LT_EXPR; 532 break; 533 case GE_EXPR: 534 if (POINTER_TYPE_P (TREE_TYPE (loop->n2))) 535 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1); 536 else 537 loop->n2 = fold_build2_loc (loc, 538 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2, 539 build_int_cst (TREE_TYPE (loop->n2), 1)); 540 loop->cond_code = GT_EXPR; 541 break; 542 default: 543 gcc_unreachable (); 544 } 545 546 t = gimple_omp_for_incr (for_stmt, i); 547 gcc_assert (TREE_OPERAND (t, 0) == var); 548 switch (TREE_CODE (t)) 549 { 550 case PLUS_EXPR: 551 loop->step = TREE_OPERAND (t, 1); 552 break; 553 case POINTER_PLUS_EXPR: 554 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1)); 555 break; 556 case MINUS_EXPR: 557 loop->step = TREE_OPERAND (t, 1); 558 loop->step = fold_build1_loc (loc, 559 NEGATE_EXPR, TREE_TYPE (loop->step), 560 loop->step); 561 break; 562 default: 563 gcc_unreachable (); 564 } 565 566 if (simd 567 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC 568 && !fd->have_ordered)) 569 { 570 if (fd->collapse == 1) 571 iter_type = TREE_TYPE (loop->v); 572 else if (i == 0 573 || TYPE_PRECISION (iter_type) 574 < TYPE_PRECISION (TREE_TYPE (loop->v))) 575 iter_type 576 = build_nonstandard_integer_type 577 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1); 578 } 579 else if (iter_type != long_long_unsigned_type_node) 580 { 581 if (POINTER_TYPE_P (TREE_TYPE (loop->v))) 582 iter_type = long_long_unsigned_type_node; 583 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v)) 584 && TYPE_PRECISION (TREE_TYPE (loop->v)) 585 >= TYPE_PRECISION (iter_type)) 586 { 587 tree n; 588 589 if (loop->cond_code == LT_EXPR) 590 n = fold_build2_loc (loc, 591 PLUS_EXPR, TREE_TYPE (loop->v), 592 loop->n2, loop->step); 593 else 594 n = loop->n1; 595 if (TREE_CODE (n) != INTEGER_CST 596 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n)) 597 iter_type = long_long_unsigned_type_node; 598 } 599 else if (TYPE_PRECISION (TREE_TYPE (loop->v)) 600 > TYPE_PRECISION (iter_type)) 601 { 602 tree n1, n2; 603 604 if (loop->cond_code == LT_EXPR) 605 { 606 n1 = loop->n1; 607 n2 = fold_build2_loc (loc, 608 PLUS_EXPR, TREE_TYPE (loop->v), 609 loop->n2, loop->step); 610 } 611 else 612 { 613 n1 = fold_build2_loc (loc, 614 MINUS_EXPR, TREE_TYPE (loop->v), 615 loop->n2, loop->step); 616 n2 = loop->n1; 617 } 618 if (TREE_CODE (n1) != INTEGER_CST 619 || TREE_CODE (n2) != INTEGER_CST 620 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1) 621 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type))) 622 iter_type = long_long_unsigned_type_node; 623 } 624 } 625 626 if (collapse_count && *collapse_count == NULL) 627 { 628 t = fold_binary (loop->cond_code, boolean_type_node, 629 fold_convert (TREE_TYPE (loop->v), loop->n1), 630 fold_convert (TREE_TYPE (loop->v), loop->n2)); 631 if (t && integer_zerop (t)) 632 count = build_zero_cst (long_long_unsigned_type_node); 633 else if ((i == 0 || count != NULL_TREE) 634 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE 635 && TREE_CONSTANT (loop->n1) 636 && TREE_CONSTANT (loop->n2) 637 && TREE_CODE (loop->step) == INTEGER_CST) 638 { 639 tree itype = TREE_TYPE (loop->v); 640 641 if (POINTER_TYPE_P (itype)) 642 itype = signed_type_for (itype); 643 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1)); 644 t = fold_build2_loc (loc, 645 PLUS_EXPR, itype, 646 fold_convert_loc (loc, itype, loop->step), t); 647 t = fold_build2_loc (loc, PLUS_EXPR, itype, t, 648 fold_convert_loc (loc, itype, loop->n2)); 649 t = fold_build2_loc (loc, MINUS_EXPR, itype, t, 650 fold_convert_loc (loc, itype, loop->n1)); 651 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR) 652 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, 653 fold_build1_loc (loc, NEGATE_EXPR, itype, t), 654 fold_build1_loc (loc, NEGATE_EXPR, itype, 655 fold_convert_loc (loc, itype, 656 loop->step))); 657 else 658 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t, 659 fold_convert_loc (loc, itype, loop->step)); 660 t = fold_convert_loc (loc, long_long_unsigned_type_node, t); 661 if (count != NULL_TREE) 662 count = fold_build2_loc (loc, 663 MULT_EXPR, long_long_unsigned_type_node, 664 count, t); 665 else 666 count = t; 667 if (TREE_CODE (count) != INTEGER_CST) 668 count = NULL_TREE; 669 } 670 else if (count && !integer_zerop (count)) 671 count = NULL_TREE; 672 } 673 } 674 675 if (count 676 && !simd 677 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC 678 || fd->have_ordered)) 679 { 680 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node))) 681 iter_type = long_long_unsigned_type_node; 682 else 683 iter_type = long_integer_type_node; 684 } 685 else if (collapse_iter && *collapse_iter != NULL) 686 iter_type = TREE_TYPE (*collapse_iter); 687 fd->iter_type = iter_type; 688 if (collapse_iter && *collapse_iter == NULL) 689 *collapse_iter = create_tmp_var (iter_type, ".iter"); 690 if (collapse_count && *collapse_count == NULL) 691 { 692 if (count) 693 *collapse_count = fold_convert_loc (loc, iter_type, count); 694 else 695 *collapse_count = create_tmp_var (iter_type, ".count"); 696 } 697 698 if (fd->collapse > 1) 699 { 700 fd->loop.v = *collapse_iter; 701 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0); 702 fd->loop.n2 = *collapse_count; 703 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1); 704 fd->loop.cond_code = LT_EXPR; 705 } 706 707 /* For OpenACC loops, force a chunk size of one, as this avoids the default 708 scheduling where several subsequent iterations are being executed by the 709 same thread. */ 710 if (gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) 711 { 712 gcc_assert (fd->chunk_size == NULL_TREE); 713 fd->chunk_size = build_int_cst (TREE_TYPE (fd->loop.v), 1); 714 } 715 } 716 717 718 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB 719 is the immediate dominator of PAR_ENTRY_BB, return true if there 720 are no data dependencies that would prevent expanding the parallel 721 directive at PAR_ENTRY_BB as a combined parallel+workshare region. 722 723 When expanding a combined parallel+workshare region, the call to 724 the child function may need additional arguments in the case of 725 GIMPLE_OMP_FOR regions. In some cases, these arguments are 726 computed out of variables passed in from the parent to the child 727 via 'struct .omp_data_s'. For instance: 728 729 #pragma omp parallel for schedule (guided, i * 4) 730 for (j ...) 731 732 Is lowered into: 733 734 # BLOCK 2 (PAR_ENTRY_BB) 735 .omp_data_o.i = i; 736 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598) 737 738 # BLOCK 3 (WS_ENTRY_BB) 739 .omp_data_i = &.omp_data_o; 740 D.1667 = .omp_data_i->i; 741 D.1598 = D.1667 * 4; 742 #pragma omp for schedule (guided, D.1598) 743 744 When we outline the parallel region, the call to the child function 745 'bar.omp_fn.0' will need the value D.1598 in its argument list, but 746 that value is computed *after* the call site. So, in principle we 747 cannot do the transformation. 748 749 To see whether the code in WS_ENTRY_BB blocks the combined 750 parallel+workshare call, we collect all the variables used in the 751 GIMPLE_OMP_FOR header check whether they appear on the LHS of any 752 statement in WS_ENTRY_BB. If so, then we cannot emit the combined 753 call. 754 755 FIXME. If we had the SSA form built at this point, we could merely 756 hoist the code in block 3 into block 2 and be done with it. But at 757 this point we don't have dataflow information and though we could 758 hack something up here, it is really not worth the aggravation. */ 759 760 static bool 761 workshare_safe_to_combine_p (basic_block ws_entry_bb) 762 { 763 struct omp_for_data fd; 764 gimple ws_stmt = last_stmt (ws_entry_bb); 765 766 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS) 767 return true; 768 769 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR); 770 771 extract_omp_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL); 772 773 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST) 774 return false; 775 if (fd.iter_type != long_integer_type_node) 776 return false; 777 778 /* FIXME. We give up too easily here. If any of these arguments 779 are not constants, they will likely involve variables that have 780 been mapped into fields of .omp_data_s for sharing with the child 781 function. With appropriate data flow, it would be possible to 782 see through this. */ 783 if (!is_gimple_min_invariant (fd.loop.n1) 784 || !is_gimple_min_invariant (fd.loop.n2) 785 || !is_gimple_min_invariant (fd.loop.step) 786 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size))) 787 return false; 788 789 return true; 790 } 791 792 793 /* Collect additional arguments needed to emit a combined 794 parallel+workshare call. WS_STMT is the workshare directive being 795 expanded. */ 796 797 static vec<tree, va_gc> * 798 get_ws_args_for (gimple par_stmt, gimple ws_stmt) 799 { 800 tree t; 801 location_t loc = gimple_location (ws_stmt); 802 vec<tree, va_gc> *ws_args; 803 804 if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt)) 805 { 806 struct omp_for_data fd; 807 tree n1, n2; 808 809 extract_omp_for_data (for_stmt, &fd, NULL); 810 n1 = fd.loop.n1; 811 n2 = fd.loop.n2; 812 813 if (gimple_omp_for_combined_into_p (for_stmt)) 814 { 815 tree innerc 816 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt), 817 OMP_CLAUSE__LOOPTEMP_); 818 gcc_assert (innerc); 819 n1 = OMP_CLAUSE_DECL (innerc); 820 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 821 OMP_CLAUSE__LOOPTEMP_); 822 gcc_assert (innerc); 823 n2 = OMP_CLAUSE_DECL (innerc); 824 } 825 826 vec_alloc (ws_args, 3 + (fd.chunk_size != 0)); 827 828 t = fold_convert_loc (loc, long_integer_type_node, n1); 829 ws_args->quick_push (t); 830 831 t = fold_convert_loc (loc, long_integer_type_node, n2); 832 ws_args->quick_push (t); 833 834 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step); 835 ws_args->quick_push (t); 836 837 if (fd.chunk_size) 838 { 839 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size); 840 ws_args->quick_push (t); 841 } 842 843 return ws_args; 844 } 845 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS) 846 { 847 /* Number of sections is equal to the number of edges from the 848 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to 849 the exit of the sections region. */ 850 basic_block bb = single_succ (gimple_bb (ws_stmt)); 851 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1); 852 vec_alloc (ws_args, 1); 853 ws_args->quick_push (t); 854 return ws_args; 855 } 856 857 gcc_unreachable (); 858 } 859 860 861 /* Discover whether REGION is a combined parallel+workshare region. */ 862 863 static void 864 determine_parallel_type (struct omp_region *region) 865 { 866 basic_block par_entry_bb, par_exit_bb; 867 basic_block ws_entry_bb, ws_exit_bb; 868 869 if (region == NULL || region->inner == NULL 870 || region->exit == NULL || region->inner->exit == NULL 871 || region->inner->cont == NULL) 872 return; 873 874 /* We only support parallel+for and parallel+sections. */ 875 if (region->type != GIMPLE_OMP_PARALLEL 876 || (region->inner->type != GIMPLE_OMP_FOR 877 && region->inner->type != GIMPLE_OMP_SECTIONS)) 878 return; 879 880 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and 881 WS_EXIT_BB -> PAR_EXIT_BB. */ 882 par_entry_bb = region->entry; 883 par_exit_bb = region->exit; 884 ws_entry_bb = region->inner->entry; 885 ws_exit_bb = region->inner->exit; 886 887 if (single_succ (par_entry_bb) == ws_entry_bb 888 && single_succ (ws_exit_bb) == par_exit_bb 889 && workshare_safe_to_combine_p (ws_entry_bb) 890 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb)) 891 || (last_and_only_stmt (ws_entry_bb) 892 && last_and_only_stmt (par_exit_bb)))) 893 { 894 gimple par_stmt = last_stmt (par_entry_bb); 895 gimple ws_stmt = last_stmt (ws_entry_bb); 896 897 if (region->inner->type == GIMPLE_OMP_FOR) 898 { 899 /* If this is a combined parallel loop, we need to determine 900 whether or not to use the combined library calls. There 901 are two cases where we do not apply the transformation: 902 static loops and any kind of ordered loop. In the first 903 case, we already open code the loop so there is no need 904 to do anything else. In the latter case, the combined 905 parallel loop call would still need extra synchronization 906 to implement ordered semantics, so there would not be any 907 gain in using the combined call. */ 908 tree clauses = gimple_omp_for_clauses (ws_stmt); 909 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE); 910 if (c == NULL 911 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC 912 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED)) 913 { 914 region->is_combined_parallel = false; 915 region->inner->is_combined_parallel = false; 916 return; 917 } 918 } 919 920 region->is_combined_parallel = true; 921 region->inner->is_combined_parallel = true; 922 region->ws_args = get_ws_args_for (par_stmt, ws_stmt); 923 } 924 } 925 926 927 /* Return true if EXPR is variable sized. */ 928 929 static inline bool 930 is_variable_sized (const_tree expr) 931 { 932 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr))); 933 } 934 935 /* Return true if DECL is a reference type. */ 936 937 static inline bool 938 is_reference (tree decl) 939 { 940 return lang_hooks.decls.omp_privatize_by_reference (decl); 941 } 942 943 /* Return the type of a decl. If the decl is reference type, 944 return its base type. */ 945 static inline tree 946 get_base_type (tree decl) 947 { 948 tree type = TREE_TYPE (decl); 949 if (is_reference (decl)) 950 type = TREE_TYPE (type); 951 return type; 952 } 953 954 /* Lookup variables. The "maybe" form 955 allows for the variable form to not have been entered, otherwise we 956 assert that the variable must have been entered. */ 957 958 static inline tree 959 lookup_decl (tree var, omp_context *ctx) 960 { 961 tree *n = ctx->cb.decl_map->get (var); 962 return *n; 963 } 964 965 static inline tree 966 maybe_lookup_decl (const_tree var, omp_context *ctx) 967 { 968 tree *n = ctx->cb.decl_map->get (const_cast<tree> (var)); 969 return n ? *n : NULL_TREE; 970 } 971 972 static inline tree 973 lookup_field (tree var, omp_context *ctx) 974 { 975 splay_tree_node n; 976 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); 977 return (tree) n->value; 978 } 979 980 static inline tree 981 lookup_sfield (tree var, omp_context *ctx) 982 { 983 splay_tree_node n; 984 n = splay_tree_lookup (ctx->sfield_map 985 ? ctx->sfield_map : ctx->field_map, 986 (splay_tree_key) var); 987 return (tree) n->value; 988 } 989 990 static inline tree 991 maybe_lookup_field (tree var, omp_context *ctx) 992 { 993 splay_tree_node n; 994 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); 995 return n ? (tree) n->value : NULL_TREE; 996 } 997 998 static inline tree 999 lookup_oacc_reduction (const char *id, omp_context *ctx) 1000 { 1001 splay_tree_node n; 1002 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) id); 1003 return (tree) n->value; 1004 } 1005 1006 static inline tree 1007 maybe_lookup_oacc_reduction (tree var, omp_context *ctx) 1008 { 1009 splay_tree_node n = NULL; 1010 if (ctx->reduction_map) 1011 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) var); 1012 return n ? (tree) n->value : NULL_TREE; 1013 } 1014 1015 /* Return true if DECL should be copied by pointer. SHARED_CTX is 1016 the parallel context if DECL is to be shared. */ 1017 1018 static bool 1019 use_pointer_for_field (tree decl, omp_context *shared_ctx) 1020 { 1021 if (AGGREGATE_TYPE_P (TREE_TYPE (decl))) 1022 return true; 1023 1024 /* We can only use copy-in/copy-out semantics for shared variables 1025 when we know the value is not accessible from an outer scope. */ 1026 if (shared_ctx) 1027 { 1028 gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt)); 1029 1030 /* ??? Trivially accessible from anywhere. But why would we even 1031 be passing an address in this case? Should we simply assert 1032 this to be false, or should we have a cleanup pass that removes 1033 these from the list of mappings? */ 1034 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl)) 1035 return true; 1036 1037 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell 1038 without analyzing the expression whether or not its location 1039 is accessible to anyone else. In the case of nested parallel 1040 regions it certainly may be. */ 1041 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl)) 1042 return true; 1043 1044 /* Do not use copy-in/copy-out for variables that have their 1045 address taken. */ 1046 if (TREE_ADDRESSABLE (decl)) 1047 return true; 1048 1049 /* lower_send_shared_vars only uses copy-in, but not copy-out 1050 for these. */ 1051 if (TREE_READONLY (decl) 1052 || ((TREE_CODE (decl) == RESULT_DECL 1053 || TREE_CODE (decl) == PARM_DECL) 1054 && DECL_BY_REFERENCE (decl))) 1055 return false; 1056 1057 /* Disallow copy-in/out in nested parallel if 1058 decl is shared in outer parallel, otherwise 1059 each thread could store the shared variable 1060 in its own copy-in location, making the 1061 variable no longer really shared. */ 1062 if (shared_ctx->is_nested) 1063 { 1064 omp_context *up; 1065 1066 for (up = shared_ctx->outer; up; up = up->outer) 1067 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up)) 1068 break; 1069 1070 if (up) 1071 { 1072 tree c; 1073 1074 for (c = gimple_omp_taskreg_clauses (up->stmt); 1075 c; c = OMP_CLAUSE_CHAIN (c)) 1076 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED 1077 && OMP_CLAUSE_DECL (c) == decl) 1078 break; 1079 1080 if (c) 1081 goto maybe_mark_addressable_and_ret; 1082 } 1083 } 1084 1085 /* For tasks avoid using copy-in/out. As tasks can be 1086 deferred or executed in different thread, when GOMP_task 1087 returns, the task hasn't necessarily terminated. */ 1088 if (is_task_ctx (shared_ctx)) 1089 { 1090 tree outer; 1091 maybe_mark_addressable_and_ret: 1092 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx); 1093 if (is_gimple_reg (outer)) 1094 { 1095 /* Taking address of OUTER in lower_send_shared_vars 1096 might need regimplification of everything that uses the 1097 variable. */ 1098 if (!task_shared_vars) 1099 task_shared_vars = BITMAP_ALLOC (NULL); 1100 bitmap_set_bit (task_shared_vars, DECL_UID (outer)); 1101 TREE_ADDRESSABLE (outer) = 1; 1102 } 1103 return true; 1104 } 1105 } 1106 1107 return false; 1108 } 1109 1110 /* Construct a new automatic decl similar to VAR. */ 1111 1112 static tree 1113 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx) 1114 { 1115 tree copy = copy_var_decl (var, name, type); 1116 1117 DECL_CONTEXT (copy) = current_function_decl; 1118 DECL_CHAIN (copy) = ctx->block_vars; 1119 ctx->block_vars = copy; 1120 1121 return copy; 1122 } 1123 1124 static tree 1125 omp_copy_decl_1 (tree var, omp_context *ctx) 1126 { 1127 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx); 1128 } 1129 1130 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it 1131 as appropriate. */ 1132 static tree 1133 omp_build_component_ref (tree obj, tree field) 1134 { 1135 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL); 1136 if (TREE_THIS_VOLATILE (field)) 1137 TREE_THIS_VOLATILE (ret) |= 1; 1138 if (TREE_READONLY (field)) 1139 TREE_READONLY (ret) |= 1; 1140 return ret; 1141 } 1142 1143 /* Build tree nodes to access the field for VAR on the receiver side. */ 1144 1145 static tree 1146 build_receiver_ref (tree var, bool by_ref, omp_context *ctx) 1147 { 1148 tree x, field = lookup_field (var, ctx); 1149 1150 /* If the receiver record type was remapped in the child function, 1151 remap the field into the new record type. */ 1152 x = maybe_lookup_field (field, ctx); 1153 if (x != NULL) 1154 field = x; 1155 1156 x = build_simple_mem_ref (ctx->receiver_decl); 1157 x = omp_build_component_ref (x, field); 1158 if (by_ref) 1159 x = build_simple_mem_ref (x); 1160 1161 return x; 1162 } 1163 1164 /* Build tree nodes to access VAR in the scope outer to CTX. In the case 1165 of a parallel, this is a component reference; for workshare constructs 1166 this is some variable. */ 1167 1168 static tree 1169 build_outer_var_ref (tree var, omp_context *ctx, 1170 enum omp_clause_code code = OMP_CLAUSE_ERROR) 1171 { 1172 tree x; 1173 1174 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))) 1175 x = var; 1176 else if (is_variable_sized (var)) 1177 { 1178 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0); 1179 x = build_outer_var_ref (x, ctx, code); 1180 x = build_simple_mem_ref (x); 1181 } 1182 else if (is_taskreg_ctx (ctx)) 1183 { 1184 bool by_ref = use_pointer_for_field (var, NULL); 1185 x = build_receiver_ref (var, by_ref, ctx); 1186 } 1187 else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 1188 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 1189 || (code == OMP_CLAUSE_PRIVATE 1190 && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 1191 || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS 1192 || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE))) 1193 { 1194 /* #pragma omp simd isn't a worksharing construct, and can reference 1195 even private vars in its linear etc. clauses. 1196 Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer 1197 to private vars in all worksharing constructs. */ 1198 x = NULL_TREE; 1199 if (ctx->outer && is_taskreg_ctx (ctx)) 1200 x = lookup_decl (var, ctx->outer); 1201 else if (ctx->outer) 1202 x = maybe_lookup_decl_in_outer_ctx (var, ctx); 1203 if (x == NULL_TREE) 1204 x = var; 1205 } 1206 else if (ctx->outer) 1207 x = lookup_decl (var, ctx->outer); 1208 else if (is_reference (var)) 1209 /* This can happen with orphaned constructs. If var is reference, it is 1210 possible it is shared and as such valid. */ 1211 x = var; 1212 else 1213 gcc_unreachable (); 1214 1215 if (is_reference (var)) 1216 x = build_simple_mem_ref (x); 1217 1218 return x; 1219 } 1220 1221 /* Build tree nodes to access the field for VAR on the sender side. */ 1222 1223 static tree 1224 build_sender_ref (tree var, omp_context *ctx) 1225 { 1226 tree field = lookup_sfield (var, ctx); 1227 return omp_build_component_ref (ctx->sender_decl, field); 1228 } 1229 1230 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */ 1231 1232 static void 1233 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx) 1234 { 1235 tree field, type, sfield = NULL_TREE; 1236 1237 gcc_assert ((mask & 1) == 0 1238 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var)); 1239 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map 1240 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var)); 1241 gcc_assert ((mask & 3) == 3 1242 || !is_gimple_omp_oacc (ctx->stmt)); 1243 1244 type = TREE_TYPE (var); 1245 if (mask & 4) 1246 { 1247 gcc_assert (TREE_CODE (type) == ARRAY_TYPE); 1248 type = build_pointer_type (build_pointer_type (type)); 1249 } 1250 else if (by_ref) 1251 type = build_pointer_type (type); 1252 else if ((mask & 3) == 1 && is_reference (var)) 1253 type = TREE_TYPE (type); 1254 1255 field = build_decl (DECL_SOURCE_LOCATION (var), 1256 FIELD_DECL, DECL_NAME (var), type); 1257 1258 /* Remember what variable this field was created for. This does have a 1259 side effect of making dwarf2out ignore this member, so for helpful 1260 debugging we clear it later in delete_omp_context. */ 1261 DECL_ABSTRACT_ORIGIN (field) = var; 1262 if (type == TREE_TYPE (var)) 1263 { 1264 DECL_ALIGN (field) = DECL_ALIGN (var); 1265 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var); 1266 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var); 1267 } 1268 else 1269 DECL_ALIGN (field) = TYPE_ALIGN (type); 1270 1271 if ((mask & 3) == 3) 1272 { 1273 insert_field_into_struct (ctx->record_type, field); 1274 if (ctx->srecord_type) 1275 { 1276 sfield = build_decl (DECL_SOURCE_LOCATION (var), 1277 FIELD_DECL, DECL_NAME (var), type); 1278 DECL_ABSTRACT_ORIGIN (sfield) = var; 1279 DECL_ALIGN (sfield) = DECL_ALIGN (field); 1280 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field); 1281 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field); 1282 insert_field_into_struct (ctx->srecord_type, sfield); 1283 } 1284 } 1285 else 1286 { 1287 if (ctx->srecord_type == NULL_TREE) 1288 { 1289 tree t; 1290 1291 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE); 1292 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 1293 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t)) 1294 { 1295 sfield = build_decl (DECL_SOURCE_LOCATION (var), 1296 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t)); 1297 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t); 1298 insert_field_into_struct (ctx->srecord_type, sfield); 1299 splay_tree_insert (ctx->sfield_map, 1300 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t), 1301 (splay_tree_value) sfield); 1302 } 1303 } 1304 sfield = field; 1305 insert_field_into_struct ((mask & 1) ? ctx->record_type 1306 : ctx->srecord_type, field); 1307 } 1308 1309 if (mask & 1) 1310 splay_tree_insert (ctx->field_map, (splay_tree_key) var, 1311 (splay_tree_value) field); 1312 if ((mask & 2) && ctx->sfield_map) 1313 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var, 1314 (splay_tree_value) sfield); 1315 } 1316 1317 static tree 1318 install_var_local (tree var, omp_context *ctx) 1319 { 1320 tree new_var = omp_copy_decl_1 (var, ctx); 1321 insert_decl_map (&ctx->cb, var, new_var); 1322 return new_var; 1323 } 1324 1325 /* Adjust the replacement for DECL in CTX for the new context. This means 1326 copying the DECL_VALUE_EXPR, and fixing up the type. */ 1327 1328 static void 1329 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug) 1330 { 1331 tree new_decl, size; 1332 1333 new_decl = lookup_decl (decl, ctx); 1334 1335 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb); 1336 1337 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug) 1338 && DECL_HAS_VALUE_EXPR_P (decl)) 1339 { 1340 tree ve = DECL_VALUE_EXPR (decl); 1341 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL); 1342 SET_DECL_VALUE_EXPR (new_decl, ve); 1343 DECL_HAS_VALUE_EXPR_P (new_decl) = 1; 1344 } 1345 1346 if (!TREE_CONSTANT (DECL_SIZE (new_decl))) 1347 { 1348 size = remap_decl (DECL_SIZE (decl), &ctx->cb); 1349 if (size == error_mark_node) 1350 size = TYPE_SIZE (TREE_TYPE (new_decl)); 1351 DECL_SIZE (new_decl) = size; 1352 1353 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb); 1354 if (size == error_mark_node) 1355 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl)); 1356 DECL_SIZE_UNIT (new_decl) = size; 1357 } 1358 } 1359 1360 /* The callback for remap_decl. Search all containing contexts for a 1361 mapping of the variable; this avoids having to duplicate the splay 1362 tree ahead of time. We know a mapping doesn't already exist in the 1363 given context. Create new mappings to implement default semantics. */ 1364 1365 static tree 1366 omp_copy_decl (tree var, copy_body_data *cb) 1367 { 1368 omp_context *ctx = (omp_context *) cb; 1369 tree new_var; 1370 1371 if (TREE_CODE (var) == LABEL_DECL) 1372 { 1373 if (FORCED_LABEL (var) || DECL_NONLOCAL (var)) 1374 return var; 1375 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var)); 1376 DECL_CONTEXT (new_var) = current_function_decl; 1377 insert_decl_map (&ctx->cb, var, new_var); 1378 return new_var; 1379 } 1380 1381 while (!is_taskreg_ctx (ctx)) 1382 { 1383 ctx = ctx->outer; 1384 if (ctx == NULL) 1385 return var; 1386 new_var = maybe_lookup_decl (var, ctx); 1387 if (new_var) 1388 return new_var; 1389 } 1390 1391 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn) 1392 return var; 1393 1394 return error_mark_node; 1395 } 1396 1397 1398 /* Debugging dumps for parallel regions. */ 1399 void dump_omp_region (FILE *, struct omp_region *, int); 1400 void debug_omp_region (struct omp_region *); 1401 void debug_all_omp_regions (void); 1402 1403 /* Dump the parallel region tree rooted at REGION. */ 1404 1405 void 1406 dump_omp_region (FILE *file, struct omp_region *region, int indent) 1407 { 1408 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index, 1409 gimple_code_name[region->type]); 1410 1411 if (region->inner) 1412 dump_omp_region (file, region->inner, indent + 4); 1413 1414 if (region->cont) 1415 { 1416 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "", 1417 region->cont->index); 1418 } 1419 1420 if (region->exit) 1421 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "", 1422 region->exit->index); 1423 else 1424 fprintf (file, "%*s[no exit marker]\n", indent, ""); 1425 1426 if (region->next) 1427 dump_omp_region (file, region->next, indent); 1428 } 1429 1430 DEBUG_FUNCTION void 1431 debug_omp_region (struct omp_region *region) 1432 { 1433 dump_omp_region (stderr, region, 0); 1434 } 1435 1436 DEBUG_FUNCTION void 1437 debug_all_omp_regions (void) 1438 { 1439 dump_omp_region (stderr, root_omp_region, 0); 1440 } 1441 1442 1443 /* Create a new parallel region starting at STMT inside region PARENT. */ 1444 1445 static struct omp_region * 1446 new_omp_region (basic_block bb, enum gimple_code type, 1447 struct omp_region *parent) 1448 { 1449 struct omp_region *region = XCNEW (struct omp_region); 1450 1451 region->outer = parent; 1452 region->entry = bb; 1453 region->type = type; 1454 1455 if (parent) 1456 { 1457 /* This is a nested region. Add it to the list of inner 1458 regions in PARENT. */ 1459 region->next = parent->inner; 1460 parent->inner = region; 1461 } 1462 else 1463 { 1464 /* This is a toplevel region. Add it to the list of toplevel 1465 regions in ROOT_OMP_REGION. */ 1466 region->next = root_omp_region; 1467 root_omp_region = region; 1468 } 1469 1470 return region; 1471 } 1472 1473 /* Release the memory associated with the region tree rooted at REGION. */ 1474 1475 static void 1476 free_omp_region_1 (struct omp_region *region) 1477 { 1478 struct omp_region *i, *n; 1479 1480 for (i = region->inner; i ; i = n) 1481 { 1482 n = i->next; 1483 free_omp_region_1 (i); 1484 } 1485 1486 free (region); 1487 } 1488 1489 /* Release the memory for the entire omp region tree. */ 1490 1491 void 1492 free_omp_regions (void) 1493 { 1494 struct omp_region *r, *n; 1495 for (r = root_omp_region; r ; r = n) 1496 { 1497 n = r->next; 1498 free_omp_region_1 (r); 1499 } 1500 root_omp_region = NULL; 1501 } 1502 1503 1504 /* Create a new context, with OUTER_CTX being the surrounding context. */ 1505 1506 static omp_context * 1507 new_omp_context (gimple stmt, omp_context *outer_ctx) 1508 { 1509 omp_context *ctx = XCNEW (omp_context); 1510 1511 splay_tree_insert (all_contexts, (splay_tree_key) stmt, 1512 (splay_tree_value) ctx); 1513 ctx->stmt = stmt; 1514 1515 if (outer_ctx) 1516 { 1517 ctx->outer = outer_ctx; 1518 ctx->cb = outer_ctx->cb; 1519 ctx->cb.block = NULL; 1520 ctx->depth = outer_ctx->depth + 1; 1521 ctx->reduction_map = outer_ctx->reduction_map; 1522 } 1523 else 1524 { 1525 ctx->cb.src_fn = current_function_decl; 1526 ctx->cb.dst_fn = current_function_decl; 1527 ctx->cb.src_node = cgraph_node::get (current_function_decl); 1528 gcc_checking_assert (ctx->cb.src_node); 1529 ctx->cb.dst_node = ctx->cb.src_node; 1530 ctx->cb.src_cfun = cfun; 1531 ctx->cb.copy_decl = omp_copy_decl; 1532 ctx->cb.eh_lp_nr = 0; 1533 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE; 1534 ctx->depth = 1; 1535 } 1536 1537 ctx->cb.decl_map = new hash_map<tree, tree>; 1538 1539 return ctx; 1540 } 1541 1542 static gimple_seq maybe_catch_exception (gimple_seq); 1543 1544 /* Finalize task copyfn. */ 1545 1546 static void 1547 finalize_task_copyfn (gomp_task *task_stmt) 1548 { 1549 struct function *child_cfun; 1550 tree child_fn; 1551 gimple_seq seq = NULL, new_seq; 1552 gbind *bind; 1553 1554 child_fn = gimple_omp_task_copy_fn (task_stmt); 1555 if (child_fn == NULL_TREE) 1556 return; 1557 1558 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 1559 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties; 1560 1561 push_cfun (child_cfun); 1562 bind = gimplify_body (child_fn, false); 1563 gimple_seq_add_stmt (&seq, bind); 1564 new_seq = maybe_catch_exception (seq); 1565 if (new_seq != seq) 1566 { 1567 bind = gimple_build_bind (NULL, new_seq, NULL); 1568 seq = NULL; 1569 gimple_seq_add_stmt (&seq, bind); 1570 } 1571 gimple_set_body (child_fn, seq); 1572 pop_cfun (); 1573 1574 /* Inform the callgraph about the new function. */ 1575 cgraph_node::add_new_function (child_fn, false); 1576 cgraph_node::get (child_fn)->parallelized_function = 1; 1577 } 1578 1579 /* Destroy a omp_context data structures. Called through the splay tree 1580 value delete callback. */ 1581 1582 static void 1583 delete_omp_context (splay_tree_value value) 1584 { 1585 omp_context *ctx = (omp_context *) value; 1586 1587 delete ctx->cb.decl_map; 1588 1589 if (ctx->field_map) 1590 splay_tree_delete (ctx->field_map); 1591 if (ctx->sfield_map) 1592 splay_tree_delete (ctx->sfield_map); 1593 /* Reduction map is copied to nested contexts, so only delete it in the 1594 owner. */ 1595 if (ctx->reduction_map 1596 && gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET 1597 && is_gimple_omp_offloaded (ctx->stmt) 1598 && is_gimple_omp_oacc (ctx->stmt)) 1599 splay_tree_delete (ctx->reduction_map); 1600 1601 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before 1602 it produces corrupt debug information. */ 1603 if (ctx->record_type) 1604 { 1605 tree t; 1606 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t)) 1607 DECL_ABSTRACT_ORIGIN (t) = NULL; 1608 } 1609 if (ctx->srecord_type) 1610 { 1611 tree t; 1612 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t)) 1613 DECL_ABSTRACT_ORIGIN (t) = NULL; 1614 } 1615 1616 if (is_task_ctx (ctx)) 1617 finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt)); 1618 1619 XDELETE (ctx); 1620 } 1621 1622 /* Fix up RECEIVER_DECL with a type that has been remapped to the child 1623 context. */ 1624 1625 static void 1626 fixup_child_record_type (omp_context *ctx) 1627 { 1628 tree f, type = ctx->record_type; 1629 1630 /* ??? It isn't sufficient to just call remap_type here, because 1631 variably_modified_type_p doesn't work the way we expect for 1632 record types. Testing each field for whether it needs remapping 1633 and creating a new record by hand works, however. */ 1634 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f)) 1635 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) 1636 break; 1637 if (f) 1638 { 1639 tree name, new_fields = NULL; 1640 1641 type = lang_hooks.types.make_type (RECORD_TYPE); 1642 name = DECL_NAME (TYPE_NAME (ctx->record_type)); 1643 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl), 1644 TYPE_DECL, name, type); 1645 TYPE_NAME (type) = name; 1646 1647 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f)) 1648 { 1649 tree new_f = copy_node (f); 1650 DECL_CONTEXT (new_f) = type; 1651 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb); 1652 DECL_CHAIN (new_f) = new_fields; 1653 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL); 1654 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, 1655 &ctx->cb, NULL); 1656 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, 1657 &ctx->cb, NULL); 1658 new_fields = new_f; 1659 1660 /* Arrange to be able to look up the receiver field 1661 given the sender field. */ 1662 splay_tree_insert (ctx->field_map, (splay_tree_key) f, 1663 (splay_tree_value) new_f); 1664 } 1665 TYPE_FIELDS (type) = nreverse (new_fields); 1666 layout_type (type); 1667 } 1668 1669 TREE_TYPE (ctx->receiver_decl) 1670 = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT); 1671 } 1672 1673 /* Instantiate decls as necessary in CTX to satisfy the data sharing 1674 specified by CLAUSES. */ 1675 1676 static void 1677 scan_sharing_clauses (tree clauses, omp_context *ctx) 1678 { 1679 tree c, decl; 1680 bool scan_array_reductions = false; 1681 1682 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 1683 { 1684 bool by_ref; 1685 1686 switch (OMP_CLAUSE_CODE (c)) 1687 { 1688 case OMP_CLAUSE_PRIVATE: 1689 decl = OMP_CLAUSE_DECL (c); 1690 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 1691 goto do_private; 1692 else if (!is_variable_sized (decl)) 1693 install_var_local (decl, ctx); 1694 break; 1695 1696 case OMP_CLAUSE_SHARED: 1697 decl = OMP_CLAUSE_DECL (c); 1698 /* Ignore shared directives in teams construct. */ 1699 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 1700 { 1701 /* Global variables don't need to be copied, 1702 the receiver side will use them directly. */ 1703 tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx); 1704 if (is_global_var (odecl)) 1705 break; 1706 insert_decl_map (&ctx->cb, decl, odecl); 1707 break; 1708 } 1709 gcc_assert (is_taskreg_ctx (ctx)); 1710 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl)) 1711 || !is_variable_sized (decl)); 1712 /* Global variables don't need to be copied, 1713 the receiver side will use them directly. */ 1714 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) 1715 break; 1716 by_ref = use_pointer_for_field (decl, ctx); 1717 if (! TREE_READONLY (decl) 1718 || TREE_ADDRESSABLE (decl) 1719 || by_ref 1720 || is_reference (decl)) 1721 { 1722 install_var_field (decl, by_ref, 3, ctx); 1723 install_var_local (decl, ctx); 1724 break; 1725 } 1726 /* We don't need to copy const scalar vars back. */ 1727 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE); 1728 goto do_private; 1729 1730 case OMP_CLAUSE_LASTPRIVATE: 1731 /* Let the corresponding firstprivate clause create 1732 the variable. */ 1733 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 1734 break; 1735 /* FALLTHRU */ 1736 1737 case OMP_CLAUSE_FIRSTPRIVATE: 1738 if (is_gimple_omp_oacc (ctx->stmt)) 1739 { 1740 sorry ("clause not supported yet"); 1741 break; 1742 } 1743 /* FALLTHRU */ 1744 case OMP_CLAUSE_REDUCTION: 1745 case OMP_CLAUSE_LINEAR: 1746 decl = OMP_CLAUSE_DECL (c); 1747 do_private: 1748 if (is_variable_sized (decl)) 1749 { 1750 if (is_task_ctx (ctx)) 1751 install_var_field (decl, false, 1, ctx); 1752 break; 1753 } 1754 else if (is_taskreg_ctx (ctx)) 1755 { 1756 bool global 1757 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)); 1758 by_ref = use_pointer_for_field (decl, NULL); 1759 1760 if (is_task_ctx (ctx) 1761 && (global || by_ref || is_reference (decl))) 1762 { 1763 install_var_field (decl, false, 1, ctx); 1764 if (!global) 1765 install_var_field (decl, by_ref, 2, ctx); 1766 } 1767 else if (!global) 1768 install_var_field (decl, by_ref, 3, ctx); 1769 } 1770 install_var_local (decl, ctx); 1771 if (is_gimple_omp_oacc (ctx->stmt) 1772 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) 1773 { 1774 /* Create a decl for the reduction array. */ 1775 tree var = OMP_CLAUSE_DECL (c); 1776 tree type = get_base_type (var); 1777 tree ptype = build_pointer_type (type); 1778 tree array = create_tmp_var (ptype, 1779 oacc_get_reduction_array_id (var)); 1780 omp_context *c = (ctx->field_map ? ctx : ctx->outer); 1781 install_var_field (array, true, 3, c); 1782 install_var_local (array, c); 1783 1784 /* Insert it into the current context. */ 1785 splay_tree_insert (ctx->reduction_map, (splay_tree_key) 1786 oacc_get_reduction_array_id (var), 1787 (splay_tree_value) array); 1788 splay_tree_insert (ctx->reduction_map, 1789 (splay_tree_key) array, 1790 (splay_tree_value) array); 1791 } 1792 break; 1793 1794 case OMP_CLAUSE__LOOPTEMP_: 1795 gcc_assert (is_parallel_ctx (ctx)); 1796 decl = OMP_CLAUSE_DECL (c); 1797 install_var_field (decl, false, 3, ctx); 1798 install_var_local (decl, ctx); 1799 break; 1800 1801 case OMP_CLAUSE_COPYPRIVATE: 1802 case OMP_CLAUSE_COPYIN: 1803 decl = OMP_CLAUSE_DECL (c); 1804 by_ref = use_pointer_for_field (decl, NULL); 1805 install_var_field (decl, by_ref, 3, ctx); 1806 break; 1807 1808 case OMP_CLAUSE_DEFAULT: 1809 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c); 1810 break; 1811 1812 case OMP_CLAUSE_FINAL: 1813 case OMP_CLAUSE_IF: 1814 case OMP_CLAUSE_NUM_THREADS: 1815 case OMP_CLAUSE_NUM_TEAMS: 1816 case OMP_CLAUSE_THREAD_LIMIT: 1817 case OMP_CLAUSE_DEVICE: 1818 case OMP_CLAUSE_SCHEDULE: 1819 case OMP_CLAUSE_DIST_SCHEDULE: 1820 case OMP_CLAUSE_DEPEND: 1821 case OMP_CLAUSE__CILK_FOR_COUNT_: 1822 case OMP_CLAUSE_NUM_GANGS: 1823 case OMP_CLAUSE_NUM_WORKERS: 1824 case OMP_CLAUSE_VECTOR_LENGTH: 1825 if (ctx->outer) 1826 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer); 1827 break; 1828 1829 case OMP_CLAUSE_TO: 1830 case OMP_CLAUSE_FROM: 1831 case OMP_CLAUSE_MAP: 1832 if (ctx->outer) 1833 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer); 1834 decl = OMP_CLAUSE_DECL (c); 1835 /* Global variables with "omp declare target" attribute 1836 don't need to be copied, the receiver side will use them 1837 directly. */ 1838 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 1839 && DECL_P (decl) 1840 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) 1841 && varpool_node::get_create (decl)->offloadable) 1842 break; 1843 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 1844 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER) 1845 { 1846 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are 1847 not offloaded; there is nothing to map for those. */ 1848 if (!is_gimple_omp_offloaded (ctx->stmt) 1849 && !POINTER_TYPE_P (TREE_TYPE (decl)) 1850 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) 1851 break; 1852 } 1853 if (DECL_P (decl)) 1854 { 1855 if (DECL_SIZE (decl) 1856 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) 1857 { 1858 tree decl2 = DECL_VALUE_EXPR (decl); 1859 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF); 1860 decl2 = TREE_OPERAND (decl2, 0); 1861 gcc_assert (DECL_P (decl2)); 1862 install_var_field (decl2, true, 3, ctx); 1863 install_var_local (decl2, ctx); 1864 install_var_local (decl, ctx); 1865 } 1866 else 1867 { 1868 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 1869 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 1870 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) 1871 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) 1872 install_var_field (decl, true, 7, ctx); 1873 else 1874 install_var_field (decl, true, 3, ctx); 1875 if (is_gimple_omp_offloaded (ctx->stmt)) 1876 install_var_local (decl, ctx); 1877 } 1878 } 1879 else 1880 { 1881 tree base = get_base_address (decl); 1882 tree nc = OMP_CLAUSE_CHAIN (c); 1883 if (DECL_P (base) 1884 && nc != NULL_TREE 1885 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP 1886 && OMP_CLAUSE_DECL (nc) == base 1887 && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER 1888 && integer_zerop (OMP_CLAUSE_SIZE (nc))) 1889 { 1890 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1; 1891 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1; 1892 } 1893 else 1894 { 1895 if (ctx->outer) 1896 { 1897 scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer); 1898 decl = OMP_CLAUSE_DECL (c); 1899 } 1900 gcc_assert (!splay_tree_lookup (ctx->field_map, 1901 (splay_tree_key) decl)); 1902 tree field 1903 = build_decl (OMP_CLAUSE_LOCATION (c), 1904 FIELD_DECL, NULL_TREE, ptr_type_node); 1905 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node); 1906 insert_field_into_struct (ctx->record_type, field); 1907 splay_tree_insert (ctx->field_map, (splay_tree_key) decl, 1908 (splay_tree_value) field); 1909 } 1910 } 1911 break; 1912 1913 case OMP_CLAUSE_NOWAIT: 1914 case OMP_CLAUSE_ORDERED: 1915 case OMP_CLAUSE_COLLAPSE: 1916 case OMP_CLAUSE_UNTIED: 1917 case OMP_CLAUSE_MERGEABLE: 1918 case OMP_CLAUSE_PROC_BIND: 1919 case OMP_CLAUSE_SAFELEN: 1920 case OMP_CLAUSE_ASYNC: 1921 case OMP_CLAUSE_WAIT: 1922 case OMP_CLAUSE_GANG: 1923 case OMP_CLAUSE_WORKER: 1924 case OMP_CLAUSE_VECTOR: 1925 break; 1926 1927 case OMP_CLAUSE_ALIGNED: 1928 decl = OMP_CLAUSE_DECL (c); 1929 if (is_global_var (decl) 1930 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) 1931 install_var_local (decl, ctx); 1932 break; 1933 1934 case OMP_CLAUSE_DEVICE_RESIDENT: 1935 case OMP_CLAUSE_USE_DEVICE: 1936 case OMP_CLAUSE__CACHE_: 1937 case OMP_CLAUSE_INDEPENDENT: 1938 case OMP_CLAUSE_AUTO: 1939 case OMP_CLAUSE_SEQ: 1940 sorry ("Clause not supported yet"); 1941 break; 1942 1943 default: 1944 gcc_unreachable (); 1945 } 1946 } 1947 1948 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 1949 { 1950 switch (OMP_CLAUSE_CODE (c)) 1951 { 1952 case OMP_CLAUSE_LASTPRIVATE: 1953 /* Let the corresponding firstprivate clause create 1954 the variable. */ 1955 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) 1956 scan_array_reductions = true; 1957 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 1958 break; 1959 /* FALLTHRU */ 1960 1961 case OMP_CLAUSE_FIRSTPRIVATE: 1962 if (is_gimple_omp_oacc (ctx->stmt)) 1963 { 1964 sorry ("clause not supported yet"); 1965 break; 1966 } 1967 /* FALLTHRU */ 1968 case OMP_CLAUSE_PRIVATE: 1969 case OMP_CLAUSE_REDUCTION: 1970 case OMP_CLAUSE_LINEAR: 1971 decl = OMP_CLAUSE_DECL (c); 1972 if (is_variable_sized (decl)) 1973 install_var_local (decl, ctx); 1974 fixup_remapped_decl (decl, ctx, 1975 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE 1976 && OMP_CLAUSE_PRIVATE_DEBUG (c)); 1977 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION 1978 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 1979 scan_array_reductions = true; 1980 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 1981 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) 1982 scan_array_reductions = true; 1983 break; 1984 1985 case OMP_CLAUSE_SHARED: 1986 /* Ignore shared directives in teams construct. */ 1987 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 1988 break; 1989 decl = OMP_CLAUSE_DECL (c); 1990 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) 1991 fixup_remapped_decl (decl, ctx, false); 1992 break; 1993 1994 case OMP_CLAUSE_MAP: 1995 if (!is_gimple_omp_offloaded (ctx->stmt)) 1996 break; 1997 decl = OMP_CLAUSE_DECL (c); 1998 if (DECL_P (decl) 1999 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) 2000 && varpool_node::get_create (decl)->offloadable) 2001 break; 2002 if (DECL_P (decl)) 2003 { 2004 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 2005 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE 2006 && !COMPLETE_TYPE_P (TREE_TYPE (decl))) 2007 { 2008 tree new_decl = lookup_decl (decl, ctx); 2009 TREE_TYPE (new_decl) 2010 = remap_type (TREE_TYPE (decl), &ctx->cb); 2011 } 2012 else if (DECL_SIZE (decl) 2013 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) 2014 { 2015 tree decl2 = DECL_VALUE_EXPR (decl); 2016 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF); 2017 decl2 = TREE_OPERAND (decl2, 0); 2018 gcc_assert (DECL_P (decl2)); 2019 fixup_remapped_decl (decl2, ctx, false); 2020 fixup_remapped_decl (decl, ctx, true); 2021 } 2022 else 2023 fixup_remapped_decl (decl, ctx, false); 2024 } 2025 break; 2026 2027 case OMP_CLAUSE_COPYPRIVATE: 2028 case OMP_CLAUSE_COPYIN: 2029 case OMP_CLAUSE_DEFAULT: 2030 case OMP_CLAUSE_IF: 2031 case OMP_CLAUSE_NUM_THREADS: 2032 case OMP_CLAUSE_NUM_TEAMS: 2033 case OMP_CLAUSE_THREAD_LIMIT: 2034 case OMP_CLAUSE_DEVICE: 2035 case OMP_CLAUSE_SCHEDULE: 2036 case OMP_CLAUSE_DIST_SCHEDULE: 2037 case OMP_CLAUSE_NOWAIT: 2038 case OMP_CLAUSE_ORDERED: 2039 case OMP_CLAUSE_COLLAPSE: 2040 case OMP_CLAUSE_UNTIED: 2041 case OMP_CLAUSE_FINAL: 2042 case OMP_CLAUSE_MERGEABLE: 2043 case OMP_CLAUSE_PROC_BIND: 2044 case OMP_CLAUSE_SAFELEN: 2045 case OMP_CLAUSE_ALIGNED: 2046 case OMP_CLAUSE_DEPEND: 2047 case OMP_CLAUSE__LOOPTEMP_: 2048 case OMP_CLAUSE_TO: 2049 case OMP_CLAUSE_FROM: 2050 case OMP_CLAUSE__CILK_FOR_COUNT_: 2051 case OMP_CLAUSE_ASYNC: 2052 case OMP_CLAUSE_WAIT: 2053 case OMP_CLAUSE_NUM_GANGS: 2054 case OMP_CLAUSE_NUM_WORKERS: 2055 case OMP_CLAUSE_VECTOR_LENGTH: 2056 case OMP_CLAUSE_GANG: 2057 case OMP_CLAUSE_WORKER: 2058 case OMP_CLAUSE_VECTOR: 2059 break; 2060 2061 case OMP_CLAUSE_DEVICE_RESIDENT: 2062 case OMP_CLAUSE_USE_DEVICE: 2063 case OMP_CLAUSE__CACHE_: 2064 case OMP_CLAUSE_INDEPENDENT: 2065 case OMP_CLAUSE_AUTO: 2066 case OMP_CLAUSE_SEQ: 2067 sorry ("Clause not supported yet"); 2068 break; 2069 2070 default: 2071 gcc_unreachable (); 2072 } 2073 } 2074 2075 gcc_checking_assert (!scan_array_reductions 2076 || !is_gimple_omp_oacc (ctx->stmt)); 2077 if (scan_array_reductions) 2078 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 2079 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION 2080 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 2081 { 2082 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx); 2083 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); 2084 } 2085 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE 2086 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) 2087 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); 2088 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 2089 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) 2090 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); 2091 } 2092 2093 /* Create a new name for omp child function. Returns an identifier. If 2094 IS_CILK_FOR is true then the suffix for the child function is 2095 "_cilk_for_fn." */ 2096 2097 static tree 2098 create_omp_child_function_name (bool task_copy, bool is_cilk_for) 2099 { 2100 if (is_cilk_for) 2101 return clone_function_name (current_function_decl, "_cilk_for_fn"); 2102 return clone_function_name (current_function_decl, 2103 task_copy ? "_omp_cpyfn" : "_omp_fn"); 2104 } 2105 2106 /* Returns the type of the induction variable for the child function for 2107 _Cilk_for and the types for _high and _low variables based on TYPE. */ 2108 2109 static tree 2110 cilk_for_check_loop_diff_type (tree type) 2111 { 2112 if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node)) 2113 { 2114 if (TYPE_UNSIGNED (type)) 2115 return uint32_type_node; 2116 else 2117 return integer_type_node; 2118 } 2119 else 2120 { 2121 if (TYPE_UNSIGNED (type)) 2122 return uint64_type_node; 2123 else 2124 return long_long_integer_type_node; 2125 } 2126 } 2127 2128 /* Build a decl for the omp child function. It'll not contain a body 2129 yet, just the bare decl. */ 2130 2131 static void 2132 create_omp_child_function (omp_context *ctx, bool task_copy) 2133 { 2134 tree decl, type, name, t; 2135 2136 tree cilk_for_count 2137 = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL) 2138 ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), 2139 OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE; 2140 tree cilk_var_type = NULL_TREE; 2141 2142 name = create_omp_child_function_name (task_copy, 2143 cilk_for_count != NULL_TREE); 2144 if (task_copy) 2145 type = build_function_type_list (void_type_node, ptr_type_node, 2146 ptr_type_node, NULL_TREE); 2147 else if (cilk_for_count) 2148 { 2149 type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0)); 2150 cilk_var_type = cilk_for_check_loop_diff_type (type); 2151 type = build_function_type_list (void_type_node, ptr_type_node, 2152 cilk_var_type, cilk_var_type, NULL_TREE); 2153 } 2154 else 2155 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE); 2156 2157 decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type); 2158 2159 gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt) 2160 || !task_copy); 2161 if (!task_copy) 2162 ctx->cb.dst_fn = decl; 2163 else 2164 gimple_omp_task_set_copy_fn (ctx->stmt, decl); 2165 2166 TREE_STATIC (decl) = 1; 2167 TREE_USED (decl) = 1; 2168 DECL_ARTIFICIAL (decl) = 1; 2169 DECL_IGNORED_P (decl) = 0; 2170 TREE_PUBLIC (decl) = 0; 2171 DECL_UNINLINABLE (decl) = 1; 2172 DECL_EXTERNAL (decl) = 0; 2173 DECL_CONTEXT (decl) = NULL_TREE; 2174 DECL_INITIAL (decl) = make_node (BLOCK); 2175 if (cgraph_node::get (current_function_decl)->offloadable) 2176 cgraph_node::get_create (decl)->offloadable = 1; 2177 else 2178 { 2179 omp_context *octx; 2180 for (octx = ctx; octx; octx = octx->outer) 2181 if (is_gimple_omp_offloaded (octx->stmt)) 2182 { 2183 cgraph_node::get_create (decl)->offloadable = 1; 2184 #ifdef ENABLE_OFFLOADING 2185 g->have_offload = true; 2186 #endif 2187 break; 2188 } 2189 } 2190 2191 if (cgraph_node::get_create (decl)->offloadable 2192 && !lookup_attribute ("omp declare target", 2193 DECL_ATTRIBUTES (current_function_decl))) 2194 DECL_ATTRIBUTES (decl) 2195 = tree_cons (get_identifier ("omp target entrypoint"), 2196 NULL_TREE, DECL_ATTRIBUTES (decl)); 2197 2198 t = build_decl (DECL_SOURCE_LOCATION (decl), 2199 RESULT_DECL, NULL_TREE, void_type_node); 2200 DECL_ARTIFICIAL (t) = 1; 2201 DECL_IGNORED_P (t) = 1; 2202 DECL_CONTEXT (t) = decl; 2203 DECL_RESULT (decl) = t; 2204 2205 /* _Cilk_for's child function requires two extra parameters called 2206 __low and __high that are set the by Cilk runtime when it calls this 2207 function. */ 2208 if (cilk_for_count) 2209 { 2210 t = build_decl (DECL_SOURCE_LOCATION (decl), 2211 PARM_DECL, get_identifier ("__high"), cilk_var_type); 2212 DECL_ARTIFICIAL (t) = 1; 2213 DECL_NAMELESS (t) = 1; 2214 DECL_ARG_TYPE (t) = ptr_type_node; 2215 DECL_CONTEXT (t) = current_function_decl; 2216 TREE_USED (t) = 1; 2217 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2218 DECL_ARGUMENTS (decl) = t; 2219 2220 t = build_decl (DECL_SOURCE_LOCATION (decl), 2221 PARM_DECL, get_identifier ("__low"), cilk_var_type); 2222 DECL_ARTIFICIAL (t) = 1; 2223 DECL_NAMELESS (t) = 1; 2224 DECL_ARG_TYPE (t) = ptr_type_node; 2225 DECL_CONTEXT (t) = current_function_decl; 2226 TREE_USED (t) = 1; 2227 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2228 DECL_ARGUMENTS (decl) = t; 2229 } 2230 2231 tree data_name = get_identifier (".omp_data_i"); 2232 t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name, 2233 ptr_type_node); 2234 DECL_ARTIFICIAL (t) = 1; 2235 DECL_NAMELESS (t) = 1; 2236 DECL_ARG_TYPE (t) = ptr_type_node; 2237 DECL_CONTEXT (t) = current_function_decl; 2238 TREE_USED (t) = 1; 2239 if (cilk_for_count) 2240 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2241 DECL_ARGUMENTS (decl) = t; 2242 if (!task_copy) 2243 ctx->receiver_decl = t; 2244 else 2245 { 2246 t = build_decl (DECL_SOURCE_LOCATION (decl), 2247 PARM_DECL, get_identifier (".omp_data_o"), 2248 ptr_type_node); 2249 DECL_ARTIFICIAL (t) = 1; 2250 DECL_NAMELESS (t) = 1; 2251 DECL_ARG_TYPE (t) = ptr_type_node; 2252 DECL_CONTEXT (t) = current_function_decl; 2253 TREE_USED (t) = 1; 2254 TREE_ADDRESSABLE (t) = 1; 2255 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2256 DECL_ARGUMENTS (decl) = t; 2257 } 2258 2259 /* Allocate memory for the function structure. The call to 2260 allocate_struct_function clobbers CFUN, so we need to restore 2261 it afterward. */ 2262 push_struct_function (decl); 2263 cfun->function_end_locus = gimple_location (ctx->stmt); 2264 pop_cfun (); 2265 } 2266 2267 /* Callback for walk_gimple_seq. Check if combined parallel 2268 contains gimple_omp_for_combined_into_p OMP_FOR. */ 2269 2270 static tree 2271 find_combined_for (gimple_stmt_iterator *gsi_p, 2272 bool *handled_ops_p, 2273 struct walk_stmt_info *wi) 2274 { 2275 gimple stmt = gsi_stmt (*gsi_p); 2276 2277 *handled_ops_p = true; 2278 switch (gimple_code (stmt)) 2279 { 2280 WALK_SUBSTMTS; 2281 2282 case GIMPLE_OMP_FOR: 2283 if (gimple_omp_for_combined_into_p (stmt) 2284 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR) 2285 { 2286 wi->info = stmt; 2287 return integer_zero_node; 2288 } 2289 break; 2290 default: 2291 break; 2292 } 2293 return NULL; 2294 } 2295 2296 /* Scan an OpenMP parallel directive. */ 2297 2298 static void 2299 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx) 2300 { 2301 omp_context *ctx; 2302 tree name; 2303 gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi)); 2304 2305 /* Ignore parallel directives with empty bodies, unless there 2306 are copyin clauses. */ 2307 if (optimize > 0 2308 && empty_body_p (gimple_omp_body (stmt)) 2309 && find_omp_clause (gimple_omp_parallel_clauses (stmt), 2310 OMP_CLAUSE_COPYIN) == NULL) 2311 { 2312 gsi_replace (gsi, gimple_build_nop (), false); 2313 return; 2314 } 2315 2316 if (gimple_omp_parallel_combined_p (stmt)) 2317 { 2318 struct walk_stmt_info wi; 2319 2320 memset (&wi, 0, sizeof (wi)); 2321 wi.val_only = true; 2322 walk_gimple_seq (gimple_omp_body (stmt), 2323 find_combined_for, NULL, &wi); 2324 if (wi.info) 2325 { 2326 gomp_for *for_stmt = as_a <gomp_for *> ((gimple) wi.info); 2327 struct omp_for_data fd; 2328 extract_omp_for_data (for_stmt, &fd, NULL); 2329 /* We need two temporaries with fd.loop.v type (istart/iend) 2330 and then (fd.collapse - 1) temporaries with the same 2331 type for count2 ... countN-1 vars if not constant. */ 2332 size_t count = 2, i; 2333 tree type = fd.iter_type; 2334 if (fd.collapse > 1 2335 && TREE_CODE (fd.loop.n2) != INTEGER_CST) 2336 count += fd.collapse - 1; 2337 for (i = 0; i < count; i++) 2338 { 2339 tree temp = create_tmp_var (type); 2340 tree c = build_omp_clause (UNKNOWN_LOCATION, 2341 OMP_CLAUSE__LOOPTEMP_); 2342 insert_decl_map (&outer_ctx->cb, temp, temp); 2343 OMP_CLAUSE_DECL (c) = temp; 2344 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt); 2345 gimple_omp_parallel_set_clauses (stmt, c); 2346 } 2347 } 2348 } 2349 2350 ctx = new_omp_context (stmt, outer_ctx); 2351 taskreg_contexts.safe_push (ctx); 2352 if (taskreg_nesting_level > 1) 2353 ctx->is_nested = true; 2354 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2355 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; 2356 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2357 name = create_tmp_var_name (".omp_data_s"); 2358 name = build_decl (gimple_location (stmt), 2359 TYPE_DECL, name, ctx->record_type); 2360 DECL_ARTIFICIAL (name) = 1; 2361 DECL_NAMELESS (name) = 1; 2362 TYPE_NAME (ctx->record_type) = name; 2363 TYPE_ARTIFICIAL (ctx->record_type) = 1; 2364 create_omp_child_function (ctx, false); 2365 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn); 2366 2367 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx); 2368 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2369 2370 if (TYPE_FIELDS (ctx->record_type) == NULL) 2371 ctx->record_type = ctx->receiver_decl = NULL; 2372 } 2373 2374 /* Scan an OpenMP task directive. */ 2375 2376 static void 2377 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx) 2378 { 2379 omp_context *ctx; 2380 tree name, t; 2381 gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi)); 2382 2383 /* Ignore task directives with empty bodies, unless they have depend 2384 clause. */ 2385 if (optimize > 0 2386 && empty_body_p (gimple_omp_body (stmt)) 2387 && !find_omp_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND)) 2388 { 2389 gsi_replace (gsi, gimple_build_nop (), false); 2390 return; 2391 } 2392 2393 ctx = new_omp_context (stmt, outer_ctx); 2394 taskreg_contexts.safe_push (ctx); 2395 if (taskreg_nesting_level > 1) 2396 ctx->is_nested = true; 2397 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2398 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; 2399 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2400 name = create_tmp_var_name (".omp_data_s"); 2401 name = build_decl (gimple_location (stmt), 2402 TYPE_DECL, name, ctx->record_type); 2403 DECL_ARTIFICIAL (name) = 1; 2404 DECL_NAMELESS (name) = 1; 2405 TYPE_NAME (ctx->record_type) = name; 2406 TYPE_ARTIFICIAL (ctx->record_type) = 1; 2407 create_omp_child_function (ctx, false); 2408 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn); 2409 2410 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx); 2411 2412 if (ctx->srecord_type) 2413 { 2414 name = create_tmp_var_name (".omp_data_a"); 2415 name = build_decl (gimple_location (stmt), 2416 TYPE_DECL, name, ctx->srecord_type); 2417 DECL_ARTIFICIAL (name) = 1; 2418 DECL_NAMELESS (name) = 1; 2419 TYPE_NAME (ctx->srecord_type) = name; 2420 TYPE_ARTIFICIAL (ctx->srecord_type) = 1; 2421 create_omp_child_function (ctx, true); 2422 } 2423 2424 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2425 2426 if (TYPE_FIELDS (ctx->record_type) == NULL) 2427 { 2428 ctx->record_type = ctx->receiver_decl = NULL; 2429 t = build_int_cst (long_integer_type_node, 0); 2430 gimple_omp_task_set_arg_size (stmt, t); 2431 t = build_int_cst (long_integer_type_node, 1); 2432 gimple_omp_task_set_arg_align (stmt, t); 2433 } 2434 } 2435 2436 2437 /* If any decls have been made addressable during scan_omp, 2438 adjust their fields if needed, and layout record types 2439 of parallel/task constructs. */ 2440 2441 static void 2442 finish_taskreg_scan (omp_context *ctx) 2443 { 2444 if (ctx->record_type == NULL_TREE) 2445 return; 2446 2447 /* If any task_shared_vars were needed, verify all 2448 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK} 2449 statements if use_pointer_for_field hasn't changed 2450 because of that. If it did, update field types now. */ 2451 if (task_shared_vars) 2452 { 2453 tree c; 2454 2455 for (c = gimple_omp_taskreg_clauses (ctx->stmt); 2456 c; c = OMP_CLAUSE_CHAIN (c)) 2457 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED) 2458 { 2459 tree decl = OMP_CLAUSE_DECL (c); 2460 2461 /* Global variables don't need to be copied, 2462 the receiver side will use them directly. */ 2463 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) 2464 continue; 2465 if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl)) 2466 || !use_pointer_for_field (decl, ctx)) 2467 continue; 2468 tree field = lookup_field (decl, ctx); 2469 if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE 2470 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl)) 2471 continue; 2472 TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl)); 2473 TREE_THIS_VOLATILE (field) = 0; 2474 DECL_USER_ALIGN (field) = 0; 2475 DECL_ALIGN (field) = TYPE_ALIGN (TREE_TYPE (field)); 2476 if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field)) 2477 TYPE_ALIGN (ctx->record_type) = DECL_ALIGN (field); 2478 if (ctx->srecord_type) 2479 { 2480 tree sfield = lookup_sfield (decl, ctx); 2481 TREE_TYPE (sfield) = TREE_TYPE (field); 2482 TREE_THIS_VOLATILE (sfield) = 0; 2483 DECL_USER_ALIGN (sfield) = 0; 2484 DECL_ALIGN (sfield) = DECL_ALIGN (field); 2485 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield)) 2486 TYPE_ALIGN (ctx->srecord_type) = DECL_ALIGN (sfield); 2487 } 2488 } 2489 } 2490 2491 if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL) 2492 { 2493 layout_type (ctx->record_type); 2494 fixup_child_record_type (ctx); 2495 } 2496 else 2497 { 2498 location_t loc = gimple_location (ctx->stmt); 2499 tree *p, vla_fields = NULL_TREE, *q = &vla_fields; 2500 /* Move VLA fields to the end. */ 2501 p = &TYPE_FIELDS (ctx->record_type); 2502 while (*p) 2503 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p)) 2504 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p)))) 2505 { 2506 *q = *p; 2507 *p = TREE_CHAIN (*p); 2508 TREE_CHAIN (*q) = NULL_TREE; 2509 q = &TREE_CHAIN (*q); 2510 } 2511 else 2512 p = &DECL_CHAIN (*p); 2513 *p = vla_fields; 2514 layout_type (ctx->record_type); 2515 fixup_child_record_type (ctx); 2516 if (ctx->srecord_type) 2517 layout_type (ctx->srecord_type); 2518 tree t = fold_convert_loc (loc, long_integer_type_node, 2519 TYPE_SIZE_UNIT (ctx->record_type)); 2520 gimple_omp_task_set_arg_size (ctx->stmt, t); 2521 t = build_int_cst (long_integer_type_node, 2522 TYPE_ALIGN_UNIT (ctx->record_type)); 2523 gimple_omp_task_set_arg_align (ctx->stmt, t); 2524 } 2525 } 2526 2527 2528 static omp_context * 2529 enclosing_target_ctx (omp_context *ctx) 2530 { 2531 while (ctx != NULL 2532 && gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET) 2533 ctx = ctx->outer; 2534 gcc_assert (ctx != NULL); 2535 return ctx; 2536 } 2537 2538 static bool 2539 oacc_loop_or_target_p (gimple stmt) 2540 { 2541 enum gimple_code outer_type = gimple_code (stmt); 2542 return ((outer_type == GIMPLE_OMP_TARGET 2543 && ((gimple_omp_target_kind (stmt) 2544 == GF_OMP_TARGET_KIND_OACC_PARALLEL) 2545 || (gimple_omp_target_kind (stmt) 2546 == GF_OMP_TARGET_KIND_OACC_KERNELS))) 2547 || (outer_type == GIMPLE_OMP_FOR 2548 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)); 2549 } 2550 2551 /* Scan a GIMPLE_OMP_FOR. */ 2552 2553 static void 2554 scan_omp_for (gomp_for *stmt, omp_context *outer_ctx) 2555 { 2556 enum gimple_code outer_type = GIMPLE_ERROR_MARK; 2557 omp_context *ctx; 2558 size_t i; 2559 tree clauses = gimple_omp_for_clauses (stmt); 2560 2561 if (outer_ctx) 2562 outer_type = gimple_code (outer_ctx->stmt); 2563 2564 ctx = new_omp_context (stmt, outer_ctx); 2565 2566 if (is_gimple_omp_oacc (stmt)) 2567 { 2568 if (outer_ctx && outer_type == GIMPLE_OMP_FOR) 2569 ctx->gwv_this = outer_ctx->gwv_this; 2570 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 2571 { 2572 int val; 2573 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_GANG) 2574 val = MASK_GANG; 2575 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WORKER) 2576 val = MASK_WORKER; 2577 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR) 2578 val = MASK_VECTOR; 2579 else 2580 continue; 2581 ctx->gwv_this |= val; 2582 if (!outer_ctx) 2583 { 2584 /* Skip; not nested inside a region. */ 2585 continue; 2586 } 2587 if (!oacc_loop_or_target_p (outer_ctx->stmt)) 2588 { 2589 /* Skip; not nested inside an OpenACC region. */ 2590 continue; 2591 } 2592 if (outer_type == GIMPLE_OMP_FOR) 2593 outer_ctx->gwv_below |= val; 2594 if (OMP_CLAUSE_OPERAND (c, 0) != NULL_TREE) 2595 { 2596 omp_context *enclosing = enclosing_target_ctx (outer_ctx); 2597 if (gimple_omp_target_kind (enclosing->stmt) 2598 == GF_OMP_TARGET_KIND_OACC_PARALLEL) 2599 error_at (gimple_location (stmt), 2600 "no arguments allowed to gang, worker and vector clauses inside parallel"); 2601 } 2602 } 2603 } 2604 2605 scan_sharing_clauses (clauses, ctx); 2606 2607 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx); 2608 for (i = 0; i < gimple_omp_for_collapse (stmt); i++) 2609 { 2610 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx); 2611 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx); 2612 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx); 2613 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx); 2614 } 2615 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2616 2617 if (is_gimple_omp_oacc (stmt)) 2618 { 2619 if (ctx->gwv_this & ctx->gwv_below) 2620 error_at (gimple_location (stmt), 2621 "gang, worker and vector may occur only once in a loop nest"); 2622 else if (ctx->gwv_below != 0 2623 && ctx->gwv_this > ctx->gwv_below) 2624 error_at (gimple_location (stmt), 2625 "gang, worker and vector must occur in this order in a loop nest"); 2626 if (outer_ctx && outer_type == GIMPLE_OMP_FOR) 2627 outer_ctx->gwv_below |= ctx->gwv_below; 2628 } 2629 } 2630 2631 /* Scan an OpenMP sections directive. */ 2632 2633 static void 2634 scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx) 2635 { 2636 omp_context *ctx; 2637 2638 ctx = new_omp_context (stmt, outer_ctx); 2639 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx); 2640 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2641 } 2642 2643 /* Scan an OpenMP single directive. */ 2644 2645 static void 2646 scan_omp_single (gomp_single *stmt, omp_context *outer_ctx) 2647 { 2648 omp_context *ctx; 2649 tree name; 2650 2651 ctx = new_omp_context (stmt, outer_ctx); 2652 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2653 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2654 name = create_tmp_var_name (".omp_copy_s"); 2655 name = build_decl (gimple_location (stmt), 2656 TYPE_DECL, name, ctx->record_type); 2657 TYPE_NAME (ctx->record_type) = name; 2658 2659 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx); 2660 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2661 2662 if (TYPE_FIELDS (ctx->record_type) == NULL) 2663 ctx->record_type = NULL; 2664 else 2665 layout_type (ctx->record_type); 2666 } 2667 2668 /* Scan a GIMPLE_OMP_TARGET. */ 2669 2670 static void 2671 scan_omp_target (gomp_target *stmt, omp_context *outer_ctx) 2672 { 2673 omp_context *ctx; 2674 tree name; 2675 bool offloaded = is_gimple_omp_offloaded (stmt); 2676 tree clauses = gimple_omp_target_clauses (stmt); 2677 2678 ctx = new_omp_context (stmt, outer_ctx); 2679 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2680 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; 2681 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2682 name = create_tmp_var_name (".omp_data_t"); 2683 name = build_decl (gimple_location (stmt), 2684 TYPE_DECL, name, ctx->record_type); 2685 DECL_ARTIFICIAL (name) = 1; 2686 DECL_NAMELESS (name) = 1; 2687 TYPE_NAME (ctx->record_type) = name; 2688 TYPE_ARTIFICIAL (ctx->record_type) = 1; 2689 if (offloaded) 2690 { 2691 if (is_gimple_omp_oacc (stmt)) 2692 ctx->reduction_map = splay_tree_new (splay_tree_compare_pointers, 2693 0, 0); 2694 2695 create_omp_child_function (ctx, false); 2696 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn); 2697 } 2698 2699 if (is_gimple_omp_oacc (stmt)) 2700 { 2701 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 2702 { 2703 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_GANGS) 2704 ctx->gwv_this |= MASK_GANG; 2705 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_WORKERS) 2706 ctx->gwv_this |= MASK_WORKER; 2707 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR_LENGTH) 2708 ctx->gwv_this |= MASK_VECTOR; 2709 } 2710 } 2711 2712 scan_sharing_clauses (clauses, ctx); 2713 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2714 2715 if (TYPE_FIELDS (ctx->record_type) == NULL) 2716 ctx->record_type = ctx->receiver_decl = NULL; 2717 else 2718 { 2719 TYPE_FIELDS (ctx->record_type) 2720 = nreverse (TYPE_FIELDS (ctx->record_type)); 2721 #ifdef ENABLE_CHECKING 2722 tree field; 2723 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type)); 2724 for (field = TYPE_FIELDS (ctx->record_type); 2725 field; 2726 field = DECL_CHAIN (field)) 2727 gcc_assert (DECL_ALIGN (field) == align); 2728 #endif 2729 layout_type (ctx->record_type); 2730 if (offloaded) 2731 fixup_child_record_type (ctx); 2732 } 2733 } 2734 2735 /* Scan an OpenMP teams directive. */ 2736 2737 static void 2738 scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx) 2739 { 2740 omp_context *ctx = new_omp_context (stmt, outer_ctx); 2741 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx); 2742 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2743 } 2744 2745 /* Check nesting restrictions. */ 2746 static bool 2747 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx) 2748 { 2749 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin) 2750 inside an OpenACC CTX. */ 2751 if (!(is_gimple_omp (stmt) 2752 && is_gimple_omp_oacc (stmt))) 2753 { 2754 for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer) 2755 if (is_gimple_omp (ctx_->stmt) 2756 && is_gimple_omp_oacc (ctx_->stmt)) 2757 { 2758 error_at (gimple_location (stmt), 2759 "non-OpenACC construct inside of OpenACC region"); 2760 return false; 2761 } 2762 } 2763 2764 if (ctx != NULL) 2765 { 2766 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 2767 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 2768 { 2769 error_at (gimple_location (stmt), 2770 "OpenMP constructs may not be nested inside simd region"); 2771 return false; 2772 } 2773 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 2774 { 2775 if ((gimple_code (stmt) != GIMPLE_OMP_FOR 2776 || (gimple_omp_for_kind (stmt) 2777 != GF_OMP_FOR_KIND_DISTRIBUTE)) 2778 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL) 2779 { 2780 error_at (gimple_location (stmt), 2781 "only distribute or parallel constructs are allowed to " 2782 "be closely nested inside teams construct"); 2783 return false; 2784 } 2785 } 2786 } 2787 switch (gimple_code (stmt)) 2788 { 2789 case GIMPLE_OMP_FOR: 2790 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD) 2791 return true; 2792 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE) 2793 { 2794 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS) 2795 { 2796 error_at (gimple_location (stmt), 2797 "distribute construct must be closely nested inside " 2798 "teams construct"); 2799 return false; 2800 } 2801 return true; 2802 } 2803 /* FALLTHRU */ 2804 case GIMPLE_CALL: 2805 if (is_gimple_call (stmt) 2806 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2807 == BUILT_IN_GOMP_CANCEL 2808 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2809 == BUILT_IN_GOMP_CANCELLATION_POINT)) 2810 { 2811 const char *bad = NULL; 2812 const char *kind = NULL; 2813 if (ctx == NULL) 2814 { 2815 error_at (gimple_location (stmt), "orphaned %qs construct", 2816 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2817 == BUILT_IN_GOMP_CANCEL 2818 ? "#pragma omp cancel" 2819 : "#pragma omp cancellation point"); 2820 return false; 2821 } 2822 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0)) 2823 ? tree_to_shwi (gimple_call_arg (stmt, 0)) 2824 : 0) 2825 { 2826 case 1: 2827 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL) 2828 bad = "#pragma omp parallel"; 2829 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2830 == BUILT_IN_GOMP_CANCEL 2831 && !integer_zerop (gimple_call_arg (stmt, 1))) 2832 ctx->cancellable = true; 2833 kind = "parallel"; 2834 break; 2835 case 2: 2836 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR 2837 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR) 2838 bad = "#pragma omp for"; 2839 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2840 == BUILT_IN_GOMP_CANCEL 2841 && !integer_zerop (gimple_call_arg (stmt, 1))) 2842 { 2843 ctx->cancellable = true; 2844 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 2845 OMP_CLAUSE_NOWAIT)) 2846 warning_at (gimple_location (stmt), 0, 2847 "%<#pragma omp cancel for%> inside " 2848 "%<nowait%> for construct"); 2849 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 2850 OMP_CLAUSE_ORDERED)) 2851 warning_at (gimple_location (stmt), 0, 2852 "%<#pragma omp cancel for%> inside " 2853 "%<ordered%> for construct"); 2854 } 2855 kind = "for"; 2856 break; 2857 case 4: 2858 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS 2859 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION) 2860 bad = "#pragma omp sections"; 2861 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2862 == BUILT_IN_GOMP_CANCEL 2863 && !integer_zerop (gimple_call_arg (stmt, 1))) 2864 { 2865 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS) 2866 { 2867 ctx->cancellable = true; 2868 if (find_omp_clause (gimple_omp_sections_clauses 2869 (ctx->stmt), 2870 OMP_CLAUSE_NOWAIT)) 2871 warning_at (gimple_location (stmt), 0, 2872 "%<#pragma omp cancel sections%> inside " 2873 "%<nowait%> sections construct"); 2874 } 2875 else 2876 { 2877 gcc_assert (ctx->outer 2878 && gimple_code (ctx->outer->stmt) 2879 == GIMPLE_OMP_SECTIONS); 2880 ctx->outer->cancellable = true; 2881 if (find_omp_clause (gimple_omp_sections_clauses 2882 (ctx->outer->stmt), 2883 OMP_CLAUSE_NOWAIT)) 2884 warning_at (gimple_location (stmt), 0, 2885 "%<#pragma omp cancel sections%> inside " 2886 "%<nowait%> sections construct"); 2887 } 2888 } 2889 kind = "sections"; 2890 break; 2891 case 8: 2892 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK) 2893 bad = "#pragma omp task"; 2894 else 2895 ctx->cancellable = true; 2896 kind = "taskgroup"; 2897 break; 2898 default: 2899 error_at (gimple_location (stmt), "invalid arguments"); 2900 return false; 2901 } 2902 if (bad) 2903 { 2904 error_at (gimple_location (stmt), 2905 "%<%s %s%> construct not closely nested inside of %qs", 2906 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2907 == BUILT_IN_GOMP_CANCEL 2908 ? "#pragma omp cancel" 2909 : "#pragma omp cancellation point", kind, bad); 2910 return false; 2911 } 2912 } 2913 /* FALLTHRU */ 2914 case GIMPLE_OMP_SECTIONS: 2915 case GIMPLE_OMP_SINGLE: 2916 for (; ctx != NULL; ctx = ctx->outer) 2917 switch (gimple_code (ctx->stmt)) 2918 { 2919 case GIMPLE_OMP_FOR: 2920 case GIMPLE_OMP_SECTIONS: 2921 case GIMPLE_OMP_SINGLE: 2922 case GIMPLE_OMP_ORDERED: 2923 case GIMPLE_OMP_MASTER: 2924 case GIMPLE_OMP_TASK: 2925 case GIMPLE_OMP_CRITICAL: 2926 if (is_gimple_call (stmt)) 2927 { 2928 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2929 != BUILT_IN_GOMP_BARRIER) 2930 return true; 2931 error_at (gimple_location (stmt), 2932 "barrier region may not be closely nested inside " 2933 "of work-sharing, critical, ordered, master or " 2934 "explicit task region"); 2935 return false; 2936 } 2937 error_at (gimple_location (stmt), 2938 "work-sharing region may not be closely nested inside " 2939 "of work-sharing, critical, ordered, master or explicit " 2940 "task region"); 2941 return false; 2942 case GIMPLE_OMP_PARALLEL: 2943 return true; 2944 default: 2945 break; 2946 } 2947 break; 2948 case GIMPLE_OMP_MASTER: 2949 for (; ctx != NULL; ctx = ctx->outer) 2950 switch (gimple_code (ctx->stmt)) 2951 { 2952 case GIMPLE_OMP_FOR: 2953 case GIMPLE_OMP_SECTIONS: 2954 case GIMPLE_OMP_SINGLE: 2955 case GIMPLE_OMP_TASK: 2956 error_at (gimple_location (stmt), 2957 "master region may not be closely nested inside " 2958 "of work-sharing or explicit task region"); 2959 return false; 2960 case GIMPLE_OMP_PARALLEL: 2961 return true; 2962 default: 2963 break; 2964 } 2965 break; 2966 case GIMPLE_OMP_ORDERED: 2967 for (; ctx != NULL; ctx = ctx->outer) 2968 switch (gimple_code (ctx->stmt)) 2969 { 2970 case GIMPLE_OMP_CRITICAL: 2971 case GIMPLE_OMP_TASK: 2972 error_at (gimple_location (stmt), 2973 "ordered region may not be closely nested inside " 2974 "of critical or explicit task region"); 2975 return false; 2976 case GIMPLE_OMP_FOR: 2977 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 2978 OMP_CLAUSE_ORDERED) == NULL) 2979 { 2980 error_at (gimple_location (stmt), 2981 "ordered region must be closely nested inside " 2982 "a loop region with an ordered clause"); 2983 return false; 2984 } 2985 return true; 2986 case GIMPLE_OMP_PARALLEL: 2987 error_at (gimple_location (stmt), 2988 "ordered region must be closely nested inside " 2989 "a loop region with an ordered clause"); 2990 return false; 2991 default: 2992 break; 2993 } 2994 break; 2995 case GIMPLE_OMP_CRITICAL: 2996 { 2997 tree this_stmt_name 2998 = gimple_omp_critical_name (as_a <gomp_critical *> (stmt)); 2999 for (; ctx != NULL; ctx = ctx->outer) 3000 if (gomp_critical *other_crit 3001 = dyn_cast <gomp_critical *> (ctx->stmt)) 3002 if (this_stmt_name == gimple_omp_critical_name (other_crit)) 3003 { 3004 error_at (gimple_location (stmt), 3005 "critical region may not be nested inside a critical " 3006 "region with the same name"); 3007 return false; 3008 } 3009 } 3010 break; 3011 case GIMPLE_OMP_TEAMS: 3012 if (ctx == NULL 3013 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET 3014 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION) 3015 { 3016 error_at (gimple_location (stmt), 3017 "teams construct not closely nested inside of target " 3018 "region"); 3019 return false; 3020 } 3021 break; 3022 case GIMPLE_OMP_TARGET: 3023 for (; ctx != NULL; ctx = ctx->outer) 3024 { 3025 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET) 3026 { 3027 if (is_gimple_omp (stmt) 3028 && is_gimple_omp_oacc (stmt) 3029 && is_gimple_omp (ctx->stmt)) 3030 { 3031 error_at (gimple_location (stmt), 3032 "OpenACC construct inside of non-OpenACC region"); 3033 return false; 3034 } 3035 continue; 3036 } 3037 3038 const char *stmt_name, *ctx_stmt_name; 3039 switch (gimple_omp_target_kind (stmt)) 3040 { 3041 case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break; 3042 case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break; 3043 case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break; 3044 case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break; 3045 case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break; 3046 case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break; 3047 case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break; 3048 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: stmt_name = "enter/exit data"; break; 3049 default: gcc_unreachable (); 3050 } 3051 switch (gimple_omp_target_kind (ctx->stmt)) 3052 { 3053 case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break; 3054 case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break; 3055 case GF_OMP_TARGET_KIND_OACC_PARALLEL: ctx_stmt_name = "parallel"; break; 3056 case GF_OMP_TARGET_KIND_OACC_KERNELS: ctx_stmt_name = "kernels"; break; 3057 case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break; 3058 default: gcc_unreachable (); 3059 } 3060 3061 /* OpenACC/OpenMP mismatch? */ 3062 if (is_gimple_omp_oacc (stmt) 3063 != is_gimple_omp_oacc (ctx->stmt)) 3064 { 3065 error_at (gimple_location (stmt), 3066 "%s %s construct inside of %s %s region", 3067 (is_gimple_omp_oacc (stmt) 3068 ? "OpenACC" : "OpenMP"), stmt_name, 3069 (is_gimple_omp_oacc (ctx->stmt) 3070 ? "OpenACC" : "OpenMP"), ctx_stmt_name); 3071 return false; 3072 } 3073 if (is_gimple_omp_offloaded (ctx->stmt)) 3074 { 3075 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */ 3076 if (is_gimple_omp_oacc (ctx->stmt)) 3077 { 3078 error_at (gimple_location (stmt), 3079 "%s construct inside of %s region", 3080 stmt_name, ctx_stmt_name); 3081 return false; 3082 } 3083 else 3084 { 3085 gcc_checking_assert (!is_gimple_omp_oacc (stmt)); 3086 warning_at (gimple_location (stmt), 0, 3087 "%s construct inside of %s region", 3088 stmt_name, ctx_stmt_name); 3089 } 3090 } 3091 } 3092 break; 3093 default: 3094 break; 3095 } 3096 return true; 3097 } 3098 3099 3100 /* Helper function scan_omp. 3101 3102 Callback for walk_tree or operators in walk_gimple_stmt used to 3103 scan for OMP directives in TP. */ 3104 3105 static tree 3106 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data) 3107 { 3108 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 3109 omp_context *ctx = (omp_context *) wi->info; 3110 tree t = *tp; 3111 3112 switch (TREE_CODE (t)) 3113 { 3114 case VAR_DECL: 3115 case PARM_DECL: 3116 case LABEL_DECL: 3117 case RESULT_DECL: 3118 if (ctx) 3119 *tp = remap_decl (t, &ctx->cb); 3120 break; 3121 3122 default: 3123 if (ctx && TYPE_P (t)) 3124 *tp = remap_type (t, &ctx->cb); 3125 else if (!DECL_P (t)) 3126 { 3127 *walk_subtrees = 1; 3128 if (ctx) 3129 { 3130 tree tem = remap_type (TREE_TYPE (t), &ctx->cb); 3131 if (tem != TREE_TYPE (t)) 3132 { 3133 if (TREE_CODE (t) == INTEGER_CST) 3134 *tp = wide_int_to_tree (tem, t); 3135 else 3136 TREE_TYPE (t) = tem; 3137 } 3138 } 3139 } 3140 break; 3141 } 3142 3143 return NULL_TREE; 3144 } 3145 3146 /* Return true if FNDECL is a setjmp or a longjmp. */ 3147 3148 static bool 3149 setjmp_or_longjmp_p (const_tree fndecl) 3150 { 3151 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL 3152 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP 3153 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP)) 3154 return true; 3155 3156 tree declname = DECL_NAME (fndecl); 3157 if (!declname) 3158 return false; 3159 const char *name = IDENTIFIER_POINTER (declname); 3160 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp"); 3161 } 3162 3163 3164 /* Helper function for scan_omp. 3165 3166 Callback for walk_gimple_stmt used to scan for OMP directives in 3167 the current statement in GSI. */ 3168 3169 static tree 3170 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, 3171 struct walk_stmt_info *wi) 3172 { 3173 gimple stmt = gsi_stmt (*gsi); 3174 omp_context *ctx = (omp_context *) wi->info; 3175 3176 if (gimple_has_location (stmt)) 3177 input_location = gimple_location (stmt); 3178 3179 /* Check the nesting restrictions. */ 3180 bool remove = false; 3181 if (is_gimple_omp (stmt)) 3182 remove = !check_omp_nesting_restrictions (stmt, ctx); 3183 else if (is_gimple_call (stmt)) 3184 { 3185 tree fndecl = gimple_call_fndecl (stmt); 3186 if (fndecl) 3187 { 3188 if (setjmp_or_longjmp_p (fndecl) 3189 && ctx 3190 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 3191 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 3192 { 3193 remove = true; 3194 error_at (gimple_location (stmt), 3195 "setjmp/longjmp inside simd construct"); 3196 } 3197 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 3198 switch (DECL_FUNCTION_CODE (fndecl)) 3199 { 3200 case BUILT_IN_GOMP_BARRIER: 3201 case BUILT_IN_GOMP_CANCEL: 3202 case BUILT_IN_GOMP_CANCELLATION_POINT: 3203 case BUILT_IN_GOMP_TASKYIELD: 3204 case BUILT_IN_GOMP_TASKWAIT: 3205 case BUILT_IN_GOMP_TASKGROUP_START: 3206 case BUILT_IN_GOMP_TASKGROUP_END: 3207 remove = !check_omp_nesting_restrictions (stmt, ctx); 3208 break; 3209 default: 3210 break; 3211 } 3212 } 3213 } 3214 if (remove) 3215 { 3216 stmt = gimple_build_nop (); 3217 gsi_replace (gsi, stmt, false); 3218 } 3219 3220 *handled_ops_p = true; 3221 3222 switch (gimple_code (stmt)) 3223 { 3224 case GIMPLE_OMP_PARALLEL: 3225 taskreg_nesting_level++; 3226 scan_omp_parallel (gsi, ctx); 3227 taskreg_nesting_level--; 3228 break; 3229 3230 case GIMPLE_OMP_TASK: 3231 taskreg_nesting_level++; 3232 scan_omp_task (gsi, ctx); 3233 taskreg_nesting_level--; 3234 break; 3235 3236 case GIMPLE_OMP_FOR: 3237 scan_omp_for (as_a <gomp_for *> (stmt), ctx); 3238 break; 3239 3240 case GIMPLE_OMP_SECTIONS: 3241 scan_omp_sections (as_a <gomp_sections *> (stmt), ctx); 3242 break; 3243 3244 case GIMPLE_OMP_SINGLE: 3245 scan_omp_single (as_a <gomp_single *> (stmt), ctx); 3246 break; 3247 3248 case GIMPLE_OMP_SECTION: 3249 case GIMPLE_OMP_MASTER: 3250 case GIMPLE_OMP_TASKGROUP: 3251 case GIMPLE_OMP_ORDERED: 3252 case GIMPLE_OMP_CRITICAL: 3253 ctx = new_omp_context (stmt, ctx); 3254 scan_omp (gimple_omp_body_ptr (stmt), ctx); 3255 break; 3256 3257 case GIMPLE_OMP_TARGET: 3258 scan_omp_target (as_a <gomp_target *> (stmt), ctx); 3259 break; 3260 3261 case GIMPLE_OMP_TEAMS: 3262 scan_omp_teams (as_a <gomp_teams *> (stmt), ctx); 3263 break; 3264 3265 case GIMPLE_BIND: 3266 { 3267 tree var; 3268 3269 *handled_ops_p = false; 3270 if (ctx) 3271 for (var = gimple_bind_vars (as_a <gbind *> (stmt)); 3272 var ; 3273 var = DECL_CHAIN (var)) 3274 insert_decl_map (&ctx->cb, var, var); 3275 } 3276 break; 3277 default: 3278 *handled_ops_p = false; 3279 break; 3280 } 3281 3282 return NULL_TREE; 3283 } 3284 3285 3286 /* Scan all the statements starting at the current statement. CTX 3287 contains context information about the OMP directives and 3288 clauses found during the scan. */ 3289 3290 static void 3291 scan_omp (gimple_seq *body_p, omp_context *ctx) 3292 { 3293 location_t saved_location; 3294 struct walk_stmt_info wi; 3295 3296 memset (&wi, 0, sizeof (wi)); 3297 wi.info = ctx; 3298 wi.want_locations = true; 3299 3300 saved_location = input_location; 3301 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi); 3302 input_location = saved_location; 3303 } 3304 3305 /* Re-gimplification and code generation routines. */ 3306 3307 /* Build a call to GOMP_barrier. */ 3308 3309 static gimple 3310 build_omp_barrier (tree lhs) 3311 { 3312 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL 3313 : BUILT_IN_GOMP_BARRIER); 3314 gcall *g = gimple_build_call (fndecl, 0); 3315 if (lhs) 3316 gimple_call_set_lhs (g, lhs); 3317 return g; 3318 } 3319 3320 /* If a context was created for STMT when it was scanned, return it. */ 3321 3322 static omp_context * 3323 maybe_lookup_ctx (gimple stmt) 3324 { 3325 splay_tree_node n; 3326 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt); 3327 return n ? (omp_context *) n->value : NULL; 3328 } 3329 3330 3331 /* Find the mapping for DECL in CTX or the immediately enclosing 3332 context that has a mapping for DECL. 3333 3334 If CTX is a nested parallel directive, we may have to use the decl 3335 mappings created in CTX's parent context. Suppose that we have the 3336 following parallel nesting (variable UIDs showed for clarity): 3337 3338 iD.1562 = 0; 3339 #omp parallel shared(iD.1562) -> outer parallel 3340 iD.1562 = iD.1562 + 1; 3341 3342 #omp parallel shared (iD.1562) -> inner parallel 3343 iD.1562 = iD.1562 - 1; 3344 3345 Each parallel structure will create a distinct .omp_data_s structure 3346 for copying iD.1562 in/out of the directive: 3347 3348 outer parallel .omp_data_s.1.i -> iD.1562 3349 inner parallel .omp_data_s.2.i -> iD.1562 3350 3351 A shared variable mapping will produce a copy-out operation before 3352 the parallel directive and a copy-in operation after it. So, in 3353 this case we would have: 3354 3355 iD.1562 = 0; 3356 .omp_data_o.1.i = iD.1562; 3357 #omp parallel shared(iD.1562) -> outer parallel 3358 .omp_data_i.1 = &.omp_data_o.1 3359 .omp_data_i.1->i = .omp_data_i.1->i + 1; 3360 3361 .omp_data_o.2.i = iD.1562; -> ** 3362 #omp parallel shared(iD.1562) -> inner parallel 3363 .omp_data_i.2 = &.omp_data_o.2 3364 .omp_data_i.2->i = .omp_data_i.2->i - 1; 3365 3366 3367 ** This is a problem. The symbol iD.1562 cannot be referenced 3368 inside the body of the outer parallel region. But since we are 3369 emitting this copy operation while expanding the inner parallel 3370 directive, we need to access the CTX structure of the outer 3371 parallel directive to get the correct mapping: 3372 3373 .omp_data_o.2.i = .omp_data_i.1->i 3374 3375 Since there may be other workshare or parallel directives enclosing 3376 the parallel directive, it may be necessary to walk up the context 3377 parent chain. This is not a problem in general because nested 3378 parallelism happens only rarely. */ 3379 3380 static tree 3381 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) 3382 { 3383 tree t; 3384 omp_context *up; 3385 3386 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) 3387 t = maybe_lookup_decl (decl, up); 3388 3389 gcc_assert (!ctx->is_nested || t || is_global_var (decl)); 3390 3391 return t ? t : decl; 3392 } 3393 3394 3395 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found 3396 in outer contexts. */ 3397 3398 static tree 3399 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) 3400 { 3401 tree t = NULL; 3402 omp_context *up; 3403 3404 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) 3405 t = maybe_lookup_decl (decl, up); 3406 3407 return t ? t : decl; 3408 } 3409 3410 3411 /* Construct the initialization value for reduction CLAUSE. */ 3412 3413 tree 3414 omp_reduction_init (tree clause, tree type) 3415 { 3416 location_t loc = OMP_CLAUSE_LOCATION (clause); 3417 switch (OMP_CLAUSE_REDUCTION_CODE (clause)) 3418 { 3419 case PLUS_EXPR: 3420 case MINUS_EXPR: 3421 case BIT_IOR_EXPR: 3422 case BIT_XOR_EXPR: 3423 case TRUTH_OR_EXPR: 3424 case TRUTH_ORIF_EXPR: 3425 case TRUTH_XOR_EXPR: 3426 case NE_EXPR: 3427 return build_zero_cst (type); 3428 3429 case MULT_EXPR: 3430 case TRUTH_AND_EXPR: 3431 case TRUTH_ANDIF_EXPR: 3432 case EQ_EXPR: 3433 return fold_convert_loc (loc, type, integer_one_node); 3434 3435 case BIT_AND_EXPR: 3436 return fold_convert_loc (loc, type, integer_minus_one_node); 3437 3438 case MAX_EXPR: 3439 if (SCALAR_FLOAT_TYPE_P (type)) 3440 { 3441 REAL_VALUE_TYPE max, min; 3442 if (HONOR_INFINITIES (type)) 3443 { 3444 real_inf (&max); 3445 real_arithmetic (&min, NEGATE_EXPR, &max, NULL); 3446 } 3447 else 3448 real_maxval (&min, 1, TYPE_MODE (type)); 3449 return build_real (type, min); 3450 } 3451 else 3452 { 3453 gcc_assert (INTEGRAL_TYPE_P (type)); 3454 return TYPE_MIN_VALUE (type); 3455 } 3456 3457 case MIN_EXPR: 3458 if (SCALAR_FLOAT_TYPE_P (type)) 3459 { 3460 REAL_VALUE_TYPE max; 3461 if (HONOR_INFINITIES (type)) 3462 real_inf (&max); 3463 else 3464 real_maxval (&max, 0, TYPE_MODE (type)); 3465 return build_real (type, max); 3466 } 3467 else 3468 { 3469 gcc_assert (INTEGRAL_TYPE_P (type)); 3470 return TYPE_MAX_VALUE (type); 3471 } 3472 3473 default: 3474 gcc_unreachable (); 3475 } 3476 } 3477 3478 /* Return alignment to be assumed for var in CLAUSE, which should be 3479 OMP_CLAUSE_ALIGNED. */ 3480 3481 static tree 3482 omp_clause_aligned_alignment (tree clause) 3483 { 3484 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause)) 3485 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause); 3486 3487 /* Otherwise return implementation defined alignment. */ 3488 unsigned int al = 1; 3489 machine_mode mode, vmode; 3490 int vs = targetm.vectorize.autovectorize_vector_sizes (); 3491 if (vs) 3492 vs = 1 << floor_log2 (vs); 3493 static enum mode_class classes[] 3494 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT }; 3495 for (int i = 0; i < 4; i += 2) 3496 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]); 3497 mode != VOIDmode; 3498 mode = GET_MODE_WIDER_MODE (mode)) 3499 { 3500 vmode = targetm.vectorize.preferred_simd_mode (mode); 3501 if (GET_MODE_CLASS (vmode) != classes[i + 1]) 3502 continue; 3503 while (vs 3504 && GET_MODE_SIZE (vmode) < vs 3505 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode) 3506 vmode = GET_MODE_2XWIDER_MODE (vmode); 3507 3508 tree type = lang_hooks.types.type_for_mode (mode, 1); 3509 if (type == NULL_TREE || TYPE_MODE (type) != mode) 3510 continue; 3511 type = build_vector_type (type, GET_MODE_SIZE (vmode) 3512 / GET_MODE_SIZE (mode)); 3513 if (TYPE_MODE (type) != vmode) 3514 continue; 3515 if (TYPE_ALIGN_UNIT (type) > al) 3516 al = TYPE_ALIGN_UNIT (type); 3517 } 3518 return build_int_cst (integer_type_node, al); 3519 } 3520 3521 /* Return maximum possible vectorization factor for the target. */ 3522 3523 static int 3524 omp_max_vf (void) 3525 { 3526 if (!optimize 3527 || optimize_debug 3528 || !flag_tree_loop_optimize 3529 || (!flag_tree_loop_vectorize 3530 && (global_options_set.x_flag_tree_loop_vectorize 3531 || global_options_set.x_flag_tree_vectorize))) 3532 return 1; 3533 3534 int vs = targetm.vectorize.autovectorize_vector_sizes (); 3535 if (vs) 3536 { 3537 vs = 1 << floor_log2 (vs); 3538 return vs; 3539 } 3540 machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode); 3541 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT) 3542 return GET_MODE_NUNITS (vqimode); 3543 return 1; 3544 } 3545 3546 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd 3547 privatization. */ 3548 3549 static bool 3550 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf, 3551 tree &idx, tree &lane, tree &ivar, tree &lvar) 3552 { 3553 if (max_vf == 0) 3554 { 3555 max_vf = omp_max_vf (); 3556 if (max_vf > 1) 3557 { 3558 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 3559 OMP_CLAUSE_SAFELEN); 3560 if (c 3561 && (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST 3562 || tree_int_cst_sgn (OMP_CLAUSE_SAFELEN_EXPR (c)) != 1)) 3563 max_vf = 1; 3564 else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), 3565 max_vf) == -1) 3566 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c)); 3567 } 3568 if (max_vf > 1) 3569 { 3570 idx = create_tmp_var (unsigned_type_node); 3571 lane = create_tmp_var (unsigned_type_node); 3572 } 3573 } 3574 if (max_vf == 1) 3575 return false; 3576 3577 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf); 3578 tree avar = create_tmp_var_raw (atype); 3579 if (TREE_ADDRESSABLE (new_var)) 3580 TREE_ADDRESSABLE (avar) = 1; 3581 DECL_ATTRIBUTES (avar) 3582 = tree_cons (get_identifier ("omp simd array"), NULL, 3583 DECL_ATTRIBUTES (avar)); 3584 gimple_add_tmp_var (avar); 3585 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx, 3586 NULL_TREE, NULL_TREE); 3587 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane, 3588 NULL_TREE, NULL_TREE); 3589 if (DECL_P (new_var)) 3590 { 3591 SET_DECL_VALUE_EXPR (new_var, lvar); 3592 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3593 } 3594 return true; 3595 } 3596 3597 /* Helper function of lower_rec_input_clauses. For a reference 3598 in simd reduction, add an underlying variable it will reference. */ 3599 3600 static void 3601 handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist) 3602 { 3603 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard))); 3604 if (TREE_CONSTANT (z)) 3605 { 3606 const char *name = NULL; 3607 if (DECL_NAME (new_vard)) 3608 name = IDENTIFIER_POINTER (DECL_NAME (new_vard)); 3609 3610 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name); 3611 gimple_add_tmp_var (z); 3612 TREE_ADDRESSABLE (z) = 1; 3613 z = build_fold_addr_expr_loc (loc, z); 3614 gimplify_assign (new_vard, z, ilist); 3615 } 3616 } 3617 3618 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN, 3619 from the receiver (aka child) side and initializers for REFERENCE_TYPE 3620 private variables. Initialization statements go in ILIST, while calls 3621 to destructors go in DLIST. */ 3622 3623 static void 3624 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, 3625 omp_context *ctx, struct omp_for_data *fd) 3626 { 3627 tree c, dtor, copyin_seq, x, ptr; 3628 bool copyin_by_ref = false; 3629 bool lastprivate_firstprivate = false; 3630 bool reduction_omp_orig_ref = false; 3631 int pass; 3632 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 3633 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD); 3634 int max_vf = 0; 3635 tree lane = NULL_TREE, idx = NULL_TREE; 3636 tree ivar = NULL_TREE, lvar = NULL_TREE; 3637 gimple_seq llist[2] = { NULL, NULL }; 3638 3639 copyin_seq = NULL; 3640 3641 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops 3642 with data sharing clauses referencing variable sized vars. That 3643 is unnecessarily hard to support and very unlikely to result in 3644 vectorized code anyway. */ 3645 if (is_simd) 3646 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 3647 switch (OMP_CLAUSE_CODE (c)) 3648 { 3649 case OMP_CLAUSE_LINEAR: 3650 if (OMP_CLAUSE_LINEAR_ARRAY (c)) 3651 max_vf = 1; 3652 /* FALLTHRU */ 3653 case OMP_CLAUSE_REDUCTION: 3654 case OMP_CLAUSE_PRIVATE: 3655 case OMP_CLAUSE_FIRSTPRIVATE: 3656 case OMP_CLAUSE_LASTPRIVATE: 3657 if (is_variable_sized (OMP_CLAUSE_DECL (c))) 3658 max_vf = 1; 3659 break; 3660 default: 3661 continue; 3662 } 3663 3664 /* Do all the fixed sized types in the first pass, and the variable sized 3665 types in the second pass. This makes sure that the scalar arguments to 3666 the variable sized types are processed before we use them in the 3667 variable sized operations. */ 3668 for (pass = 0; pass < 2; ++pass) 3669 { 3670 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 3671 { 3672 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c); 3673 tree var, new_var; 3674 bool by_ref; 3675 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 3676 3677 switch (c_kind) 3678 { 3679 case OMP_CLAUSE_PRIVATE: 3680 if (OMP_CLAUSE_PRIVATE_DEBUG (c)) 3681 continue; 3682 break; 3683 case OMP_CLAUSE_SHARED: 3684 /* Ignore shared directives in teams construct. */ 3685 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 3686 continue; 3687 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL) 3688 { 3689 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c))); 3690 continue; 3691 } 3692 case OMP_CLAUSE_FIRSTPRIVATE: 3693 case OMP_CLAUSE_COPYIN: 3694 case OMP_CLAUSE_LINEAR: 3695 break; 3696 case OMP_CLAUSE_REDUCTION: 3697 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) 3698 reduction_omp_orig_ref = true; 3699 break; 3700 case OMP_CLAUSE__LOOPTEMP_: 3701 /* Handle _looptemp_ clauses only on parallel. */ 3702 if (fd) 3703 continue; 3704 break; 3705 case OMP_CLAUSE_LASTPRIVATE: 3706 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 3707 { 3708 lastprivate_firstprivate = true; 3709 if (pass != 0) 3710 continue; 3711 } 3712 /* Even without corresponding firstprivate, if 3713 decl is Fortran allocatable, it needs outer var 3714 reference. */ 3715 else if (pass == 0 3716 && lang_hooks.decls.omp_private_outer_ref 3717 (OMP_CLAUSE_DECL (c))) 3718 lastprivate_firstprivate = true; 3719 break; 3720 case OMP_CLAUSE_ALIGNED: 3721 if (pass == 0) 3722 continue; 3723 var = OMP_CLAUSE_DECL (c); 3724 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE 3725 && !is_global_var (var)) 3726 { 3727 new_var = maybe_lookup_decl (var, ctx); 3728 if (new_var == NULL_TREE) 3729 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx); 3730 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 3731 x = build_call_expr_loc (clause_loc, x, 2, new_var, 3732 omp_clause_aligned_alignment (c)); 3733 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); 3734 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); 3735 gimplify_and_add (x, ilist); 3736 } 3737 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE 3738 && is_global_var (var)) 3739 { 3740 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2; 3741 new_var = lookup_decl (var, ctx); 3742 t = maybe_lookup_decl_in_outer_ctx (var, ctx); 3743 t = build_fold_addr_expr_loc (clause_loc, t); 3744 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 3745 t = build_call_expr_loc (clause_loc, t2, 2, t, 3746 omp_clause_aligned_alignment (c)); 3747 t = fold_convert_loc (clause_loc, ptype, t); 3748 x = create_tmp_var (ptype); 3749 t = build2 (MODIFY_EXPR, ptype, x, t); 3750 gimplify_and_add (t, ilist); 3751 t = build_simple_mem_ref_loc (clause_loc, x); 3752 SET_DECL_VALUE_EXPR (new_var, t); 3753 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3754 } 3755 continue; 3756 default: 3757 continue; 3758 } 3759 3760 new_var = var = OMP_CLAUSE_DECL (c); 3761 if (c_kind != OMP_CLAUSE_COPYIN) 3762 new_var = lookup_decl (var, ctx); 3763 3764 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN) 3765 { 3766 if (pass != 0) 3767 continue; 3768 } 3769 else if (is_variable_sized (var)) 3770 { 3771 /* For variable sized types, we need to allocate the 3772 actual storage here. Call alloca and store the 3773 result in the pointer decl that we created elsewhere. */ 3774 if (pass == 0) 3775 continue; 3776 3777 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx)) 3778 { 3779 gcall *stmt; 3780 tree tmp, atmp; 3781 3782 ptr = DECL_VALUE_EXPR (new_var); 3783 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF); 3784 ptr = TREE_OPERAND (ptr, 0); 3785 gcc_assert (DECL_P (ptr)); 3786 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var)); 3787 3788 /* void *tmp = __builtin_alloca */ 3789 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA); 3790 stmt = gimple_build_call (atmp, 1, x); 3791 tmp = create_tmp_var_raw (ptr_type_node); 3792 gimple_add_tmp_var (tmp); 3793 gimple_call_set_lhs (stmt, tmp); 3794 3795 gimple_seq_add_stmt (ilist, stmt); 3796 3797 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp); 3798 gimplify_assign (ptr, x, ilist); 3799 } 3800 } 3801 else if (is_reference (var)) 3802 { 3803 /* For references that are being privatized for Fortran, 3804 allocate new backing storage for the new pointer 3805 variable. This allows us to avoid changing all the 3806 code that expects a pointer to something that expects 3807 a direct variable. */ 3808 if (pass == 0) 3809 continue; 3810 3811 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var))); 3812 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx)) 3813 { 3814 x = build_receiver_ref (var, false, ctx); 3815 x = build_fold_addr_expr_loc (clause_loc, x); 3816 } 3817 else if (TREE_CONSTANT (x)) 3818 { 3819 /* For reduction in SIMD loop, defer adding the 3820 initialization of the reference, because if we decide 3821 to use SIMD array for it, the initilization could cause 3822 expansion ICE. */ 3823 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd) 3824 x = NULL_TREE; 3825 else 3826 { 3827 const char *name = NULL; 3828 if (DECL_NAME (var)) 3829 name = IDENTIFIER_POINTER (DECL_NAME (new_var)); 3830 3831 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)), 3832 name); 3833 gimple_add_tmp_var (x); 3834 TREE_ADDRESSABLE (x) = 1; 3835 x = build_fold_addr_expr_loc (clause_loc, x); 3836 } 3837 } 3838 else 3839 { 3840 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA); 3841 x = build_call_expr_loc (clause_loc, atmp, 1, x); 3842 } 3843 3844 if (x) 3845 { 3846 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); 3847 gimplify_assign (new_var, x, ilist); 3848 } 3849 3850 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 3851 } 3852 else if (c_kind == OMP_CLAUSE_REDUCTION 3853 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 3854 { 3855 if (pass == 0) 3856 continue; 3857 } 3858 else if (pass != 0) 3859 continue; 3860 3861 switch (OMP_CLAUSE_CODE (c)) 3862 { 3863 case OMP_CLAUSE_SHARED: 3864 /* Ignore shared directives in teams construct. */ 3865 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 3866 continue; 3867 /* Shared global vars are just accessed directly. */ 3868 if (is_global_var (new_var)) 3869 break; 3870 /* Set up the DECL_VALUE_EXPR for shared variables now. This 3871 needs to be delayed until after fixup_child_record_type so 3872 that we get the correct type during the dereference. */ 3873 by_ref = use_pointer_for_field (var, ctx); 3874 x = build_receiver_ref (var, by_ref, ctx); 3875 SET_DECL_VALUE_EXPR (new_var, x); 3876 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3877 3878 /* ??? If VAR is not passed by reference, and the variable 3879 hasn't been initialized yet, then we'll get a warning for 3880 the store into the omp_data_s structure. Ideally, we'd be 3881 able to notice this and not store anything at all, but 3882 we're generating code too early. Suppress the warning. */ 3883 if (!by_ref) 3884 TREE_NO_WARNING (var) = 1; 3885 break; 3886 3887 case OMP_CLAUSE_LASTPRIVATE: 3888 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 3889 break; 3890 /* FALLTHRU */ 3891 3892 case OMP_CLAUSE_PRIVATE: 3893 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE) 3894 x = build_outer_var_ref (var, ctx); 3895 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 3896 { 3897 if (is_task_ctx (ctx)) 3898 x = build_receiver_ref (var, false, ctx); 3899 else 3900 x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE); 3901 } 3902 else 3903 x = NULL; 3904 do_private: 3905 tree nx; 3906 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x); 3907 if (is_simd) 3908 { 3909 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var); 3910 if ((TREE_ADDRESSABLE (new_var) || nx || y 3911 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE) 3912 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 3913 idx, lane, ivar, lvar)) 3914 { 3915 if (nx) 3916 x = lang_hooks.decls.omp_clause_default_ctor 3917 (c, unshare_expr (ivar), x); 3918 if (nx && x) 3919 gimplify_and_add (x, &llist[0]); 3920 if (y) 3921 { 3922 y = lang_hooks.decls.omp_clause_dtor (c, ivar); 3923 if (y) 3924 { 3925 gimple_seq tseq = NULL; 3926 3927 dtor = y; 3928 gimplify_stmt (&dtor, &tseq); 3929 gimple_seq_add_seq (&llist[1], tseq); 3930 } 3931 } 3932 break; 3933 } 3934 } 3935 if (nx) 3936 gimplify_and_add (nx, ilist); 3937 /* FALLTHRU */ 3938 3939 do_dtor: 3940 x = lang_hooks.decls.omp_clause_dtor (c, new_var); 3941 if (x) 3942 { 3943 gimple_seq tseq = NULL; 3944 3945 dtor = x; 3946 gimplify_stmt (&dtor, &tseq); 3947 gimple_seq_add_seq (dlist, tseq); 3948 } 3949 break; 3950 3951 case OMP_CLAUSE_LINEAR: 3952 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)) 3953 goto do_firstprivate; 3954 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c)) 3955 x = NULL; 3956 else 3957 x = build_outer_var_ref (var, ctx); 3958 goto do_private; 3959 3960 case OMP_CLAUSE_FIRSTPRIVATE: 3961 if (is_task_ctx (ctx)) 3962 { 3963 if (is_reference (var) || is_variable_sized (var)) 3964 goto do_dtor; 3965 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, 3966 ctx)) 3967 || use_pointer_for_field (var, NULL)) 3968 { 3969 x = build_receiver_ref (var, false, ctx); 3970 SET_DECL_VALUE_EXPR (new_var, x); 3971 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3972 goto do_dtor; 3973 } 3974 } 3975 do_firstprivate: 3976 x = build_outer_var_ref (var, ctx); 3977 if (is_simd) 3978 { 3979 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 3980 && gimple_omp_for_combined_into_p (ctx->stmt)) 3981 { 3982 tree t = OMP_CLAUSE_LINEAR_STEP (c); 3983 tree stept = TREE_TYPE (t); 3984 tree ct = find_omp_clause (clauses, 3985 OMP_CLAUSE__LOOPTEMP_); 3986 gcc_assert (ct); 3987 tree l = OMP_CLAUSE_DECL (ct); 3988 tree n1 = fd->loop.n1; 3989 tree step = fd->loop.step; 3990 tree itype = TREE_TYPE (l); 3991 if (POINTER_TYPE_P (itype)) 3992 itype = signed_type_for (itype); 3993 l = fold_build2 (MINUS_EXPR, itype, l, n1); 3994 if (TYPE_UNSIGNED (itype) 3995 && fd->loop.cond_code == GT_EXPR) 3996 l = fold_build2 (TRUNC_DIV_EXPR, itype, 3997 fold_build1 (NEGATE_EXPR, itype, l), 3998 fold_build1 (NEGATE_EXPR, 3999 itype, step)); 4000 else 4001 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step); 4002 t = fold_build2 (MULT_EXPR, stept, 4003 fold_convert (stept, l), t); 4004 4005 if (OMP_CLAUSE_LINEAR_ARRAY (c)) 4006 { 4007 x = lang_hooks.decls.omp_clause_linear_ctor 4008 (c, new_var, x, t); 4009 gimplify_and_add (x, ilist); 4010 goto do_dtor; 4011 } 4012 4013 if (POINTER_TYPE_P (TREE_TYPE (x))) 4014 x = fold_build2 (POINTER_PLUS_EXPR, 4015 TREE_TYPE (x), x, t); 4016 else 4017 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t); 4018 } 4019 4020 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR 4021 || TREE_ADDRESSABLE (new_var)) 4022 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 4023 idx, lane, ivar, lvar)) 4024 { 4025 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR) 4026 { 4027 tree iv = create_tmp_var (TREE_TYPE (new_var)); 4028 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x); 4029 gimplify_and_add (x, ilist); 4030 gimple_stmt_iterator gsi 4031 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt)); 4032 gassign *g 4033 = gimple_build_assign (unshare_expr (lvar), iv); 4034 gsi_insert_before_without_update (&gsi, g, 4035 GSI_SAME_STMT); 4036 tree t = OMP_CLAUSE_LINEAR_STEP (c); 4037 enum tree_code code = PLUS_EXPR; 4038 if (POINTER_TYPE_P (TREE_TYPE (new_var))) 4039 code = POINTER_PLUS_EXPR; 4040 g = gimple_build_assign (iv, code, iv, t); 4041 gsi_insert_before_without_update (&gsi, g, 4042 GSI_SAME_STMT); 4043 break; 4044 } 4045 x = lang_hooks.decls.omp_clause_copy_ctor 4046 (c, unshare_expr (ivar), x); 4047 gimplify_and_add (x, &llist[0]); 4048 x = lang_hooks.decls.omp_clause_dtor (c, ivar); 4049 if (x) 4050 { 4051 gimple_seq tseq = NULL; 4052 4053 dtor = x; 4054 gimplify_stmt (&dtor, &tseq); 4055 gimple_seq_add_seq (&llist[1], tseq); 4056 } 4057 break; 4058 } 4059 } 4060 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x); 4061 gimplify_and_add (x, ilist); 4062 goto do_dtor; 4063 4064 case OMP_CLAUSE__LOOPTEMP_: 4065 gcc_assert (is_parallel_ctx (ctx)); 4066 x = build_outer_var_ref (var, ctx); 4067 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); 4068 gimplify_and_add (x, ilist); 4069 break; 4070 4071 case OMP_CLAUSE_COPYIN: 4072 by_ref = use_pointer_for_field (var, NULL); 4073 x = build_receiver_ref (var, by_ref, ctx); 4074 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x); 4075 append_to_statement_list (x, ©in_seq); 4076 copyin_by_ref |= by_ref; 4077 break; 4078 4079 case OMP_CLAUSE_REDUCTION: 4080 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 4081 { 4082 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); 4083 gimple tseq; 4084 x = build_outer_var_ref (var, ctx); 4085 4086 if (is_reference (var) 4087 && !useless_type_conversion_p (TREE_TYPE (placeholder), 4088 TREE_TYPE (x))) 4089 x = build_fold_addr_expr_loc (clause_loc, x); 4090 SET_DECL_VALUE_EXPR (placeholder, x); 4091 DECL_HAS_VALUE_EXPR_P (placeholder) = 1; 4092 tree new_vard = new_var; 4093 if (is_reference (var)) 4094 { 4095 gcc_assert (TREE_CODE (new_var) == MEM_REF); 4096 new_vard = TREE_OPERAND (new_var, 0); 4097 gcc_assert (DECL_P (new_vard)); 4098 } 4099 if (is_simd 4100 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 4101 idx, lane, ivar, lvar)) 4102 { 4103 if (new_vard == new_var) 4104 { 4105 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar); 4106 SET_DECL_VALUE_EXPR (new_var, ivar); 4107 } 4108 else 4109 { 4110 SET_DECL_VALUE_EXPR (new_vard, 4111 build_fold_addr_expr (ivar)); 4112 DECL_HAS_VALUE_EXPR_P (new_vard) = 1; 4113 } 4114 x = lang_hooks.decls.omp_clause_default_ctor 4115 (c, unshare_expr (ivar), 4116 build_outer_var_ref (var, ctx)); 4117 if (x) 4118 gimplify_and_add (x, &llist[0]); 4119 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) 4120 { 4121 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); 4122 lower_omp (&tseq, ctx); 4123 gimple_seq_add_seq (&llist[0], tseq); 4124 } 4125 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; 4126 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); 4127 lower_omp (&tseq, ctx); 4128 gimple_seq_add_seq (&llist[1], tseq); 4129 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; 4130 DECL_HAS_VALUE_EXPR_P (placeholder) = 0; 4131 if (new_vard == new_var) 4132 SET_DECL_VALUE_EXPR (new_var, lvar); 4133 else 4134 SET_DECL_VALUE_EXPR (new_vard, 4135 build_fold_addr_expr (lvar)); 4136 x = lang_hooks.decls.omp_clause_dtor (c, ivar); 4137 if (x) 4138 { 4139 tseq = NULL; 4140 dtor = x; 4141 gimplify_stmt (&dtor, &tseq); 4142 gimple_seq_add_seq (&llist[1], tseq); 4143 } 4144 break; 4145 } 4146 /* If this is a reference to constant size reduction var 4147 with placeholder, we haven't emitted the initializer 4148 for it because it is undesirable if SIMD arrays are used. 4149 But if they aren't used, we need to emit the deferred 4150 initialization now. */ 4151 else if (is_reference (var) && is_simd) 4152 handle_simd_reference (clause_loc, new_vard, ilist); 4153 x = lang_hooks.decls.omp_clause_default_ctor 4154 (c, unshare_expr (new_var), 4155 build_outer_var_ref (var, ctx)); 4156 if (x) 4157 gimplify_and_add (x, ilist); 4158 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) 4159 { 4160 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); 4161 lower_omp (&tseq, ctx); 4162 gimple_seq_add_seq (ilist, tseq); 4163 } 4164 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; 4165 if (is_simd) 4166 { 4167 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); 4168 lower_omp (&tseq, ctx); 4169 gimple_seq_add_seq (dlist, tseq); 4170 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; 4171 } 4172 DECL_HAS_VALUE_EXPR_P (placeholder) = 0; 4173 goto do_dtor; 4174 } 4175 else 4176 { 4177 x = omp_reduction_init (c, TREE_TYPE (new_var)); 4178 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE); 4179 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); 4180 4181 /* reduction(-:var) sums up the partial results, so it 4182 acts identically to reduction(+:var). */ 4183 if (code == MINUS_EXPR) 4184 code = PLUS_EXPR; 4185 4186 tree new_vard = new_var; 4187 if (is_simd && is_reference (var)) 4188 { 4189 gcc_assert (TREE_CODE (new_var) == MEM_REF); 4190 new_vard = TREE_OPERAND (new_var, 0); 4191 gcc_assert (DECL_P (new_vard)); 4192 } 4193 if (is_simd 4194 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 4195 idx, lane, ivar, lvar)) 4196 { 4197 tree ref = build_outer_var_ref (var, ctx); 4198 4199 gimplify_assign (unshare_expr (ivar), x, &llist[0]); 4200 4201 x = build2 (code, TREE_TYPE (ref), ref, ivar); 4202 ref = build_outer_var_ref (var, ctx); 4203 gimplify_assign (ref, x, &llist[1]); 4204 4205 if (new_vard != new_var) 4206 { 4207 SET_DECL_VALUE_EXPR (new_vard, 4208 build_fold_addr_expr (lvar)); 4209 DECL_HAS_VALUE_EXPR_P (new_vard) = 1; 4210 } 4211 } 4212 else 4213 { 4214 if (is_reference (var) && is_simd) 4215 handle_simd_reference (clause_loc, new_vard, ilist); 4216 gimplify_assign (new_var, x, ilist); 4217 if (is_simd) 4218 { 4219 tree ref = build_outer_var_ref (var, ctx); 4220 4221 x = build2 (code, TREE_TYPE (ref), ref, new_var); 4222 ref = build_outer_var_ref (var, ctx); 4223 gimplify_assign (ref, x, dlist); 4224 } 4225 } 4226 } 4227 break; 4228 4229 default: 4230 gcc_unreachable (); 4231 } 4232 } 4233 } 4234 4235 if (lane) 4236 { 4237 tree uid = create_tmp_var (ptr_type_node, "simduid"); 4238 /* Don't want uninit warnings on simduid, it is always uninitialized, 4239 but we use it not for the value, but for the DECL_UID only. */ 4240 TREE_NO_WARNING (uid) = 1; 4241 gimple g 4242 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid); 4243 gimple_call_set_lhs (g, lane); 4244 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt)); 4245 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT); 4246 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_); 4247 OMP_CLAUSE__SIMDUID__DECL (c) = uid; 4248 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt); 4249 gimple_omp_for_set_clauses (ctx->stmt, c); 4250 g = gimple_build_assign (lane, INTEGER_CST, 4251 build_int_cst (unsigned_type_node, 0)); 4252 gimple_seq_add_stmt (ilist, g); 4253 for (int i = 0; i < 2; i++) 4254 if (llist[i]) 4255 { 4256 tree vf = create_tmp_var (unsigned_type_node); 4257 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid); 4258 gimple_call_set_lhs (g, vf); 4259 gimple_seq *seq = i == 0 ? ilist : dlist; 4260 gimple_seq_add_stmt (seq, g); 4261 tree t = build_int_cst (unsigned_type_node, 0); 4262 g = gimple_build_assign (idx, INTEGER_CST, t); 4263 gimple_seq_add_stmt (seq, g); 4264 tree body = create_artificial_label (UNKNOWN_LOCATION); 4265 tree header = create_artificial_label (UNKNOWN_LOCATION); 4266 tree end = create_artificial_label (UNKNOWN_LOCATION); 4267 gimple_seq_add_stmt (seq, gimple_build_goto (header)); 4268 gimple_seq_add_stmt (seq, gimple_build_label (body)); 4269 gimple_seq_add_seq (seq, llist[i]); 4270 t = build_int_cst (unsigned_type_node, 1); 4271 g = gimple_build_assign (idx, PLUS_EXPR, idx, t); 4272 gimple_seq_add_stmt (seq, g); 4273 gimple_seq_add_stmt (seq, gimple_build_label (header)); 4274 g = gimple_build_cond (LT_EXPR, idx, vf, body, end); 4275 gimple_seq_add_stmt (seq, g); 4276 gimple_seq_add_stmt (seq, gimple_build_label (end)); 4277 } 4278 } 4279 4280 /* The copyin sequence is not to be executed by the main thread, since 4281 that would result in self-copies. Perhaps not visible to scalars, 4282 but it certainly is to C++ operator=. */ 4283 if (copyin_seq) 4284 { 4285 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM), 4286 0); 4287 x = build2 (NE_EXPR, boolean_type_node, x, 4288 build_int_cst (TREE_TYPE (x), 0)); 4289 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL); 4290 gimplify_and_add (x, ilist); 4291 } 4292 4293 /* If any copyin variable is passed by reference, we must ensure the 4294 master thread doesn't modify it before it is copied over in all 4295 threads. Similarly for variables in both firstprivate and 4296 lastprivate clauses we need to ensure the lastprivate copying 4297 happens after firstprivate copying in all threads. And similarly 4298 for UDRs if initializer expression refers to omp_orig. */ 4299 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref) 4300 { 4301 /* Don't add any barrier for #pragma omp simd or 4302 #pragma omp distribute. */ 4303 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR 4304 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR) 4305 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE)); 4306 } 4307 4308 /* If max_vf is non-zero, then we can use only a vectorization factor 4309 up to the max_vf we chose. So stick it into the safelen clause. */ 4310 if (max_vf) 4311 { 4312 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 4313 OMP_CLAUSE_SAFELEN); 4314 if (c == NULL_TREE 4315 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST 4316 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), 4317 max_vf) == 1)) 4318 { 4319 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN); 4320 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node, 4321 max_vf); 4322 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt); 4323 gimple_omp_for_set_clauses (ctx->stmt, c); 4324 } 4325 } 4326 } 4327 4328 4329 /* Generate code to implement the LASTPRIVATE clauses. This is used for 4330 both parallel and workshare constructs. PREDICATE may be NULL if it's 4331 always true. */ 4332 4333 static void 4334 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list, 4335 omp_context *ctx) 4336 { 4337 tree x, c, label = NULL, orig_clauses = clauses; 4338 bool par_clauses = false; 4339 tree simduid = NULL, lastlane = NULL; 4340 4341 /* Early exit if there are no lastprivate or linear clauses. */ 4342 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) 4343 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE 4344 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR 4345 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses))) 4346 break; 4347 if (clauses == NULL) 4348 { 4349 /* If this was a workshare clause, see if it had been combined 4350 with its parallel. In that case, look for the clauses on the 4351 parallel statement itself. */ 4352 if (is_parallel_ctx (ctx)) 4353 return; 4354 4355 ctx = ctx->outer; 4356 if (ctx == NULL || !is_parallel_ctx (ctx)) 4357 return; 4358 4359 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), 4360 OMP_CLAUSE_LASTPRIVATE); 4361 if (clauses == NULL) 4362 return; 4363 par_clauses = true; 4364 } 4365 4366 if (predicate) 4367 { 4368 gcond *stmt; 4369 tree label_true, arm1, arm2; 4370 4371 label = create_artificial_label (UNKNOWN_LOCATION); 4372 label_true = create_artificial_label (UNKNOWN_LOCATION); 4373 arm1 = TREE_OPERAND (predicate, 0); 4374 arm2 = TREE_OPERAND (predicate, 1); 4375 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue); 4376 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue); 4377 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2, 4378 label_true, label); 4379 gimple_seq_add_stmt (stmt_list, stmt); 4380 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true)); 4381 } 4382 4383 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 4384 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 4385 { 4386 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_); 4387 if (simduid) 4388 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); 4389 } 4390 4391 for (c = clauses; c ;) 4392 { 4393 tree var, new_var; 4394 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4395 4396 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE 4397 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 4398 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))) 4399 { 4400 var = OMP_CLAUSE_DECL (c); 4401 new_var = lookup_decl (var, ctx); 4402 4403 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var)) 4404 { 4405 tree val = DECL_VALUE_EXPR (new_var); 4406 if (TREE_CODE (val) == ARRAY_REF 4407 && VAR_P (TREE_OPERAND (val, 0)) 4408 && lookup_attribute ("omp simd array", 4409 DECL_ATTRIBUTES (TREE_OPERAND (val, 4410 0)))) 4411 { 4412 if (lastlane == NULL) 4413 { 4414 lastlane = create_tmp_var (unsigned_type_node); 4415 gcall *g 4416 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE, 4417 2, simduid, 4418 TREE_OPERAND (val, 1)); 4419 gimple_call_set_lhs (g, lastlane); 4420 gimple_seq_add_stmt (stmt_list, g); 4421 } 4422 new_var = build4 (ARRAY_REF, TREE_TYPE (val), 4423 TREE_OPERAND (val, 0), lastlane, 4424 NULL_TREE, NULL_TREE); 4425 } 4426 } 4427 4428 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE 4429 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) 4430 { 4431 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); 4432 gimple_seq_add_seq (stmt_list, 4433 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)); 4434 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL; 4435 } 4436 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 4437 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) 4438 { 4439 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); 4440 gimple_seq_add_seq (stmt_list, 4441 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)); 4442 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL; 4443 } 4444 4445 x = build_outer_var_ref (var, ctx); 4446 if (is_reference (var)) 4447 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 4448 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var); 4449 gimplify_and_add (x, stmt_list); 4450 } 4451 c = OMP_CLAUSE_CHAIN (c); 4452 if (c == NULL && !par_clauses) 4453 { 4454 /* If this was a workshare clause, see if it had been combined 4455 with its parallel. In that case, continue looking for the 4456 clauses also on the parallel statement itself. */ 4457 if (is_parallel_ctx (ctx)) 4458 break; 4459 4460 ctx = ctx->outer; 4461 if (ctx == NULL || !is_parallel_ctx (ctx)) 4462 break; 4463 4464 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), 4465 OMP_CLAUSE_LASTPRIVATE); 4466 par_clauses = true; 4467 } 4468 } 4469 4470 if (label) 4471 gimple_seq_add_stmt (stmt_list, gimple_build_label (label)); 4472 } 4473 4474 static void 4475 oacc_lower_reduction_var_helper (gimple_seq *stmt_seqp, omp_context *ctx, 4476 tree tid, tree var, tree new_var) 4477 { 4478 /* The atomic add at the end of the sum creates unnecessary 4479 write contention on accelerators. To work around this, 4480 create an array to store the partial reductions. Later, in 4481 lower_omp_for (for openacc), the values of array will be 4482 combined. */ 4483 4484 tree t = NULL_TREE, array, x; 4485 tree type = get_base_type (var); 4486 gimple stmt; 4487 4488 /* Now insert the partial reductions into the array. */ 4489 4490 /* Find the reduction array. */ 4491 4492 tree ptype = build_pointer_type (type); 4493 4494 t = lookup_oacc_reduction (oacc_get_reduction_array_id (var), ctx); 4495 t = build_receiver_ref (t, false, ctx->outer); 4496 4497 array = create_tmp_var (ptype); 4498 gimplify_assign (array, t, stmt_seqp); 4499 4500 tree ptr = create_tmp_var (TREE_TYPE (array)); 4501 4502 /* Find the reduction array. */ 4503 4504 /* testing a unary conversion. */ 4505 tree offset = create_tmp_var (sizetype); 4506 gimplify_assign (offset, TYPE_SIZE_UNIT (type), 4507 stmt_seqp); 4508 t = create_tmp_var (sizetype); 4509 gimplify_assign (t, unshare_expr (fold_build1 (NOP_EXPR, sizetype, tid)), 4510 stmt_seqp); 4511 stmt = gimple_build_assign (offset, MULT_EXPR, offset, t); 4512 gimple_seq_add_stmt (stmt_seqp, stmt); 4513 4514 /* Offset expression. Does the POINTER_PLUS_EXPR take care 4515 of adding sizeof(var) to the array? */ 4516 ptr = create_tmp_var (ptype); 4517 stmt = gimple_build_assign (unshare_expr (ptr), POINTER_PLUS_EXPR, array, 4518 offset); 4519 gimple_seq_add_stmt (stmt_seqp, stmt); 4520 4521 /* Move the local sum to gfc$sum[i]. */ 4522 x = unshare_expr (build_simple_mem_ref (ptr)); 4523 stmt = gimplify_assign (x, new_var, stmt_seqp); 4524 } 4525 4526 /* Generate code to implement the REDUCTION clauses. */ 4527 4528 static void 4529 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx) 4530 { 4531 gimple_seq sub_seq = NULL; 4532 gimple stmt; 4533 tree x, c, tid = NULL_TREE; 4534 int count = 0; 4535 4536 /* SIMD reductions are handled in lower_rec_input_clauses. */ 4537 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 4538 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 4539 return; 4540 4541 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC 4542 update in that case, otherwise use a lock. */ 4543 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c)) 4544 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) 4545 { 4546 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 4547 { 4548 /* Never use OMP_ATOMIC for array reductions or UDRs. */ 4549 count = -1; 4550 break; 4551 } 4552 count++; 4553 } 4554 4555 if (count == 0) 4556 return; 4557 4558 /* Initialize thread info for OpenACC. */ 4559 if (is_gimple_omp_oacc (ctx->stmt)) 4560 { 4561 /* Get the current thread id. */ 4562 tree call = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); 4563 tid = create_tmp_var (TREE_TYPE (TREE_TYPE (call))); 4564 gimple stmt = gimple_build_call (call, 0); 4565 gimple_call_set_lhs (stmt, tid); 4566 gimple_seq_add_stmt (stmt_seqp, stmt); 4567 } 4568 4569 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 4570 { 4571 tree var, ref, new_var; 4572 enum tree_code code; 4573 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4574 4575 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) 4576 continue; 4577 4578 var = OMP_CLAUSE_DECL (c); 4579 new_var = lookup_decl (var, ctx); 4580 if (is_reference (var)) 4581 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 4582 ref = build_outer_var_ref (var, ctx); 4583 code = OMP_CLAUSE_REDUCTION_CODE (c); 4584 4585 /* reduction(-:var) sums up the partial results, so it acts 4586 identically to reduction(+:var). */ 4587 if (code == MINUS_EXPR) 4588 code = PLUS_EXPR; 4589 4590 if (is_gimple_omp_oacc (ctx->stmt)) 4591 { 4592 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)); 4593 4594 oacc_lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var); 4595 } 4596 else if (count == 1) 4597 { 4598 tree addr = build_fold_addr_expr_loc (clause_loc, ref); 4599 4600 addr = save_expr (addr); 4601 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr); 4602 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var); 4603 x = build2 (OMP_ATOMIC, void_type_node, addr, x); 4604 gimplify_and_add (x, stmt_seqp); 4605 return; 4606 } 4607 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 4608 { 4609 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); 4610 4611 if (is_reference (var) 4612 && !useless_type_conversion_p (TREE_TYPE (placeholder), 4613 TREE_TYPE (ref))) 4614 ref = build_fold_addr_expr_loc (clause_loc, ref); 4615 SET_DECL_VALUE_EXPR (placeholder, ref); 4616 DECL_HAS_VALUE_EXPR_P (placeholder) = 1; 4617 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); 4618 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); 4619 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; 4620 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; 4621 } 4622 else 4623 { 4624 x = build2 (code, TREE_TYPE (ref), ref, new_var); 4625 ref = build_outer_var_ref (var, ctx); 4626 gimplify_assign (ref, x, &sub_seq); 4627 } 4628 } 4629 4630 if (is_gimple_omp_oacc (ctx->stmt)) 4631 return; 4632 4633 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START), 4634 0); 4635 gimple_seq_add_stmt (stmt_seqp, stmt); 4636 4637 gimple_seq_add_seq (stmt_seqp, sub_seq); 4638 4639 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END), 4640 0); 4641 gimple_seq_add_stmt (stmt_seqp, stmt); 4642 } 4643 4644 4645 /* Generate code to implement the COPYPRIVATE clauses. */ 4646 4647 static void 4648 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist, 4649 omp_context *ctx) 4650 { 4651 tree c; 4652 4653 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 4654 { 4655 tree var, new_var, ref, x; 4656 bool by_ref; 4657 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4658 4659 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE) 4660 continue; 4661 4662 var = OMP_CLAUSE_DECL (c); 4663 by_ref = use_pointer_for_field (var, NULL); 4664 4665 ref = build_sender_ref (var, ctx); 4666 x = new_var = lookup_decl_in_outer_ctx (var, ctx); 4667 if (by_ref) 4668 { 4669 x = build_fold_addr_expr_loc (clause_loc, new_var); 4670 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x); 4671 } 4672 gimplify_assign (ref, x, slist); 4673 4674 ref = build_receiver_ref (var, false, ctx); 4675 if (by_ref) 4676 { 4677 ref = fold_convert_loc (clause_loc, 4678 build_pointer_type (TREE_TYPE (new_var)), 4679 ref); 4680 ref = build_fold_indirect_ref_loc (clause_loc, ref); 4681 } 4682 if (is_reference (var)) 4683 { 4684 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref); 4685 ref = build_simple_mem_ref_loc (clause_loc, ref); 4686 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 4687 } 4688 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref); 4689 gimplify_and_add (x, rlist); 4690 } 4691 } 4692 4693 4694 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE, 4695 and REDUCTION from the sender (aka parent) side. */ 4696 4697 static void 4698 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist, 4699 omp_context *ctx) 4700 { 4701 tree c; 4702 4703 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 4704 { 4705 tree val, ref, x, var; 4706 bool by_ref, do_in = false, do_out = false; 4707 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4708 4709 switch (OMP_CLAUSE_CODE (c)) 4710 { 4711 case OMP_CLAUSE_PRIVATE: 4712 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 4713 break; 4714 continue; 4715 case OMP_CLAUSE_FIRSTPRIVATE: 4716 case OMP_CLAUSE_COPYIN: 4717 case OMP_CLAUSE_LASTPRIVATE: 4718 case OMP_CLAUSE_REDUCTION: 4719 case OMP_CLAUSE__LOOPTEMP_: 4720 break; 4721 default: 4722 continue; 4723 } 4724 4725 val = OMP_CLAUSE_DECL (c); 4726 var = lookup_decl_in_outer_ctx (val, ctx); 4727 4728 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN 4729 && is_global_var (var)) 4730 continue; 4731 if (is_variable_sized (val)) 4732 continue; 4733 by_ref = use_pointer_for_field (val, NULL); 4734 4735 switch (OMP_CLAUSE_CODE (c)) 4736 { 4737 case OMP_CLAUSE_PRIVATE: 4738 case OMP_CLAUSE_FIRSTPRIVATE: 4739 case OMP_CLAUSE_COPYIN: 4740 case OMP_CLAUSE__LOOPTEMP_: 4741 do_in = true; 4742 break; 4743 4744 case OMP_CLAUSE_LASTPRIVATE: 4745 if (by_ref || is_reference (val)) 4746 { 4747 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 4748 continue; 4749 do_in = true; 4750 } 4751 else 4752 { 4753 do_out = true; 4754 if (lang_hooks.decls.omp_private_outer_ref (val)) 4755 do_in = true; 4756 } 4757 break; 4758 4759 case OMP_CLAUSE_REDUCTION: 4760 do_in = true; 4761 do_out = !(by_ref || is_reference (val)); 4762 break; 4763 4764 default: 4765 gcc_unreachable (); 4766 } 4767 4768 if (do_in) 4769 { 4770 ref = build_sender_ref (val, ctx); 4771 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var; 4772 gimplify_assign (ref, x, ilist); 4773 if (is_task_ctx (ctx)) 4774 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL; 4775 } 4776 4777 if (do_out) 4778 { 4779 ref = build_sender_ref (val, ctx); 4780 gimplify_assign (var, ref, olist); 4781 } 4782 } 4783 } 4784 4785 /* Generate code to implement SHARED from the sender (aka parent) 4786 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't 4787 list things that got automatically shared. */ 4788 4789 static void 4790 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx) 4791 { 4792 tree var, ovar, nvar, f, x, record_type; 4793 4794 if (ctx->record_type == NULL) 4795 return; 4796 4797 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type; 4798 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) 4799 { 4800 ovar = DECL_ABSTRACT_ORIGIN (f); 4801 nvar = maybe_lookup_decl (ovar, ctx); 4802 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar)) 4803 continue; 4804 4805 /* If CTX is a nested parallel directive. Find the immediately 4806 enclosing parallel or workshare construct that contains a 4807 mapping for OVAR. */ 4808 var = lookup_decl_in_outer_ctx (ovar, ctx); 4809 4810 if (use_pointer_for_field (ovar, ctx)) 4811 { 4812 x = build_sender_ref (ovar, ctx); 4813 var = build_fold_addr_expr (var); 4814 gimplify_assign (x, var, ilist); 4815 } 4816 else 4817 { 4818 x = build_sender_ref (ovar, ctx); 4819 gimplify_assign (x, var, ilist); 4820 4821 if (!TREE_READONLY (var) 4822 /* We don't need to receive a new reference to a result 4823 or parm decl. In fact we may not store to it as we will 4824 invalidate any pending RSO and generate wrong gimple 4825 during inlining. */ 4826 && !((TREE_CODE (var) == RESULT_DECL 4827 || TREE_CODE (var) == PARM_DECL) 4828 && DECL_BY_REFERENCE (var))) 4829 { 4830 x = build_sender_ref (ovar, ctx); 4831 gimplify_assign (var, x, olist); 4832 } 4833 } 4834 } 4835 } 4836 4837 4838 /* A convenience function to build an empty GIMPLE_COND with just the 4839 condition. */ 4840 4841 static gcond * 4842 gimple_build_cond_empty (tree cond) 4843 { 4844 enum tree_code pred_code; 4845 tree lhs, rhs; 4846 4847 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs); 4848 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE); 4849 } 4850 4851 4852 /* Build the function calls to GOMP_parallel_start etc to actually 4853 generate the parallel operation. REGION is the parallel region 4854 being expanded. BB is the block where to insert the code. WS_ARGS 4855 will be set if this is a call to a combined parallel+workshare 4856 construct, it contains the list of additional arguments needed by 4857 the workshare construct. */ 4858 4859 static void 4860 expand_parallel_call (struct omp_region *region, basic_block bb, 4861 gomp_parallel *entry_stmt, 4862 vec<tree, va_gc> *ws_args) 4863 { 4864 tree t, t1, t2, val, cond, c, clauses, flags; 4865 gimple_stmt_iterator gsi; 4866 gimple stmt; 4867 enum built_in_function start_ix; 4868 int start_ix2; 4869 location_t clause_loc; 4870 vec<tree, va_gc> *args; 4871 4872 clauses = gimple_omp_parallel_clauses (entry_stmt); 4873 4874 /* Determine what flavor of GOMP_parallel we will be 4875 emitting. */ 4876 start_ix = BUILT_IN_GOMP_PARALLEL; 4877 if (is_combined_parallel (region)) 4878 { 4879 switch (region->inner->type) 4880 { 4881 case GIMPLE_OMP_FOR: 4882 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO); 4883 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC 4884 + (region->inner->sched_kind 4885 == OMP_CLAUSE_SCHEDULE_RUNTIME 4886 ? 3 : region->inner->sched_kind)); 4887 start_ix = (enum built_in_function)start_ix2; 4888 break; 4889 case GIMPLE_OMP_SECTIONS: 4890 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS; 4891 break; 4892 default: 4893 gcc_unreachable (); 4894 } 4895 } 4896 4897 /* By default, the value of NUM_THREADS is zero (selected at run time) 4898 and there is no conditional. */ 4899 cond = NULL_TREE; 4900 val = build_int_cst (unsigned_type_node, 0); 4901 flags = build_int_cst (unsigned_type_node, 0); 4902 4903 c = find_omp_clause (clauses, OMP_CLAUSE_IF); 4904 if (c) 4905 cond = OMP_CLAUSE_IF_EXPR (c); 4906 4907 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS); 4908 if (c) 4909 { 4910 val = OMP_CLAUSE_NUM_THREADS_EXPR (c); 4911 clause_loc = OMP_CLAUSE_LOCATION (c); 4912 } 4913 else 4914 clause_loc = gimple_location (entry_stmt); 4915 4916 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND); 4917 if (c) 4918 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c)); 4919 4920 /* Ensure 'val' is of the correct type. */ 4921 val = fold_convert_loc (clause_loc, unsigned_type_node, val); 4922 4923 /* If we found the clause 'if (cond)', build either 4924 (cond != 0) or (cond ? val : 1u). */ 4925 if (cond) 4926 { 4927 cond = gimple_boolify (cond); 4928 4929 if (integer_zerop (val)) 4930 val = fold_build2_loc (clause_loc, 4931 EQ_EXPR, unsigned_type_node, cond, 4932 build_int_cst (TREE_TYPE (cond), 0)); 4933 else 4934 { 4935 basic_block cond_bb, then_bb, else_bb; 4936 edge e, e_then, e_else; 4937 tree tmp_then, tmp_else, tmp_join, tmp_var; 4938 4939 tmp_var = create_tmp_var (TREE_TYPE (val)); 4940 if (gimple_in_ssa_p (cfun)) 4941 { 4942 tmp_then = make_ssa_name (tmp_var); 4943 tmp_else = make_ssa_name (tmp_var); 4944 tmp_join = make_ssa_name (tmp_var); 4945 } 4946 else 4947 { 4948 tmp_then = tmp_var; 4949 tmp_else = tmp_var; 4950 tmp_join = tmp_var; 4951 } 4952 4953 e = split_block (bb, NULL); 4954 cond_bb = e->src; 4955 bb = e->dest; 4956 remove_edge (e); 4957 4958 then_bb = create_empty_bb (cond_bb); 4959 else_bb = create_empty_bb (then_bb); 4960 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb); 4961 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb); 4962 4963 stmt = gimple_build_cond_empty (cond); 4964 gsi = gsi_start_bb (cond_bb); 4965 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 4966 4967 gsi = gsi_start_bb (then_bb); 4968 stmt = gimple_build_assign (tmp_then, val); 4969 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 4970 4971 gsi = gsi_start_bb (else_bb); 4972 stmt = gimple_build_assign 4973 (tmp_else, build_int_cst (unsigned_type_node, 1)); 4974 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 4975 4976 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); 4977 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE); 4978 add_bb_to_loop (then_bb, cond_bb->loop_father); 4979 add_bb_to_loop (else_bb, cond_bb->loop_father); 4980 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU); 4981 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU); 4982 4983 if (gimple_in_ssa_p (cfun)) 4984 { 4985 gphi *phi = create_phi_node (tmp_join, bb); 4986 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION); 4987 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION); 4988 } 4989 4990 val = tmp_join; 4991 } 4992 4993 gsi = gsi_start_bb (bb); 4994 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE, 4995 false, GSI_CONTINUE_LINKING); 4996 } 4997 4998 gsi = gsi_last_bb (bb); 4999 t = gimple_omp_parallel_data_arg (entry_stmt); 5000 if (t == NULL) 5001 t1 = null_pointer_node; 5002 else 5003 t1 = build_fold_addr_expr (t); 5004 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt)); 5005 5006 vec_alloc (args, 4 + vec_safe_length (ws_args)); 5007 args->quick_push (t2); 5008 args->quick_push (t1); 5009 args->quick_push (val); 5010 if (ws_args) 5011 args->splice (*ws_args); 5012 args->quick_push (flags); 5013 5014 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, 5015 builtin_decl_explicit (start_ix), args); 5016 5017 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 5018 false, GSI_CONTINUE_LINKING); 5019 } 5020 5021 /* Insert a function call whose name is FUNC_NAME with the information from 5022 ENTRY_STMT into the basic_block BB. */ 5023 5024 static void 5025 expand_cilk_for_call (basic_block bb, gomp_parallel *entry_stmt, 5026 vec <tree, va_gc> *ws_args) 5027 { 5028 tree t, t1, t2; 5029 gimple_stmt_iterator gsi; 5030 vec <tree, va_gc> *args; 5031 5032 gcc_assert (vec_safe_length (ws_args) == 2); 5033 tree func_name = (*ws_args)[0]; 5034 tree grain = (*ws_args)[1]; 5035 5036 tree clauses = gimple_omp_parallel_clauses (entry_stmt); 5037 tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_); 5038 gcc_assert (count != NULL_TREE); 5039 count = OMP_CLAUSE_OPERAND (count, 0); 5040 5041 gsi = gsi_last_bb (bb); 5042 t = gimple_omp_parallel_data_arg (entry_stmt); 5043 if (t == NULL) 5044 t1 = null_pointer_node; 5045 else 5046 t1 = build_fold_addr_expr (t); 5047 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt)); 5048 5049 vec_alloc (args, 4); 5050 args->quick_push (t2); 5051 args->quick_push (t1); 5052 args->quick_push (count); 5053 args->quick_push (grain); 5054 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args); 5055 5056 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false, 5057 GSI_CONTINUE_LINKING); 5058 } 5059 5060 /* Build the function call to GOMP_task to actually 5061 generate the task operation. BB is the block where to insert the code. */ 5062 5063 static void 5064 expand_task_call (basic_block bb, gomp_task *entry_stmt) 5065 { 5066 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend; 5067 gimple_stmt_iterator gsi; 5068 location_t loc = gimple_location (entry_stmt); 5069 5070 clauses = gimple_omp_task_clauses (entry_stmt); 5071 5072 c = find_omp_clause (clauses, OMP_CLAUSE_IF); 5073 if (c) 5074 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c)); 5075 else 5076 cond = boolean_true_node; 5077 5078 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED); 5079 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE); 5080 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND); 5081 flags = build_int_cst (unsigned_type_node, 5082 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0)); 5083 5084 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL); 5085 if (c) 5086 { 5087 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c)); 5088 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c, 5089 build_int_cst (unsigned_type_node, 2), 5090 build_int_cst (unsigned_type_node, 0)); 5091 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c); 5092 } 5093 if (depend) 5094 depend = OMP_CLAUSE_DECL (depend); 5095 else 5096 depend = build_int_cst (ptr_type_node, 0); 5097 5098 gsi = gsi_last_bb (bb); 5099 t = gimple_omp_task_data_arg (entry_stmt); 5100 if (t == NULL) 5101 t2 = null_pointer_node; 5102 else 5103 t2 = build_fold_addr_expr_loc (loc, t); 5104 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt)); 5105 t = gimple_omp_task_copy_fn (entry_stmt); 5106 if (t == NULL) 5107 t3 = null_pointer_node; 5108 else 5109 t3 = build_fold_addr_expr_loc (loc, t); 5110 5111 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK), 5112 8, t1, t2, t3, 5113 gimple_omp_task_arg_size (entry_stmt), 5114 gimple_omp_task_arg_align (entry_stmt), cond, flags, 5115 depend); 5116 5117 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 5118 false, GSI_CONTINUE_LINKING); 5119 } 5120 5121 5122 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW 5123 catch handler and return it. This prevents programs from violating the 5124 structured block semantics with throws. */ 5125 5126 static gimple_seq 5127 maybe_catch_exception (gimple_seq body) 5128 { 5129 gimple g; 5130 tree decl; 5131 5132 if (!flag_exceptions) 5133 return body; 5134 5135 if (lang_hooks.eh_protect_cleanup_actions != NULL) 5136 decl = lang_hooks.eh_protect_cleanup_actions (); 5137 else 5138 decl = builtin_decl_explicit (BUILT_IN_TRAP); 5139 5140 g = gimple_build_eh_must_not_throw (decl); 5141 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g), 5142 GIMPLE_TRY_CATCH); 5143 5144 return gimple_seq_alloc_with_stmt (g); 5145 } 5146 5147 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */ 5148 5149 static tree 5150 vec2chain (vec<tree, va_gc> *v) 5151 { 5152 tree chain = NULL_TREE, t; 5153 unsigned ix; 5154 5155 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t) 5156 { 5157 DECL_CHAIN (t) = chain; 5158 chain = t; 5159 } 5160 5161 return chain; 5162 } 5163 5164 5165 /* Remove barriers in REGION->EXIT's block. Note that this is only 5166 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region 5167 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that 5168 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be 5169 removed. */ 5170 5171 static void 5172 remove_exit_barrier (struct omp_region *region) 5173 { 5174 gimple_stmt_iterator gsi; 5175 basic_block exit_bb; 5176 edge_iterator ei; 5177 edge e; 5178 gimple stmt; 5179 int any_addressable_vars = -1; 5180 5181 exit_bb = region->exit; 5182 5183 /* If the parallel region doesn't return, we don't have REGION->EXIT 5184 block at all. */ 5185 if (! exit_bb) 5186 return; 5187 5188 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The 5189 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of 5190 statements that can appear in between are extremely limited -- no 5191 memory operations at all. Here, we allow nothing at all, so the 5192 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */ 5193 gsi = gsi_last_bb (exit_bb); 5194 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); 5195 gsi_prev (&gsi); 5196 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL) 5197 return; 5198 5199 FOR_EACH_EDGE (e, ei, exit_bb->preds) 5200 { 5201 gsi = gsi_last_bb (e->src); 5202 if (gsi_end_p (gsi)) 5203 continue; 5204 stmt = gsi_stmt (gsi); 5205 if (gimple_code (stmt) == GIMPLE_OMP_RETURN 5206 && !gimple_omp_return_nowait_p (stmt)) 5207 { 5208 /* OpenMP 3.0 tasks unfortunately prevent this optimization 5209 in many cases. If there could be tasks queued, the barrier 5210 might be needed to let the tasks run before some local 5211 variable of the parallel that the task uses as shared 5212 runs out of scope. The task can be spawned either 5213 from within current function (this would be easy to check) 5214 or from some function it calls and gets passed an address 5215 of such a variable. */ 5216 if (any_addressable_vars < 0) 5217 { 5218 gomp_parallel *parallel_stmt 5219 = as_a <gomp_parallel *> (last_stmt (region->entry)); 5220 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt); 5221 tree local_decls, block, decl; 5222 unsigned ix; 5223 5224 any_addressable_vars = 0; 5225 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl) 5226 if (TREE_ADDRESSABLE (decl)) 5227 { 5228 any_addressable_vars = 1; 5229 break; 5230 } 5231 for (block = gimple_block (stmt); 5232 !any_addressable_vars 5233 && block 5234 && TREE_CODE (block) == BLOCK; 5235 block = BLOCK_SUPERCONTEXT (block)) 5236 { 5237 for (local_decls = BLOCK_VARS (block); 5238 local_decls; 5239 local_decls = DECL_CHAIN (local_decls)) 5240 if (TREE_ADDRESSABLE (local_decls)) 5241 { 5242 any_addressable_vars = 1; 5243 break; 5244 } 5245 if (block == gimple_block (parallel_stmt)) 5246 break; 5247 } 5248 } 5249 if (!any_addressable_vars) 5250 gimple_omp_return_set_nowait (stmt); 5251 } 5252 } 5253 } 5254 5255 static void 5256 remove_exit_barriers (struct omp_region *region) 5257 { 5258 if (region->type == GIMPLE_OMP_PARALLEL) 5259 remove_exit_barrier (region); 5260 5261 if (region->inner) 5262 { 5263 region = region->inner; 5264 remove_exit_barriers (region); 5265 while (region->next) 5266 { 5267 region = region->next; 5268 remove_exit_barriers (region); 5269 } 5270 } 5271 } 5272 5273 /* Optimize omp_get_thread_num () and omp_get_num_threads () 5274 calls. These can't be declared as const functions, but 5275 within one parallel body they are constant, so they can be 5276 transformed there into __builtin_omp_get_{thread_num,num_threads} () 5277 which are declared const. Similarly for task body, except 5278 that in untied task omp_get_thread_num () can change at any task 5279 scheduling point. */ 5280 5281 static void 5282 optimize_omp_library_calls (gimple entry_stmt) 5283 { 5284 basic_block bb; 5285 gimple_stmt_iterator gsi; 5286 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 5287 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree); 5288 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 5289 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree); 5290 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK 5291 && find_omp_clause (gimple_omp_task_clauses (entry_stmt), 5292 OMP_CLAUSE_UNTIED) != NULL); 5293 5294 FOR_EACH_BB_FN (bb, cfun) 5295 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 5296 { 5297 gimple call = gsi_stmt (gsi); 5298 tree decl; 5299 5300 if (is_gimple_call (call) 5301 && (decl = gimple_call_fndecl (call)) 5302 && DECL_EXTERNAL (decl) 5303 && TREE_PUBLIC (decl) 5304 && DECL_INITIAL (decl) == NULL) 5305 { 5306 tree built_in; 5307 5308 if (DECL_NAME (decl) == thr_num_id) 5309 { 5310 /* In #pragma omp task untied omp_get_thread_num () can change 5311 during the execution of the task region. */ 5312 if (untied_task) 5313 continue; 5314 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 5315 } 5316 else if (DECL_NAME (decl) == num_thr_id) 5317 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 5318 else 5319 continue; 5320 5321 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in) 5322 || gimple_call_num_args (call) != 0) 5323 continue; 5324 5325 if (flag_exceptions && !TREE_NOTHROW (decl)) 5326 continue; 5327 5328 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE 5329 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)), 5330 TREE_TYPE (TREE_TYPE (built_in)))) 5331 continue; 5332 5333 gimple_call_set_fndecl (call, built_in); 5334 } 5335 } 5336 } 5337 5338 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be 5339 regimplified. */ 5340 5341 static tree 5342 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *) 5343 { 5344 tree t = *tp; 5345 5346 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ 5347 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t)) 5348 return t; 5349 5350 if (TREE_CODE (t) == ADDR_EXPR) 5351 recompute_tree_invariant_for_addr_expr (t); 5352 5353 *walk_subtrees = !TYPE_P (t) && !DECL_P (t); 5354 return NULL_TREE; 5355 } 5356 5357 /* Prepend TO = FROM assignment before *GSI_P. */ 5358 5359 static void 5360 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from) 5361 { 5362 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to); 5363 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE, 5364 true, GSI_SAME_STMT); 5365 gimple stmt = gimple_build_assign (to, from); 5366 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT); 5367 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL) 5368 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL)) 5369 { 5370 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 5371 gimple_regimplify_operands (stmt, &gsi); 5372 } 5373 } 5374 5375 /* Expand the OpenMP parallel or task directive starting at REGION. */ 5376 5377 static void 5378 expand_omp_taskreg (struct omp_region *region) 5379 { 5380 basic_block entry_bb, exit_bb, new_bb; 5381 struct function *child_cfun; 5382 tree child_fn, block, t; 5383 gimple_stmt_iterator gsi; 5384 gimple entry_stmt, stmt; 5385 edge e; 5386 vec<tree, va_gc> *ws_args; 5387 5388 entry_stmt = last_stmt (region->entry); 5389 child_fn = gimple_omp_taskreg_child_fn (entry_stmt); 5390 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 5391 5392 entry_bb = region->entry; 5393 if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK) 5394 exit_bb = region->cont; 5395 else 5396 exit_bb = region->exit; 5397 5398 bool is_cilk_for 5399 = (flag_cilkplus 5400 && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL 5401 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt), 5402 OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE); 5403 5404 if (is_cilk_for) 5405 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for, 5406 and the inner statement contains the name of the built-in function 5407 and grain. */ 5408 ws_args = region->inner->ws_args; 5409 else if (is_combined_parallel (region)) 5410 ws_args = region->ws_args; 5411 else 5412 ws_args = NULL; 5413 5414 if (child_cfun->cfg) 5415 { 5416 /* Due to inlining, it may happen that we have already outlined 5417 the region, in which case all we need to do is make the 5418 sub-graph unreachable and emit the parallel call. */ 5419 edge entry_succ_e, exit_succ_e; 5420 5421 entry_succ_e = single_succ_edge (entry_bb); 5422 5423 gsi = gsi_last_bb (entry_bb); 5424 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL 5425 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK); 5426 gsi_remove (&gsi, true); 5427 5428 new_bb = entry_bb; 5429 if (exit_bb) 5430 { 5431 exit_succ_e = single_succ_edge (exit_bb); 5432 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU); 5433 } 5434 remove_edge_and_dominated_blocks (entry_succ_e); 5435 } 5436 else 5437 { 5438 unsigned srcidx, dstidx, num; 5439 5440 /* If the parallel region needs data sent from the parent 5441 function, then the very first statement (except possible 5442 tree profile counter updates) of the parallel body 5443 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since 5444 &.OMP_DATA_O is passed as an argument to the child function, 5445 we need to replace it with the argument as seen by the child 5446 function. 5447 5448 In most cases, this will end up being the identity assignment 5449 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had 5450 a function call that has been inlined, the original PARM_DECL 5451 .OMP_DATA_I may have been converted into a different local 5452 variable. In which case, we need to keep the assignment. */ 5453 if (gimple_omp_taskreg_data_arg (entry_stmt)) 5454 { 5455 basic_block entry_succ_bb 5456 = single_succ_p (entry_bb) ? single_succ (entry_bb) 5457 : FALLTHRU_EDGE (entry_bb)->dest; 5458 tree arg, narg; 5459 gimple parcopy_stmt = NULL; 5460 5461 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi)) 5462 { 5463 gimple stmt; 5464 5465 gcc_assert (!gsi_end_p (gsi)); 5466 stmt = gsi_stmt (gsi); 5467 if (gimple_code (stmt) != GIMPLE_ASSIGN) 5468 continue; 5469 5470 if (gimple_num_ops (stmt) == 2) 5471 { 5472 tree arg = gimple_assign_rhs1 (stmt); 5473 5474 /* We're ignore the subcode because we're 5475 effectively doing a STRIP_NOPS. */ 5476 5477 if (TREE_CODE (arg) == ADDR_EXPR 5478 && TREE_OPERAND (arg, 0) 5479 == gimple_omp_taskreg_data_arg (entry_stmt)) 5480 { 5481 parcopy_stmt = stmt; 5482 break; 5483 } 5484 } 5485 } 5486 5487 gcc_assert (parcopy_stmt != NULL); 5488 arg = DECL_ARGUMENTS (child_fn); 5489 5490 if (!gimple_in_ssa_p (cfun)) 5491 { 5492 if (gimple_assign_lhs (parcopy_stmt) == arg) 5493 gsi_remove (&gsi, true); 5494 else 5495 { 5496 /* ?? Is setting the subcode really necessary ?? */ 5497 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg)); 5498 gimple_assign_set_rhs1 (parcopy_stmt, arg); 5499 } 5500 } 5501 else 5502 { 5503 /* If we are in ssa form, we must load the value from the default 5504 definition of the argument. That should not be defined now, 5505 since the argument is not used uninitialized. */ 5506 gcc_assert (ssa_default_def (cfun, arg) == NULL); 5507 narg = make_ssa_name (arg, gimple_build_nop ()); 5508 set_ssa_default_def (cfun, arg, narg); 5509 /* ?? Is setting the subcode really necessary ?? */ 5510 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg)); 5511 gimple_assign_set_rhs1 (parcopy_stmt, narg); 5512 update_stmt (parcopy_stmt); 5513 } 5514 } 5515 5516 /* Declare local variables needed in CHILD_CFUN. */ 5517 block = DECL_INITIAL (child_fn); 5518 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls); 5519 /* The gimplifier could record temporaries in parallel/task block 5520 rather than in containing function's local_decls chain, 5521 which would mean cgraph missed finalizing them. Do it now. */ 5522 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t)) 5523 if (TREE_CODE (t) == VAR_DECL 5524 && TREE_STATIC (t) 5525 && !DECL_EXTERNAL (t)) 5526 varpool_node::finalize_decl (t); 5527 DECL_SAVED_TREE (child_fn) = NULL; 5528 /* We'll create a CFG for child_fn, so no gimple body is needed. */ 5529 gimple_set_body (child_fn, NULL); 5530 TREE_USED (block) = 1; 5531 5532 /* Reset DECL_CONTEXT on function arguments. */ 5533 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) 5534 DECL_CONTEXT (t) = child_fn; 5535 5536 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK, 5537 so that it can be moved to the child function. */ 5538 gsi = gsi_last_bb (entry_bb); 5539 stmt = gsi_stmt (gsi); 5540 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL 5541 || gimple_code (stmt) == GIMPLE_OMP_TASK)); 5542 e = split_block (entry_bb, stmt); 5543 gsi_remove (&gsi, true); 5544 entry_bb = e->dest; 5545 edge e2 = NULL; 5546 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL) 5547 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 5548 else 5549 { 5550 e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL); 5551 gcc_assert (e2->dest == region->exit); 5552 remove_edge (BRANCH_EDGE (entry_bb)); 5553 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e->src); 5554 gsi = gsi_last_bb (region->exit); 5555 gcc_assert (!gsi_end_p (gsi) 5556 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); 5557 gsi_remove (&gsi, true); 5558 } 5559 5560 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */ 5561 if (exit_bb) 5562 { 5563 gsi = gsi_last_bb (exit_bb); 5564 gcc_assert (!gsi_end_p (gsi) 5565 && (gimple_code (gsi_stmt (gsi)) 5566 == (e2 ? GIMPLE_OMP_CONTINUE : GIMPLE_OMP_RETURN))); 5567 stmt = gimple_build_return (NULL); 5568 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); 5569 gsi_remove (&gsi, true); 5570 } 5571 5572 /* Move the parallel region into CHILD_CFUN. */ 5573 5574 if (gimple_in_ssa_p (cfun)) 5575 { 5576 init_tree_ssa (child_cfun); 5577 init_ssa_operands (child_cfun); 5578 child_cfun->gimple_df->in_ssa_p = true; 5579 block = NULL_TREE; 5580 } 5581 else 5582 block = gimple_block (entry_stmt); 5583 5584 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block); 5585 if (exit_bb) 5586 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU; 5587 if (e2) 5588 { 5589 basic_block dest_bb = e2->dest; 5590 if (!exit_bb) 5591 make_edge (new_bb, dest_bb, EDGE_FALLTHRU); 5592 remove_edge (e2); 5593 set_immediate_dominator (CDI_DOMINATORS, dest_bb, new_bb); 5594 } 5595 /* When the OMP expansion process cannot guarantee an up-to-date 5596 loop tree arrange for the child function to fixup loops. */ 5597 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP)) 5598 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP; 5599 5600 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */ 5601 num = vec_safe_length (child_cfun->local_decls); 5602 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++) 5603 { 5604 t = (*child_cfun->local_decls)[srcidx]; 5605 if (DECL_CONTEXT (t) == cfun->decl) 5606 continue; 5607 if (srcidx != dstidx) 5608 (*child_cfun->local_decls)[dstidx] = t; 5609 dstidx++; 5610 } 5611 if (dstidx != num) 5612 vec_safe_truncate (child_cfun->local_decls, dstidx); 5613 5614 /* Inform the callgraph about the new function. */ 5615 child_cfun->curr_properties = cfun->curr_properties; 5616 child_cfun->has_simduid_loops |= cfun->has_simduid_loops; 5617 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops; 5618 cgraph_node::add_new_function (child_fn, true); 5619 cgraph_node::get (child_fn)->parallelized_function = 1; 5620 5621 /* Fix the callgraph edges for child_cfun. Those for cfun will be 5622 fixed in a following pass. */ 5623 push_cfun (child_cfun); 5624 if (optimize) 5625 optimize_omp_library_calls (entry_stmt); 5626 cgraph_edge::rebuild_edges (); 5627 5628 /* Some EH regions might become dead, see PR34608. If 5629 pass_cleanup_cfg isn't the first pass to happen with the 5630 new child, these dead EH edges might cause problems. 5631 Clean them up now. */ 5632 if (flag_exceptions) 5633 { 5634 basic_block bb; 5635 bool changed = false; 5636 5637 FOR_EACH_BB_FN (bb, cfun) 5638 changed |= gimple_purge_dead_eh_edges (bb); 5639 if (changed) 5640 cleanup_tree_cfg (); 5641 } 5642 if (gimple_in_ssa_p (cfun)) 5643 update_ssa (TODO_update_ssa); 5644 pop_cfun (); 5645 } 5646 5647 /* Emit a library call to launch the children threads. */ 5648 if (is_cilk_for) 5649 expand_cilk_for_call (new_bb, 5650 as_a <gomp_parallel *> (entry_stmt), ws_args); 5651 else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL) 5652 expand_parallel_call (region, new_bb, 5653 as_a <gomp_parallel *> (entry_stmt), ws_args); 5654 else 5655 expand_task_call (new_bb, as_a <gomp_task *> (entry_stmt)); 5656 if (gimple_in_ssa_p (cfun)) 5657 update_ssa (TODO_update_ssa_only_virtuals); 5658 } 5659 5660 5661 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost 5662 of the combined collapse > 1 loop constructs, generate code like: 5663 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB; 5664 if (cond3 is <) 5665 adj = STEP3 - 1; 5666 else 5667 adj = STEP3 + 1; 5668 count3 = (adj + N32 - N31) / STEP3; 5669 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB; 5670 if (cond2 is <) 5671 adj = STEP2 - 1; 5672 else 5673 adj = STEP2 + 1; 5674 count2 = (adj + N22 - N21) / STEP2; 5675 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB; 5676 if (cond1 is <) 5677 adj = STEP1 - 1; 5678 else 5679 adj = STEP1 + 1; 5680 count1 = (adj + N12 - N11) / STEP1; 5681 count = count1 * count2 * count3; 5682 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does: 5683 count = 0; 5684 and set ZERO_ITER_BB to that bb. If this isn't the outermost 5685 of the combined loop constructs, just initialize COUNTS array 5686 from the _looptemp_ clauses. */ 5687 5688 /* NOTE: It *could* be better to moosh all of the BBs together, 5689 creating one larger BB with all the computation and the unexpected 5690 jump at the end. I.e. 5691 5692 bool zero3, zero2, zero1, zero; 5693 5694 zero3 = N32 c3 N31; 5695 count3 = (N32 - N31) /[cl] STEP3; 5696 zero2 = N22 c2 N21; 5697 count2 = (N22 - N21) /[cl] STEP2; 5698 zero1 = N12 c1 N11; 5699 count1 = (N12 - N11) /[cl] STEP1; 5700 zero = zero3 || zero2 || zero1; 5701 count = count1 * count2 * count3; 5702 if (__builtin_expect(zero, false)) goto zero_iter_bb; 5703 5704 After all, we expect the zero=false, and thus we expect to have to 5705 evaluate all of the comparison expressions, so short-circuiting 5706 oughtn't be a win. Since the condition isn't protecting a 5707 denominator, we're not concerned about divide-by-zero, so we can 5708 fully evaluate count even if a numerator turned out to be wrong. 5709 5710 It seems like putting this all together would create much better 5711 scheduling opportunities, and less pressure on the chip's branch 5712 predictor. */ 5713 5714 static void 5715 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi, 5716 basic_block &entry_bb, tree *counts, 5717 basic_block &zero_iter_bb, int &first_zero_iter, 5718 basic_block &l2_dom_bb) 5719 { 5720 tree t, type = TREE_TYPE (fd->loop.v); 5721 edge e, ne; 5722 int i; 5723 5724 /* Collapsed loops need work for expansion into SSA form. */ 5725 gcc_assert (!gimple_in_ssa_p (cfun)); 5726 5727 if (gimple_omp_for_combined_into_p (fd->for_stmt) 5728 && TREE_CODE (fd->loop.n2) != INTEGER_CST) 5729 { 5730 /* First two _looptemp_ clauses are for istart/iend, counts[0] 5731 isn't supposed to be handled, as the inner loop doesn't 5732 use it. */ 5733 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 5734 OMP_CLAUSE__LOOPTEMP_); 5735 gcc_assert (innerc); 5736 for (i = 0; i < fd->collapse; i++) 5737 { 5738 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 5739 OMP_CLAUSE__LOOPTEMP_); 5740 gcc_assert (innerc); 5741 if (i) 5742 counts[i] = OMP_CLAUSE_DECL (innerc); 5743 else 5744 counts[0] = NULL_TREE; 5745 } 5746 return; 5747 } 5748 5749 for (i = 0; i < fd->collapse; i++) 5750 { 5751 tree itype = TREE_TYPE (fd->loops[i].v); 5752 5753 if (SSA_VAR_P (fd->loop.n2) 5754 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node, 5755 fold_convert (itype, fd->loops[i].n1), 5756 fold_convert (itype, fd->loops[i].n2))) 5757 == NULL_TREE || !integer_onep (t))) 5758 { 5759 gcond *cond_stmt; 5760 tree n1, n2; 5761 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1)); 5762 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE, 5763 true, GSI_SAME_STMT); 5764 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2)); 5765 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE, 5766 true, GSI_SAME_STMT); 5767 cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2, 5768 NULL_TREE, NULL_TREE); 5769 gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT); 5770 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 5771 expand_omp_regimplify_p, NULL, NULL) 5772 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 5773 expand_omp_regimplify_p, NULL, NULL)) 5774 { 5775 *gsi = gsi_for_stmt (cond_stmt); 5776 gimple_regimplify_operands (cond_stmt, gsi); 5777 } 5778 e = split_block (entry_bb, cond_stmt); 5779 if (zero_iter_bb == NULL) 5780 { 5781 gassign *assign_stmt; 5782 first_zero_iter = i; 5783 zero_iter_bb = create_empty_bb (entry_bb); 5784 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father); 5785 *gsi = gsi_after_labels (zero_iter_bb); 5786 assign_stmt = gimple_build_assign (fd->loop.n2, 5787 build_zero_cst (type)); 5788 gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT); 5789 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb, 5790 entry_bb); 5791 } 5792 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE); 5793 ne->probability = REG_BR_PROB_BASE / 2000 - 1; 5794 e->flags = EDGE_TRUE_VALUE; 5795 e->probability = REG_BR_PROB_BASE - ne->probability; 5796 if (l2_dom_bb == NULL) 5797 l2_dom_bb = entry_bb; 5798 entry_bb = e->dest; 5799 *gsi = gsi_last_bb (entry_bb); 5800 } 5801 5802 if (POINTER_TYPE_P (itype)) 5803 itype = signed_type_for (itype); 5804 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR 5805 ? -1 : 1)); 5806 t = fold_build2 (PLUS_EXPR, itype, 5807 fold_convert (itype, fd->loops[i].step), t); 5808 t = fold_build2 (PLUS_EXPR, itype, t, 5809 fold_convert (itype, fd->loops[i].n2)); 5810 t = fold_build2 (MINUS_EXPR, itype, t, 5811 fold_convert (itype, fd->loops[i].n1)); 5812 /* ?? We could probably use CEIL_DIV_EXPR instead of 5813 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't 5814 generate the same code in the end because generically we 5815 don't know that the values involved must be negative for 5816 GT?? */ 5817 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR) 5818 t = fold_build2 (TRUNC_DIV_EXPR, itype, 5819 fold_build1 (NEGATE_EXPR, itype, t), 5820 fold_build1 (NEGATE_EXPR, itype, 5821 fold_convert (itype, 5822 fd->loops[i].step))); 5823 else 5824 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, 5825 fold_convert (itype, fd->loops[i].step)); 5826 t = fold_convert (type, t); 5827 if (TREE_CODE (t) == INTEGER_CST) 5828 counts[i] = t; 5829 else 5830 { 5831 counts[i] = create_tmp_reg (type, ".count"); 5832 expand_omp_build_assign (gsi, counts[i], t); 5833 } 5834 if (SSA_VAR_P (fd->loop.n2)) 5835 { 5836 if (i == 0) 5837 t = counts[0]; 5838 else 5839 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]); 5840 expand_omp_build_assign (gsi, fd->loop.n2, t); 5841 } 5842 } 5843 } 5844 5845 5846 /* Helper function for expand_omp_{for_*,simd}. Generate code like: 5847 T = V; 5848 V3 = N31 + (T % count3) * STEP3; 5849 T = T / count3; 5850 V2 = N21 + (T % count2) * STEP2; 5851 T = T / count2; 5852 V1 = N11 + T * STEP1; 5853 if this loop doesn't have an inner loop construct combined with it. 5854 If it does have an inner loop construct combined with it and the 5855 iteration count isn't known constant, store values from counts array 5856 into its _looptemp_ temporaries instead. */ 5857 5858 static void 5859 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi, 5860 tree *counts, gimple inner_stmt, tree startvar) 5861 { 5862 int i; 5863 if (gimple_omp_for_combined_p (fd->for_stmt)) 5864 { 5865 /* If fd->loop.n2 is constant, then no propagation of the counts 5866 is needed, they are constant. */ 5867 if (TREE_CODE (fd->loop.n2) == INTEGER_CST) 5868 return; 5869 5870 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL 5871 ? gimple_omp_parallel_clauses (inner_stmt) 5872 : gimple_omp_for_clauses (inner_stmt); 5873 /* First two _looptemp_ clauses are for istart/iend, counts[0] 5874 isn't supposed to be handled, as the inner loop doesn't 5875 use it. */ 5876 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); 5877 gcc_assert (innerc); 5878 for (i = 0; i < fd->collapse; i++) 5879 { 5880 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 5881 OMP_CLAUSE__LOOPTEMP_); 5882 gcc_assert (innerc); 5883 if (i) 5884 { 5885 tree tem = OMP_CLAUSE_DECL (innerc); 5886 tree t = fold_convert (TREE_TYPE (tem), counts[i]); 5887 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE, 5888 false, GSI_CONTINUE_LINKING); 5889 gassign *stmt = gimple_build_assign (tem, t); 5890 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5891 } 5892 } 5893 return; 5894 } 5895 5896 tree type = TREE_TYPE (fd->loop.v); 5897 tree tem = create_tmp_reg (type, ".tem"); 5898 gassign *stmt = gimple_build_assign (tem, startvar); 5899 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5900 5901 for (i = fd->collapse - 1; i >= 0; i--) 5902 { 5903 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t; 5904 itype = vtype; 5905 if (POINTER_TYPE_P (vtype)) 5906 itype = signed_type_for (vtype); 5907 if (i != 0) 5908 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]); 5909 else 5910 t = tem; 5911 t = fold_convert (itype, t); 5912 t = fold_build2 (MULT_EXPR, itype, t, 5913 fold_convert (itype, fd->loops[i].step)); 5914 if (POINTER_TYPE_P (vtype)) 5915 t = fold_build_pointer_plus (fd->loops[i].n1, t); 5916 else 5917 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t); 5918 t = force_gimple_operand_gsi (gsi, t, 5919 DECL_P (fd->loops[i].v) 5920 && TREE_ADDRESSABLE (fd->loops[i].v), 5921 NULL_TREE, false, 5922 GSI_CONTINUE_LINKING); 5923 stmt = gimple_build_assign (fd->loops[i].v, t); 5924 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5925 if (i != 0) 5926 { 5927 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]); 5928 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE, 5929 false, GSI_CONTINUE_LINKING); 5930 stmt = gimple_build_assign (tem, t); 5931 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5932 } 5933 } 5934 } 5935 5936 5937 /* Helper function for expand_omp_for_*. Generate code like: 5938 L10: 5939 V3 += STEP3; 5940 if (V3 cond3 N32) goto BODY_BB; else goto L11; 5941 L11: 5942 V3 = N31; 5943 V2 += STEP2; 5944 if (V2 cond2 N22) goto BODY_BB; else goto L12; 5945 L12: 5946 V2 = N21; 5947 V1 += STEP1; 5948 goto BODY_BB; */ 5949 5950 static basic_block 5951 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb, 5952 basic_block body_bb) 5953 { 5954 basic_block last_bb, bb, collapse_bb = NULL; 5955 int i; 5956 gimple_stmt_iterator gsi; 5957 edge e; 5958 tree t; 5959 gimple stmt; 5960 5961 last_bb = cont_bb; 5962 for (i = fd->collapse - 1; i >= 0; i--) 5963 { 5964 tree vtype = TREE_TYPE (fd->loops[i].v); 5965 5966 bb = create_empty_bb (last_bb); 5967 add_bb_to_loop (bb, last_bb->loop_father); 5968 gsi = gsi_start_bb (bb); 5969 5970 if (i < fd->collapse - 1) 5971 { 5972 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE); 5973 e->probability = REG_BR_PROB_BASE / 8; 5974 5975 t = fd->loops[i + 1].n1; 5976 t = force_gimple_operand_gsi (&gsi, t, 5977 DECL_P (fd->loops[i + 1].v) 5978 && TREE_ADDRESSABLE (fd->loops[i 5979 + 1].v), 5980 NULL_TREE, false, 5981 GSI_CONTINUE_LINKING); 5982 stmt = gimple_build_assign (fd->loops[i + 1].v, t); 5983 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 5984 } 5985 else 5986 collapse_bb = bb; 5987 5988 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb); 5989 5990 if (POINTER_TYPE_P (vtype)) 5991 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step); 5992 else 5993 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step); 5994 t = force_gimple_operand_gsi (&gsi, t, 5995 DECL_P (fd->loops[i].v) 5996 && TREE_ADDRESSABLE (fd->loops[i].v), 5997 NULL_TREE, false, GSI_CONTINUE_LINKING); 5998 stmt = gimple_build_assign (fd->loops[i].v, t); 5999 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 6000 6001 if (i > 0) 6002 { 6003 t = fd->loops[i].n2; 6004 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6005 false, GSI_CONTINUE_LINKING); 6006 tree v = fd->loops[i].v; 6007 if (DECL_P (v) && TREE_ADDRESSABLE (v)) 6008 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE, 6009 false, GSI_CONTINUE_LINKING); 6010 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t); 6011 stmt = gimple_build_cond_empty (t); 6012 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 6013 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE); 6014 e->probability = REG_BR_PROB_BASE * 7 / 8; 6015 } 6016 else 6017 make_edge (bb, body_bb, EDGE_FALLTHRU); 6018 last_bb = bb; 6019 } 6020 6021 return collapse_bb; 6022 } 6023 6024 6025 /* A subroutine of expand_omp_for. Generate code for a parallel 6026 loop with any schedule. Given parameters: 6027 6028 for (V = N1; V cond N2; V += STEP) BODY; 6029 6030 where COND is "<" or ">", we generate pseudocode 6031 6032 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0); 6033 if (more) goto L0; else goto L3; 6034 L0: 6035 V = istart0; 6036 iend = iend0; 6037 L1: 6038 BODY; 6039 V += STEP; 6040 if (V cond iend) goto L1; else goto L2; 6041 L2: 6042 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3; 6043 L3: 6044 6045 If this is a combined omp parallel loop, instead of the call to 6046 GOMP_loop_foo_start, we call GOMP_loop_foo_next. 6047 If this is gimple_omp_for_combined_p loop, then instead of assigning 6048 V and iend in L0 we assign the first two _looptemp_ clause decls of the 6049 inner GIMPLE_OMP_FOR and V += STEP; and 6050 if (V cond iend) goto L1; else goto L2; are removed. 6051 6052 For collapsed loops, given parameters: 6053 collapse(3) 6054 for (V1 = N11; V1 cond1 N12; V1 += STEP1) 6055 for (V2 = N21; V2 cond2 N22; V2 += STEP2) 6056 for (V3 = N31; V3 cond3 N32; V3 += STEP3) 6057 BODY; 6058 6059 we generate pseudocode 6060 6061 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0; 6062 if (cond3 is <) 6063 adj = STEP3 - 1; 6064 else 6065 adj = STEP3 + 1; 6066 count3 = (adj + N32 - N31) / STEP3; 6067 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0; 6068 if (cond2 is <) 6069 adj = STEP2 - 1; 6070 else 6071 adj = STEP2 + 1; 6072 count2 = (adj + N22 - N21) / STEP2; 6073 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0; 6074 if (cond1 is <) 6075 adj = STEP1 - 1; 6076 else 6077 adj = STEP1 + 1; 6078 count1 = (adj + N12 - N11) / STEP1; 6079 count = count1 * count2 * count3; 6080 goto Z1; 6081 Z0: 6082 count = 0; 6083 Z1: 6084 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0); 6085 if (more) goto L0; else goto L3; 6086 L0: 6087 V = istart0; 6088 T = V; 6089 V3 = N31 + (T % count3) * STEP3; 6090 T = T / count3; 6091 V2 = N21 + (T % count2) * STEP2; 6092 T = T / count2; 6093 V1 = N11 + T * STEP1; 6094 iend = iend0; 6095 L1: 6096 BODY; 6097 V += 1; 6098 if (V < iend) goto L10; else goto L2; 6099 L10: 6100 V3 += STEP3; 6101 if (V3 cond3 N32) goto L1; else goto L11; 6102 L11: 6103 V3 = N31; 6104 V2 += STEP2; 6105 if (V2 cond2 N22) goto L1; else goto L12; 6106 L12: 6107 V2 = N21; 6108 V1 += STEP1; 6109 goto L1; 6110 L2: 6111 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3; 6112 L3: 6113 6114 */ 6115 6116 static void 6117 expand_omp_for_generic (struct omp_region *region, 6118 struct omp_for_data *fd, 6119 enum built_in_function start_fn, 6120 enum built_in_function next_fn, 6121 gimple inner_stmt) 6122 { 6123 tree type, istart0, iend0, iend; 6124 tree t, vmain, vback, bias = NULL_TREE; 6125 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb; 6126 basic_block l2_bb = NULL, l3_bb = NULL; 6127 gimple_stmt_iterator gsi; 6128 gassign *assign_stmt; 6129 bool in_combined_parallel = is_combined_parallel (region); 6130 bool broken_loop = region->cont == NULL; 6131 edge e, ne; 6132 tree *counts = NULL; 6133 int i; 6134 6135 gcc_assert (!broken_loop || !in_combined_parallel); 6136 gcc_assert (fd->iter_type == long_integer_type_node 6137 || !in_combined_parallel); 6138 6139 type = TREE_TYPE (fd->loop.v); 6140 istart0 = create_tmp_var (fd->iter_type, ".istart0"); 6141 iend0 = create_tmp_var (fd->iter_type, ".iend0"); 6142 TREE_ADDRESSABLE (istart0) = 1; 6143 TREE_ADDRESSABLE (iend0) = 1; 6144 6145 /* See if we need to bias by LLONG_MIN. */ 6146 if (fd->iter_type == long_long_unsigned_type_node 6147 && TREE_CODE (type) == INTEGER_TYPE 6148 && !TYPE_UNSIGNED (type)) 6149 { 6150 tree n1, n2; 6151 6152 if (fd->loop.cond_code == LT_EXPR) 6153 { 6154 n1 = fd->loop.n1; 6155 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step); 6156 } 6157 else 6158 { 6159 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step); 6160 n2 = fd->loop.n1; 6161 } 6162 if (TREE_CODE (n1) != INTEGER_CST 6163 || TREE_CODE (n2) != INTEGER_CST 6164 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0))) 6165 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type)); 6166 } 6167 6168 entry_bb = region->entry; 6169 cont_bb = region->cont; 6170 collapse_bb = NULL; 6171 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 6172 gcc_assert (broken_loop 6173 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); 6174 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb)); 6175 l1_bb = single_succ (l0_bb); 6176 if (!broken_loop) 6177 { 6178 l2_bb = create_empty_bb (cont_bb); 6179 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb); 6180 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 6181 } 6182 else 6183 l2_bb = NULL; 6184 l3_bb = BRANCH_EDGE (entry_bb)->dest; 6185 exit_bb = region->exit; 6186 6187 gsi = gsi_last_bb (entry_bb); 6188 6189 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6190 if (fd->collapse > 1) 6191 { 6192 int first_zero_iter = -1; 6193 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL; 6194 6195 counts = XALLOCAVEC (tree, fd->collapse); 6196 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 6197 zero_iter_bb, first_zero_iter, 6198 l2_dom_bb); 6199 6200 if (zero_iter_bb) 6201 { 6202 /* Some counts[i] vars might be uninitialized if 6203 some loop has zero iterations. But the body shouldn't 6204 be executed in that case, so just avoid uninit warnings. */ 6205 for (i = first_zero_iter; i < fd->collapse; i++) 6206 if (SSA_VAR_P (counts[i])) 6207 TREE_NO_WARNING (counts[i]) = 1; 6208 gsi_prev (&gsi); 6209 e = split_block (entry_bb, gsi_stmt (gsi)); 6210 entry_bb = e->dest; 6211 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU); 6212 gsi = gsi_last_bb (entry_bb); 6213 set_immediate_dominator (CDI_DOMINATORS, entry_bb, 6214 get_immediate_dominator (CDI_DOMINATORS, 6215 zero_iter_bb)); 6216 } 6217 } 6218 if (in_combined_parallel) 6219 { 6220 /* In a combined parallel loop, emit a call to 6221 GOMP_loop_foo_next. */ 6222 t = build_call_expr (builtin_decl_explicit (next_fn), 2, 6223 build_fold_addr_expr (istart0), 6224 build_fold_addr_expr (iend0)); 6225 } 6226 else 6227 { 6228 tree t0, t1, t2, t3, t4; 6229 /* If this is not a combined parallel loop, emit a call to 6230 GOMP_loop_foo_start in ENTRY_BB. */ 6231 t4 = build_fold_addr_expr (iend0); 6232 t3 = build_fold_addr_expr (istart0); 6233 t2 = fold_convert (fd->iter_type, fd->loop.step); 6234 t1 = fd->loop.n2; 6235 t0 = fd->loop.n1; 6236 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6237 { 6238 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 6239 OMP_CLAUSE__LOOPTEMP_); 6240 gcc_assert (innerc); 6241 t0 = OMP_CLAUSE_DECL (innerc); 6242 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6243 OMP_CLAUSE__LOOPTEMP_); 6244 gcc_assert (innerc); 6245 t1 = OMP_CLAUSE_DECL (innerc); 6246 } 6247 if (POINTER_TYPE_P (TREE_TYPE (t0)) 6248 && TYPE_PRECISION (TREE_TYPE (t0)) 6249 != TYPE_PRECISION (fd->iter_type)) 6250 { 6251 /* Avoid casting pointers to integer of a different size. */ 6252 tree itype = signed_type_for (type); 6253 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1)); 6254 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0)); 6255 } 6256 else 6257 { 6258 t1 = fold_convert (fd->iter_type, t1); 6259 t0 = fold_convert (fd->iter_type, t0); 6260 } 6261 if (bias) 6262 { 6263 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias); 6264 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias); 6265 } 6266 if (fd->iter_type == long_integer_type_node) 6267 { 6268 if (fd->chunk_size) 6269 { 6270 t = fold_convert (fd->iter_type, fd->chunk_size); 6271 t = build_call_expr (builtin_decl_explicit (start_fn), 6272 6, t0, t1, t2, t, t3, t4); 6273 } 6274 else 6275 t = build_call_expr (builtin_decl_explicit (start_fn), 6276 5, t0, t1, t2, t3, t4); 6277 } 6278 else 6279 { 6280 tree t5; 6281 tree c_bool_type; 6282 tree bfn_decl; 6283 6284 /* The GOMP_loop_ull_*start functions have additional boolean 6285 argument, true for < loops and false for > loops. 6286 In Fortran, the C bool type can be different from 6287 boolean_type_node. */ 6288 bfn_decl = builtin_decl_explicit (start_fn); 6289 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl)); 6290 t5 = build_int_cst (c_bool_type, 6291 fd->loop.cond_code == LT_EXPR ? 1 : 0); 6292 if (fd->chunk_size) 6293 { 6294 tree bfn_decl = builtin_decl_explicit (start_fn); 6295 t = fold_convert (fd->iter_type, fd->chunk_size); 6296 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4); 6297 } 6298 else 6299 t = build_call_expr (builtin_decl_explicit (start_fn), 6300 6, t5, t0, t1, t2, t3, t4); 6301 } 6302 } 6303 if (TREE_TYPE (t) != boolean_type_node) 6304 t = fold_build2 (NE_EXPR, boolean_type_node, 6305 t, build_int_cst (TREE_TYPE (t), 0)); 6306 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6307 true, GSI_SAME_STMT); 6308 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 6309 6310 /* Remove the GIMPLE_OMP_FOR statement. */ 6311 gsi_remove (&gsi, true); 6312 6313 /* Iteration setup for sequential loop goes in L0_BB. */ 6314 tree startvar = fd->loop.v; 6315 tree endvar = NULL_TREE; 6316 6317 if (gimple_omp_for_combined_p (fd->for_stmt)) 6318 { 6319 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR 6320 && gimple_omp_for_kind (inner_stmt) 6321 == GF_OMP_FOR_KIND_SIMD); 6322 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt), 6323 OMP_CLAUSE__LOOPTEMP_); 6324 gcc_assert (innerc); 6325 startvar = OMP_CLAUSE_DECL (innerc); 6326 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6327 OMP_CLAUSE__LOOPTEMP_); 6328 gcc_assert (innerc); 6329 endvar = OMP_CLAUSE_DECL (innerc); 6330 } 6331 6332 gsi = gsi_start_bb (l0_bb); 6333 t = istart0; 6334 if (bias) 6335 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias); 6336 if (POINTER_TYPE_P (TREE_TYPE (startvar))) 6337 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t); 6338 t = fold_convert (TREE_TYPE (startvar), t); 6339 t = force_gimple_operand_gsi (&gsi, t, 6340 DECL_P (startvar) 6341 && TREE_ADDRESSABLE (startvar), 6342 NULL_TREE, false, GSI_CONTINUE_LINKING); 6343 assign_stmt = gimple_build_assign (startvar, t); 6344 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6345 6346 t = iend0; 6347 if (bias) 6348 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias); 6349 if (POINTER_TYPE_P (TREE_TYPE (startvar))) 6350 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t); 6351 t = fold_convert (TREE_TYPE (startvar), t); 6352 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6353 false, GSI_CONTINUE_LINKING); 6354 if (endvar) 6355 { 6356 assign_stmt = gimple_build_assign (endvar, iend); 6357 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6358 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend))) 6359 assign_stmt = gimple_build_assign (fd->loop.v, iend); 6360 else 6361 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend); 6362 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6363 } 6364 if (fd->collapse > 1) 6365 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); 6366 6367 if (!broken_loop) 6368 { 6369 /* Code to control the increment and predicate for the sequential 6370 loop goes in the CONT_BB. */ 6371 gsi = gsi_last_bb (cont_bb); 6372 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); 6373 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE); 6374 vmain = gimple_omp_continue_control_use (cont_stmt); 6375 vback = gimple_omp_continue_control_def (cont_stmt); 6376 6377 if (!gimple_omp_for_combined_p (fd->for_stmt)) 6378 { 6379 if (POINTER_TYPE_P (type)) 6380 t = fold_build_pointer_plus (vmain, fd->loop.step); 6381 else 6382 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step); 6383 t = force_gimple_operand_gsi (&gsi, t, 6384 DECL_P (vback) 6385 && TREE_ADDRESSABLE (vback), 6386 NULL_TREE, true, GSI_SAME_STMT); 6387 assign_stmt = gimple_build_assign (vback, t); 6388 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 6389 6390 t = build2 (fd->loop.cond_code, boolean_type_node, 6391 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback, 6392 iend); 6393 gcond *cond_stmt = gimple_build_cond_empty (t); 6394 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6395 } 6396 6397 /* Remove GIMPLE_OMP_CONTINUE. */ 6398 gsi_remove (&gsi, true); 6399 6400 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) 6401 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb); 6402 6403 /* Emit code to get the next parallel iteration in L2_BB. */ 6404 gsi = gsi_start_bb (l2_bb); 6405 6406 t = build_call_expr (builtin_decl_explicit (next_fn), 2, 6407 build_fold_addr_expr (istart0), 6408 build_fold_addr_expr (iend0)); 6409 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6410 false, GSI_CONTINUE_LINKING); 6411 if (TREE_TYPE (t) != boolean_type_node) 6412 t = fold_build2 (NE_EXPR, boolean_type_node, 6413 t, build_int_cst (TREE_TYPE (t), 0)); 6414 gcond *cond_stmt = gimple_build_cond_empty (t); 6415 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING); 6416 } 6417 6418 /* Add the loop cleanup function. */ 6419 gsi = gsi_last_bb (exit_bb); 6420 if (gimple_omp_return_nowait_p (gsi_stmt (gsi))) 6421 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT); 6422 else if (gimple_omp_return_lhs (gsi_stmt (gsi))) 6423 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL); 6424 else 6425 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END); 6426 gcall *call_stmt = gimple_build_call (t, 0); 6427 if (gimple_omp_return_lhs (gsi_stmt (gsi))) 6428 gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi))); 6429 gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT); 6430 gsi_remove (&gsi, true); 6431 6432 /* Connect the new blocks. */ 6433 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE; 6434 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE; 6435 6436 if (!broken_loop) 6437 { 6438 gimple_seq phis; 6439 6440 e = find_edge (cont_bb, l3_bb); 6441 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE); 6442 6443 phis = phi_nodes (l3_bb); 6444 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi)) 6445 { 6446 gimple phi = gsi_stmt (gsi); 6447 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne), 6448 PHI_ARG_DEF_FROM_EDGE (phi, e)); 6449 } 6450 remove_edge (e); 6451 6452 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE); 6453 add_bb_to_loop (l2_bb, cont_bb->loop_father); 6454 e = find_edge (cont_bb, l1_bb); 6455 if (gimple_omp_for_combined_p (fd->for_stmt)) 6456 { 6457 remove_edge (e); 6458 e = NULL; 6459 } 6460 else if (fd->collapse > 1) 6461 { 6462 remove_edge (e); 6463 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); 6464 } 6465 else 6466 e->flags = EDGE_TRUE_VALUE; 6467 if (e) 6468 { 6469 e->probability = REG_BR_PROB_BASE * 7 / 8; 6470 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8; 6471 } 6472 else 6473 { 6474 e = find_edge (cont_bb, l2_bb); 6475 e->flags = EDGE_FALLTHRU; 6476 } 6477 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE); 6478 6479 set_immediate_dominator (CDI_DOMINATORS, l2_bb, 6480 recompute_dominator (CDI_DOMINATORS, l2_bb)); 6481 set_immediate_dominator (CDI_DOMINATORS, l3_bb, 6482 recompute_dominator (CDI_DOMINATORS, l3_bb)); 6483 set_immediate_dominator (CDI_DOMINATORS, l0_bb, 6484 recompute_dominator (CDI_DOMINATORS, l0_bb)); 6485 set_immediate_dominator (CDI_DOMINATORS, l1_bb, 6486 recompute_dominator (CDI_DOMINATORS, l1_bb)); 6487 6488 struct loop *outer_loop = alloc_loop (); 6489 outer_loop->header = l0_bb; 6490 outer_loop->latch = l2_bb; 6491 add_loop (outer_loop, l0_bb->loop_father); 6492 6493 if (!gimple_omp_for_combined_p (fd->for_stmt)) 6494 { 6495 struct loop *loop = alloc_loop (); 6496 loop->header = l1_bb; 6497 /* The loop may have multiple latches. */ 6498 add_loop (loop, outer_loop); 6499 } 6500 } 6501 } 6502 6503 6504 /* A subroutine of expand_omp_for. Generate code for a parallel 6505 loop with static schedule and no specified chunk size. Given 6506 parameters: 6507 6508 for (V = N1; V cond N2; V += STEP) BODY; 6509 6510 where COND is "<" or ">", we generate pseudocode 6511 6512 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2; 6513 if (cond is <) 6514 adj = STEP - 1; 6515 else 6516 adj = STEP + 1; 6517 if ((__typeof (V)) -1 > 0 && cond is >) 6518 n = -(adj + N2 - N1) / -STEP; 6519 else 6520 n = (adj + N2 - N1) / STEP; 6521 q = n / nthreads; 6522 tt = n % nthreads; 6523 if (threadid < tt) goto L3; else goto L4; 6524 L3: 6525 tt = 0; 6526 q = q + 1; 6527 L4: 6528 s0 = q * threadid + tt; 6529 e0 = s0 + q; 6530 V = s0 * STEP + N1; 6531 if (s0 >= e0) goto L2; else goto L0; 6532 L0: 6533 e = e0 * STEP + N1; 6534 L1: 6535 BODY; 6536 V += STEP; 6537 if (V cond e) goto L1; 6538 L2: 6539 */ 6540 6541 static void 6542 expand_omp_for_static_nochunk (struct omp_region *region, 6543 struct omp_for_data *fd, 6544 gimple inner_stmt) 6545 { 6546 tree n, q, s0, e0, e, t, tt, nthreads, threadid; 6547 tree type, itype, vmain, vback; 6548 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb; 6549 basic_block body_bb, cont_bb, collapse_bb = NULL; 6550 basic_block fin_bb; 6551 gimple_stmt_iterator gsi; 6552 edge ep; 6553 bool broken_loop = region->cont == NULL; 6554 tree *counts = NULL; 6555 tree n1, n2, step; 6556 6557 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt) 6558 != GF_OMP_FOR_KIND_OACC_LOOP) 6559 || !inner_stmt); 6560 6561 itype = type = TREE_TYPE (fd->loop.v); 6562 if (POINTER_TYPE_P (type)) 6563 itype = signed_type_for (type); 6564 6565 entry_bb = region->entry; 6566 cont_bb = region->cont; 6567 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 6568 fin_bb = BRANCH_EDGE (entry_bb)->dest; 6569 gcc_assert (broken_loop 6570 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest)); 6571 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb)); 6572 body_bb = single_succ (seq_start_bb); 6573 if (!broken_loop) 6574 { 6575 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb); 6576 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 6577 } 6578 exit_bb = region->exit; 6579 6580 /* Iteration space partitioning goes in ENTRY_BB. */ 6581 gsi = gsi_last_bb (entry_bb); 6582 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6583 6584 if (fd->collapse > 1) 6585 { 6586 int first_zero_iter = -1; 6587 basic_block l2_dom_bb = NULL; 6588 6589 counts = XALLOCAVEC (tree, fd->collapse); 6590 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 6591 fin_bb, first_zero_iter, 6592 l2_dom_bb); 6593 t = NULL_TREE; 6594 } 6595 else if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6596 t = integer_one_node; 6597 else 6598 t = fold_binary (fd->loop.cond_code, boolean_type_node, 6599 fold_convert (type, fd->loop.n1), 6600 fold_convert (type, fd->loop.n2)); 6601 if (fd->collapse == 1 6602 && TYPE_UNSIGNED (type) 6603 && (t == NULL_TREE || !integer_onep (t))) 6604 { 6605 n1 = fold_convert (type, unshare_expr (fd->loop.n1)); 6606 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE, 6607 true, GSI_SAME_STMT); 6608 n2 = fold_convert (type, unshare_expr (fd->loop.n2)); 6609 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE, 6610 true, GSI_SAME_STMT); 6611 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2, 6612 NULL_TREE, NULL_TREE); 6613 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6614 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 6615 expand_omp_regimplify_p, NULL, NULL) 6616 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 6617 expand_omp_regimplify_p, NULL, NULL)) 6618 { 6619 gsi = gsi_for_stmt (cond_stmt); 6620 gimple_regimplify_operands (cond_stmt, &gsi); 6621 } 6622 ep = split_block (entry_bb, cond_stmt); 6623 ep->flags = EDGE_TRUE_VALUE; 6624 entry_bb = ep->dest; 6625 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); 6626 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE); 6627 ep->probability = REG_BR_PROB_BASE / 2000 - 1; 6628 if (gimple_in_ssa_p (cfun)) 6629 { 6630 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; 6631 for (gphi_iterator gpi = gsi_start_phis (fin_bb); 6632 !gsi_end_p (gpi); gsi_next (&gpi)) 6633 { 6634 gphi *phi = gpi.phi (); 6635 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx), 6636 ep, UNKNOWN_LOCATION); 6637 } 6638 } 6639 gsi = gsi_last_bb (entry_bb); 6640 } 6641 6642 switch (gimple_omp_for_kind (fd->for_stmt)) 6643 { 6644 case GF_OMP_FOR_KIND_FOR: 6645 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 6646 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 6647 break; 6648 case GF_OMP_FOR_KIND_DISTRIBUTE: 6649 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS); 6650 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); 6651 break; 6652 case GF_OMP_FOR_KIND_OACC_LOOP: 6653 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS); 6654 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); 6655 break; 6656 default: 6657 gcc_unreachable (); 6658 } 6659 nthreads = build_call_expr (nthreads, 0); 6660 nthreads = fold_convert (itype, nthreads); 6661 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, 6662 true, GSI_SAME_STMT); 6663 threadid = build_call_expr (threadid, 0); 6664 threadid = fold_convert (itype, threadid); 6665 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, 6666 true, GSI_SAME_STMT); 6667 6668 n1 = fd->loop.n1; 6669 n2 = fd->loop.n2; 6670 step = fd->loop.step; 6671 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6672 { 6673 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 6674 OMP_CLAUSE__LOOPTEMP_); 6675 gcc_assert (innerc); 6676 n1 = OMP_CLAUSE_DECL (innerc); 6677 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6678 OMP_CLAUSE__LOOPTEMP_); 6679 gcc_assert (innerc); 6680 n2 = OMP_CLAUSE_DECL (innerc); 6681 } 6682 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1), 6683 true, NULL_TREE, true, GSI_SAME_STMT); 6684 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2), 6685 true, NULL_TREE, true, GSI_SAME_STMT); 6686 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step), 6687 true, NULL_TREE, true, GSI_SAME_STMT); 6688 6689 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1)); 6690 t = fold_build2 (PLUS_EXPR, itype, step, t); 6691 t = fold_build2 (PLUS_EXPR, itype, t, n2); 6692 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1)); 6693 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR) 6694 t = fold_build2 (TRUNC_DIV_EXPR, itype, 6695 fold_build1 (NEGATE_EXPR, itype, t), 6696 fold_build1 (NEGATE_EXPR, itype, step)); 6697 else 6698 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step); 6699 t = fold_convert (itype, t); 6700 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); 6701 6702 q = create_tmp_reg (itype, "q"); 6703 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads); 6704 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT); 6705 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT); 6706 6707 tt = create_tmp_reg (itype, "tt"); 6708 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads); 6709 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT); 6710 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT); 6711 6712 t = build2 (LT_EXPR, boolean_type_node, threadid, tt); 6713 gcond *cond_stmt = gimple_build_cond_empty (t); 6714 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6715 6716 second_bb = split_block (entry_bb, cond_stmt)->dest; 6717 gsi = gsi_last_bb (second_bb); 6718 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6719 6720 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)), 6721 GSI_SAME_STMT); 6722 gassign *assign_stmt 6723 = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1)); 6724 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 6725 6726 third_bb = split_block (second_bb, assign_stmt)->dest; 6727 gsi = gsi_last_bb (third_bb); 6728 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6729 6730 t = build2 (MULT_EXPR, itype, q, threadid); 6731 t = build2 (PLUS_EXPR, itype, t, tt); 6732 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); 6733 6734 t = fold_build2 (PLUS_EXPR, itype, s0, q); 6735 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); 6736 6737 t = build2 (GE_EXPR, boolean_type_node, s0, e0); 6738 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 6739 6740 /* Remove the GIMPLE_OMP_FOR statement. */ 6741 gsi_remove (&gsi, true); 6742 6743 /* Setup code for sequential iteration goes in SEQ_START_BB. */ 6744 gsi = gsi_start_bb (seq_start_bb); 6745 6746 tree startvar = fd->loop.v; 6747 tree endvar = NULL_TREE; 6748 6749 if (gimple_omp_for_combined_p (fd->for_stmt)) 6750 { 6751 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL 6752 ? gimple_omp_parallel_clauses (inner_stmt) 6753 : gimple_omp_for_clauses (inner_stmt); 6754 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); 6755 gcc_assert (innerc); 6756 startvar = OMP_CLAUSE_DECL (innerc); 6757 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6758 OMP_CLAUSE__LOOPTEMP_); 6759 gcc_assert (innerc); 6760 endvar = OMP_CLAUSE_DECL (innerc); 6761 } 6762 t = fold_convert (itype, s0); 6763 t = fold_build2 (MULT_EXPR, itype, t, step); 6764 if (POINTER_TYPE_P (type)) 6765 t = fold_build_pointer_plus (n1, t); 6766 else 6767 t = fold_build2 (PLUS_EXPR, type, t, n1); 6768 t = fold_convert (TREE_TYPE (startvar), t); 6769 t = force_gimple_operand_gsi (&gsi, t, 6770 DECL_P (startvar) 6771 && TREE_ADDRESSABLE (startvar), 6772 NULL_TREE, false, GSI_CONTINUE_LINKING); 6773 assign_stmt = gimple_build_assign (startvar, t); 6774 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6775 6776 t = fold_convert (itype, e0); 6777 t = fold_build2 (MULT_EXPR, itype, t, step); 6778 if (POINTER_TYPE_P (type)) 6779 t = fold_build_pointer_plus (n1, t); 6780 else 6781 t = fold_build2 (PLUS_EXPR, type, t, n1); 6782 t = fold_convert (TREE_TYPE (startvar), t); 6783 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6784 false, GSI_CONTINUE_LINKING); 6785 if (endvar) 6786 { 6787 assign_stmt = gimple_build_assign (endvar, e); 6788 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6789 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e))) 6790 assign_stmt = gimple_build_assign (fd->loop.v, e); 6791 else 6792 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e); 6793 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6794 } 6795 if (fd->collapse > 1) 6796 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); 6797 6798 if (!broken_loop) 6799 { 6800 /* The code controlling the sequential loop replaces the 6801 GIMPLE_OMP_CONTINUE. */ 6802 gsi = gsi_last_bb (cont_bb); 6803 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); 6804 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE); 6805 vmain = gimple_omp_continue_control_use (cont_stmt); 6806 vback = gimple_omp_continue_control_def (cont_stmt); 6807 6808 if (!gimple_omp_for_combined_p (fd->for_stmt)) 6809 { 6810 if (POINTER_TYPE_P (type)) 6811 t = fold_build_pointer_plus (vmain, step); 6812 else 6813 t = fold_build2 (PLUS_EXPR, type, vmain, step); 6814 t = force_gimple_operand_gsi (&gsi, t, 6815 DECL_P (vback) 6816 && TREE_ADDRESSABLE (vback), 6817 NULL_TREE, true, GSI_SAME_STMT); 6818 assign_stmt = gimple_build_assign (vback, t); 6819 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 6820 6821 t = build2 (fd->loop.cond_code, boolean_type_node, 6822 DECL_P (vback) && TREE_ADDRESSABLE (vback) 6823 ? t : vback, e); 6824 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 6825 } 6826 6827 /* Remove the GIMPLE_OMP_CONTINUE statement. */ 6828 gsi_remove (&gsi, true); 6829 6830 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) 6831 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb); 6832 } 6833 6834 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */ 6835 gsi = gsi_last_bb (exit_bb); 6836 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi))) 6837 { 6838 t = gimple_omp_return_lhs (gsi_stmt (gsi)); 6839 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) 6840 gcc_checking_assert (t == NULL_TREE); 6841 else 6842 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT); 6843 } 6844 gsi_remove (&gsi, true); 6845 6846 /* Connect all the blocks. */ 6847 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE); 6848 ep->probability = REG_BR_PROB_BASE / 4 * 3; 6849 ep = find_edge (entry_bb, second_bb); 6850 ep->flags = EDGE_TRUE_VALUE; 6851 ep->probability = REG_BR_PROB_BASE / 4; 6852 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE; 6853 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE; 6854 6855 if (!broken_loop) 6856 { 6857 ep = find_edge (cont_bb, body_bb); 6858 if (gimple_omp_for_combined_p (fd->for_stmt)) 6859 { 6860 remove_edge (ep); 6861 ep = NULL; 6862 } 6863 else if (fd->collapse > 1) 6864 { 6865 remove_edge (ep); 6866 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); 6867 } 6868 else 6869 ep->flags = EDGE_TRUE_VALUE; 6870 find_edge (cont_bb, fin_bb)->flags 6871 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU; 6872 } 6873 6874 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb); 6875 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb); 6876 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb); 6877 6878 set_immediate_dominator (CDI_DOMINATORS, body_bb, 6879 recompute_dominator (CDI_DOMINATORS, body_bb)); 6880 set_immediate_dominator (CDI_DOMINATORS, fin_bb, 6881 recompute_dominator (CDI_DOMINATORS, fin_bb)); 6882 6883 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt)) 6884 { 6885 struct loop *loop = alloc_loop (); 6886 loop->header = body_bb; 6887 if (collapse_bb == NULL) 6888 loop->latch = cont_bb; 6889 add_loop (loop, body_bb->loop_father); 6890 } 6891 } 6892 6893 6894 /* A subroutine of expand_omp_for. Generate code for a parallel 6895 loop with static schedule and a specified chunk size. Given 6896 parameters: 6897 6898 for (V = N1; V cond N2; V += STEP) BODY; 6899 6900 where COND is "<" or ">", we generate pseudocode 6901 6902 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2; 6903 if (cond is <) 6904 adj = STEP - 1; 6905 else 6906 adj = STEP + 1; 6907 if ((__typeof (V)) -1 > 0 && cond is >) 6908 n = -(adj + N2 - N1) / -STEP; 6909 else 6910 n = (adj + N2 - N1) / STEP; 6911 trip = 0; 6912 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is 6913 here so that V is defined 6914 if the loop is not entered 6915 L0: 6916 s0 = (trip * nthreads + threadid) * CHUNK; 6917 e0 = min(s0 + CHUNK, n); 6918 if (s0 < n) goto L1; else goto L4; 6919 L1: 6920 V = s0 * STEP + N1; 6921 e = e0 * STEP + N1; 6922 L2: 6923 BODY; 6924 V += STEP; 6925 if (V cond e) goto L2; else goto L3; 6926 L3: 6927 trip += 1; 6928 goto L0; 6929 L4: 6930 */ 6931 6932 static void 6933 expand_omp_for_static_chunk (struct omp_region *region, 6934 struct omp_for_data *fd, gimple inner_stmt) 6935 { 6936 tree n, s0, e0, e, t; 6937 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid; 6938 tree type, itype, vmain, vback, vextra; 6939 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb; 6940 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb; 6941 gimple_stmt_iterator gsi; 6942 edge se; 6943 bool broken_loop = region->cont == NULL; 6944 tree *counts = NULL; 6945 tree n1, n2, step; 6946 6947 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt) 6948 != GF_OMP_FOR_KIND_OACC_LOOP) 6949 || !inner_stmt); 6950 6951 itype = type = TREE_TYPE (fd->loop.v); 6952 if (POINTER_TYPE_P (type)) 6953 itype = signed_type_for (type); 6954 6955 entry_bb = region->entry; 6956 se = split_block (entry_bb, last_stmt (entry_bb)); 6957 entry_bb = se->src; 6958 iter_part_bb = se->dest; 6959 cont_bb = region->cont; 6960 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2); 6961 fin_bb = BRANCH_EDGE (iter_part_bb)->dest; 6962 gcc_assert (broken_loop 6963 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest); 6964 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb)); 6965 body_bb = single_succ (seq_start_bb); 6966 if (!broken_loop) 6967 { 6968 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb); 6969 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 6970 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb)); 6971 } 6972 exit_bb = region->exit; 6973 6974 /* Trip and adjustment setup goes in ENTRY_BB. */ 6975 gsi = gsi_last_bb (entry_bb); 6976 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6977 6978 if (fd->collapse > 1) 6979 { 6980 int first_zero_iter = -1; 6981 basic_block l2_dom_bb = NULL; 6982 6983 counts = XALLOCAVEC (tree, fd->collapse); 6984 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 6985 fin_bb, first_zero_iter, 6986 l2_dom_bb); 6987 t = NULL_TREE; 6988 } 6989 else if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6990 t = integer_one_node; 6991 else 6992 t = fold_binary (fd->loop.cond_code, boolean_type_node, 6993 fold_convert (type, fd->loop.n1), 6994 fold_convert (type, fd->loop.n2)); 6995 if (fd->collapse == 1 6996 && TYPE_UNSIGNED (type) 6997 && (t == NULL_TREE || !integer_onep (t))) 6998 { 6999 n1 = fold_convert (type, unshare_expr (fd->loop.n1)); 7000 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE, 7001 true, GSI_SAME_STMT); 7002 n2 = fold_convert (type, unshare_expr (fd->loop.n2)); 7003 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE, 7004 true, GSI_SAME_STMT); 7005 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2, 7006 NULL_TREE, NULL_TREE); 7007 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 7008 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 7009 expand_omp_regimplify_p, NULL, NULL) 7010 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 7011 expand_omp_regimplify_p, NULL, NULL)) 7012 { 7013 gsi = gsi_for_stmt (cond_stmt); 7014 gimple_regimplify_operands (cond_stmt, &gsi); 7015 } 7016 se = split_block (entry_bb, cond_stmt); 7017 se->flags = EDGE_TRUE_VALUE; 7018 entry_bb = se->dest; 7019 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); 7020 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE); 7021 se->probability = REG_BR_PROB_BASE / 2000 - 1; 7022 if (gimple_in_ssa_p (cfun)) 7023 { 7024 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; 7025 for (gphi_iterator gpi = gsi_start_phis (fin_bb); 7026 !gsi_end_p (gpi); gsi_next (&gpi)) 7027 { 7028 gphi *phi = gpi.phi (); 7029 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx), 7030 se, UNKNOWN_LOCATION); 7031 } 7032 } 7033 gsi = gsi_last_bb (entry_bb); 7034 } 7035 7036 switch (gimple_omp_for_kind (fd->for_stmt)) 7037 { 7038 case GF_OMP_FOR_KIND_FOR: 7039 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 7040 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 7041 break; 7042 case GF_OMP_FOR_KIND_DISTRIBUTE: 7043 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS); 7044 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); 7045 break; 7046 case GF_OMP_FOR_KIND_OACC_LOOP: 7047 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS); 7048 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); 7049 break; 7050 default: 7051 gcc_unreachable (); 7052 } 7053 nthreads = build_call_expr (nthreads, 0); 7054 nthreads = fold_convert (itype, nthreads); 7055 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, 7056 true, GSI_SAME_STMT); 7057 threadid = build_call_expr (threadid, 0); 7058 threadid = fold_convert (itype, threadid); 7059 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, 7060 true, GSI_SAME_STMT); 7061 7062 n1 = fd->loop.n1; 7063 n2 = fd->loop.n2; 7064 step = fd->loop.step; 7065 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 7066 { 7067 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7068 OMP_CLAUSE__LOOPTEMP_); 7069 gcc_assert (innerc); 7070 n1 = OMP_CLAUSE_DECL (innerc); 7071 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 7072 OMP_CLAUSE__LOOPTEMP_); 7073 gcc_assert (innerc); 7074 n2 = OMP_CLAUSE_DECL (innerc); 7075 } 7076 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1), 7077 true, NULL_TREE, true, GSI_SAME_STMT); 7078 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2), 7079 true, NULL_TREE, true, GSI_SAME_STMT); 7080 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step), 7081 true, NULL_TREE, true, GSI_SAME_STMT); 7082 fd->chunk_size 7083 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size), 7084 true, NULL_TREE, true, GSI_SAME_STMT); 7085 7086 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1)); 7087 t = fold_build2 (PLUS_EXPR, itype, step, t); 7088 t = fold_build2 (PLUS_EXPR, itype, t, n2); 7089 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1)); 7090 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR) 7091 t = fold_build2 (TRUNC_DIV_EXPR, itype, 7092 fold_build1 (NEGATE_EXPR, itype, t), 7093 fold_build1 (NEGATE_EXPR, itype, step)); 7094 else 7095 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step); 7096 t = fold_convert (itype, t); 7097 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7098 true, GSI_SAME_STMT); 7099 7100 trip_var = create_tmp_reg (itype, ".trip"); 7101 if (gimple_in_ssa_p (cfun)) 7102 { 7103 trip_init = make_ssa_name (trip_var); 7104 trip_main = make_ssa_name (trip_var); 7105 trip_back = make_ssa_name (trip_var); 7106 } 7107 else 7108 { 7109 trip_init = trip_var; 7110 trip_main = trip_var; 7111 trip_back = trip_var; 7112 } 7113 7114 gassign *assign_stmt 7115 = gimple_build_assign (trip_init, build_int_cst (itype, 0)); 7116 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 7117 7118 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size); 7119 t = fold_build2 (MULT_EXPR, itype, t, step); 7120 if (POINTER_TYPE_P (type)) 7121 t = fold_build_pointer_plus (n1, t); 7122 else 7123 t = fold_build2 (PLUS_EXPR, type, t, n1); 7124 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7125 true, GSI_SAME_STMT); 7126 7127 /* Remove the GIMPLE_OMP_FOR. */ 7128 gsi_remove (&gsi, true); 7129 7130 /* Iteration space partitioning goes in ITER_PART_BB. */ 7131 gsi = gsi_last_bb (iter_part_bb); 7132 7133 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads); 7134 t = fold_build2 (PLUS_EXPR, itype, t, threadid); 7135 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size); 7136 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7137 false, GSI_CONTINUE_LINKING); 7138 7139 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size); 7140 t = fold_build2 (MIN_EXPR, itype, t, n); 7141 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7142 false, GSI_CONTINUE_LINKING); 7143 7144 t = build2 (LT_EXPR, boolean_type_node, s0, n); 7145 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING); 7146 7147 /* Setup code for sequential iteration goes in SEQ_START_BB. */ 7148 gsi = gsi_start_bb (seq_start_bb); 7149 7150 tree startvar = fd->loop.v; 7151 tree endvar = NULL_TREE; 7152 7153 if (gimple_omp_for_combined_p (fd->for_stmt)) 7154 { 7155 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL 7156 ? gimple_omp_parallel_clauses (inner_stmt) 7157 : gimple_omp_for_clauses (inner_stmt); 7158 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); 7159 gcc_assert (innerc); 7160 startvar = OMP_CLAUSE_DECL (innerc); 7161 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 7162 OMP_CLAUSE__LOOPTEMP_); 7163 gcc_assert (innerc); 7164 endvar = OMP_CLAUSE_DECL (innerc); 7165 } 7166 7167 t = fold_convert (itype, s0); 7168 t = fold_build2 (MULT_EXPR, itype, t, step); 7169 if (POINTER_TYPE_P (type)) 7170 t = fold_build_pointer_plus (n1, t); 7171 else 7172 t = fold_build2 (PLUS_EXPR, type, t, n1); 7173 t = fold_convert (TREE_TYPE (startvar), t); 7174 t = force_gimple_operand_gsi (&gsi, t, 7175 DECL_P (startvar) 7176 && TREE_ADDRESSABLE (startvar), 7177 NULL_TREE, false, GSI_CONTINUE_LINKING); 7178 assign_stmt = gimple_build_assign (startvar, t); 7179 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7180 7181 t = fold_convert (itype, e0); 7182 t = fold_build2 (MULT_EXPR, itype, t, step); 7183 if (POINTER_TYPE_P (type)) 7184 t = fold_build_pointer_plus (n1, t); 7185 else 7186 t = fold_build2 (PLUS_EXPR, type, t, n1); 7187 t = fold_convert (TREE_TYPE (startvar), t); 7188 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7189 false, GSI_CONTINUE_LINKING); 7190 if (endvar) 7191 { 7192 assign_stmt = gimple_build_assign (endvar, e); 7193 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7194 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e))) 7195 assign_stmt = gimple_build_assign (fd->loop.v, e); 7196 else 7197 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e); 7198 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7199 } 7200 if (fd->collapse > 1) 7201 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); 7202 7203 if (!broken_loop) 7204 { 7205 /* The code controlling the sequential loop goes in CONT_BB, 7206 replacing the GIMPLE_OMP_CONTINUE. */ 7207 gsi = gsi_last_bb (cont_bb); 7208 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); 7209 vmain = gimple_omp_continue_control_use (cont_stmt); 7210 vback = gimple_omp_continue_control_def (cont_stmt); 7211 7212 if (!gimple_omp_for_combined_p (fd->for_stmt)) 7213 { 7214 if (POINTER_TYPE_P (type)) 7215 t = fold_build_pointer_plus (vmain, step); 7216 else 7217 t = fold_build2 (PLUS_EXPR, type, vmain, step); 7218 if (DECL_P (vback) && TREE_ADDRESSABLE (vback)) 7219 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7220 true, GSI_SAME_STMT); 7221 assign_stmt = gimple_build_assign (vback, t); 7222 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 7223 7224 t = build2 (fd->loop.cond_code, boolean_type_node, 7225 DECL_P (vback) && TREE_ADDRESSABLE (vback) 7226 ? t : vback, e); 7227 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 7228 } 7229 7230 /* Remove GIMPLE_OMP_CONTINUE. */ 7231 gsi_remove (&gsi, true); 7232 7233 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) 7234 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb); 7235 7236 /* Trip update code goes into TRIP_UPDATE_BB. */ 7237 gsi = gsi_start_bb (trip_update_bb); 7238 7239 t = build_int_cst (itype, 1); 7240 t = build2 (PLUS_EXPR, itype, trip_main, t); 7241 assign_stmt = gimple_build_assign (trip_back, t); 7242 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7243 } 7244 7245 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */ 7246 gsi = gsi_last_bb (exit_bb); 7247 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi))) 7248 { 7249 t = gimple_omp_return_lhs (gsi_stmt (gsi)); 7250 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) 7251 gcc_checking_assert (t == NULL_TREE); 7252 else 7253 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT); 7254 } 7255 gsi_remove (&gsi, true); 7256 7257 /* Connect the new blocks. */ 7258 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE; 7259 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE; 7260 7261 if (!broken_loop) 7262 { 7263 se = find_edge (cont_bb, body_bb); 7264 if (gimple_omp_for_combined_p (fd->for_stmt)) 7265 { 7266 remove_edge (se); 7267 se = NULL; 7268 } 7269 else if (fd->collapse > 1) 7270 { 7271 remove_edge (se); 7272 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); 7273 } 7274 else 7275 se->flags = EDGE_TRUE_VALUE; 7276 find_edge (cont_bb, trip_update_bb)->flags 7277 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU; 7278 7279 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb); 7280 } 7281 7282 if (gimple_in_ssa_p (cfun)) 7283 { 7284 gphi_iterator psi; 7285 gphi *phi; 7286 edge re, ene; 7287 edge_var_map *vm; 7288 size_t i; 7289 7290 gcc_assert (fd->collapse == 1 && !broken_loop); 7291 7292 /* When we redirect the edge from trip_update_bb to iter_part_bb, we 7293 remove arguments of the phi nodes in fin_bb. We need to create 7294 appropriate phi nodes in iter_part_bb instead. */ 7295 se = single_pred_edge (fin_bb); 7296 re = single_succ_edge (trip_update_bb); 7297 vec<edge_var_map> *head = redirect_edge_var_map_vector (re); 7298 ene = single_succ_edge (entry_bb); 7299 7300 psi = gsi_start_phis (fin_bb); 7301 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm); 7302 gsi_next (&psi), ++i) 7303 { 7304 gphi *nphi; 7305 source_location locus; 7306 7307 phi = psi.phi (); 7308 t = gimple_phi_result (phi); 7309 gcc_assert (t == redirect_edge_var_map_result (vm)); 7310 nphi = create_phi_node (t, iter_part_bb); 7311 7312 t = PHI_ARG_DEF_FROM_EDGE (phi, se); 7313 locus = gimple_phi_arg_location_from_edge (phi, se); 7314 7315 /* A special case -- fd->loop.v is not yet computed in 7316 iter_part_bb, we need to use vextra instead. */ 7317 if (t == fd->loop.v) 7318 t = vextra; 7319 add_phi_arg (nphi, t, ene, locus); 7320 locus = redirect_edge_var_map_location (vm); 7321 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus); 7322 } 7323 gcc_assert (gsi_end_p (psi) && i == head->length ()); 7324 redirect_edge_var_map_clear (re); 7325 while (1) 7326 { 7327 psi = gsi_start_phis (fin_bb); 7328 if (gsi_end_p (psi)) 7329 break; 7330 remove_phi_node (&psi, false); 7331 } 7332 7333 /* Make phi node for trip. */ 7334 phi = create_phi_node (trip_main, iter_part_bb); 7335 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb), 7336 UNKNOWN_LOCATION); 7337 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb), 7338 UNKNOWN_LOCATION); 7339 } 7340 7341 if (!broken_loop) 7342 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb); 7343 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb, 7344 recompute_dominator (CDI_DOMINATORS, iter_part_bb)); 7345 set_immediate_dominator (CDI_DOMINATORS, fin_bb, 7346 recompute_dominator (CDI_DOMINATORS, fin_bb)); 7347 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, 7348 recompute_dominator (CDI_DOMINATORS, seq_start_bb)); 7349 set_immediate_dominator (CDI_DOMINATORS, body_bb, 7350 recompute_dominator (CDI_DOMINATORS, body_bb)); 7351 7352 if (!broken_loop) 7353 { 7354 struct loop *trip_loop = alloc_loop (); 7355 trip_loop->header = iter_part_bb; 7356 trip_loop->latch = trip_update_bb; 7357 add_loop (trip_loop, iter_part_bb->loop_father); 7358 7359 if (!gimple_omp_for_combined_p (fd->for_stmt)) 7360 { 7361 struct loop *loop = alloc_loop (); 7362 loop->header = body_bb; 7363 if (collapse_bb == NULL) 7364 loop->latch = cont_bb; 7365 add_loop (loop, trip_loop); 7366 } 7367 } 7368 } 7369 7370 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop. 7371 Given parameters: 7372 for (V = N1; V cond N2; V += STEP) BODY; 7373 7374 where COND is "<" or ">" or "!=", we generate pseudocode 7375 7376 for (ind_var = low; ind_var < high; ind_var++) 7377 { 7378 V = n1 + (ind_var * STEP) 7379 7380 <BODY> 7381 } 7382 7383 In the above pseudocode, low and high are function parameters of the 7384 child function. In the function below, we are inserting a temp. 7385 variable that will be making a call to two OMP functions that will not be 7386 found in the body of _Cilk_for (since OMP_FOR cannot be mixed 7387 with _Cilk_for). These functions are replaced with low and high 7388 by the function that handles taskreg. */ 7389 7390 7391 static void 7392 expand_cilk_for (struct omp_region *region, struct omp_for_data *fd) 7393 { 7394 bool broken_loop = region->cont == NULL; 7395 basic_block entry_bb = region->entry; 7396 basic_block cont_bb = region->cont; 7397 7398 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 7399 gcc_assert (broken_loop 7400 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); 7401 basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest; 7402 basic_block l1_bb, l2_bb; 7403 7404 if (!broken_loop) 7405 { 7406 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb); 7407 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 7408 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest; 7409 l2_bb = BRANCH_EDGE (entry_bb)->dest; 7410 } 7411 else 7412 { 7413 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL; 7414 l1_bb = split_edge (BRANCH_EDGE (entry_bb)); 7415 l2_bb = single_succ (l1_bb); 7416 } 7417 basic_block exit_bb = region->exit; 7418 basic_block l2_dom_bb = NULL; 7419 7420 gimple_stmt_iterator gsi = gsi_last_bb (entry_bb); 7421 7422 /* Below statements until the "tree high_val = ..." are pseudo statements 7423 used to pass information to be used by expand_omp_taskreg. 7424 low_val and high_val will be replaced by the __low and __high 7425 parameter from the child function. 7426 7427 The call_exprs part is a place-holder, it is mainly used 7428 to distinctly identify to the top-level part that this is 7429 where we should put low and high (reasoning given in header 7430 comment). */ 7431 7432 tree child_fndecl 7433 = gimple_omp_parallel_child_fn ( 7434 as_a <gomp_parallel *> (last_stmt (region->outer->entry))); 7435 tree t, low_val = NULL_TREE, high_val = NULL_TREE; 7436 for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t)) 7437 { 7438 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high")) 7439 high_val = t; 7440 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low")) 7441 low_val = t; 7442 } 7443 gcc_assert (low_val && high_val); 7444 7445 tree type = TREE_TYPE (low_val); 7446 tree ind_var = create_tmp_reg (type, "__cilk_ind_var"); 7447 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 7448 7449 /* Not needed in SSA form right now. */ 7450 gcc_assert (!gimple_in_ssa_p (cfun)); 7451 if (l2_dom_bb == NULL) 7452 l2_dom_bb = l1_bb; 7453 7454 tree n1 = low_val; 7455 tree n2 = high_val; 7456 7457 gimple stmt = gimple_build_assign (ind_var, n1); 7458 7459 /* Replace the GIMPLE_OMP_FOR statement. */ 7460 gsi_replace (&gsi, stmt, true); 7461 7462 if (!broken_loop) 7463 { 7464 /* Code to control the increment goes in the CONT_BB. */ 7465 gsi = gsi_last_bb (cont_bb); 7466 stmt = gsi_stmt (gsi); 7467 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE); 7468 stmt = gimple_build_assign (ind_var, PLUS_EXPR, ind_var, 7469 build_one_cst (type)); 7470 7471 /* Replace GIMPLE_OMP_CONTINUE. */ 7472 gsi_replace (&gsi, stmt, true); 7473 } 7474 7475 /* Emit the condition in L1_BB. */ 7476 gsi = gsi_after_labels (l1_bb); 7477 t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step), 7478 fold_convert (TREE_TYPE (fd->loop.step), ind_var), 7479 fd->loop.step); 7480 if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1))) 7481 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1), 7482 fd->loop.n1, fold_convert (sizetype, t)); 7483 else 7484 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1), 7485 fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t)); 7486 t = fold_convert (TREE_TYPE (fd->loop.v), t); 7487 expand_omp_build_assign (&gsi, fd->loop.v, t); 7488 7489 /* The condition is always '<' since the runtime will fill in the low 7490 and high values. */ 7491 stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE); 7492 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 7493 7494 /* Remove GIMPLE_OMP_RETURN. */ 7495 gsi = gsi_last_bb (exit_bb); 7496 gsi_remove (&gsi, true); 7497 7498 /* Connect the new blocks. */ 7499 remove_edge (FALLTHRU_EDGE (entry_bb)); 7500 7501 edge e, ne; 7502 if (!broken_loop) 7503 { 7504 remove_edge (BRANCH_EDGE (entry_bb)); 7505 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU); 7506 7507 e = BRANCH_EDGE (l1_bb); 7508 ne = FALLTHRU_EDGE (l1_bb); 7509 e->flags = EDGE_TRUE_VALUE; 7510 } 7511 else 7512 { 7513 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 7514 7515 ne = single_succ_edge (l1_bb); 7516 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE); 7517 7518 } 7519 ne->flags = EDGE_FALSE_VALUE; 7520 e->probability = REG_BR_PROB_BASE * 7 / 8; 7521 ne->probability = REG_BR_PROB_BASE / 8; 7522 7523 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); 7524 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); 7525 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); 7526 7527 if (!broken_loop) 7528 { 7529 struct loop *loop = alloc_loop (); 7530 loop->header = l1_bb; 7531 loop->latch = cont_bb; 7532 add_loop (loop, l1_bb->loop_father); 7533 loop->safelen = INT_MAX; 7534 } 7535 7536 /* Pick the correct library function based on the precision of the 7537 induction variable type. */ 7538 tree lib_fun = NULL_TREE; 7539 if (TYPE_PRECISION (type) == 32) 7540 lib_fun = cilk_for_32_fndecl; 7541 else if (TYPE_PRECISION (type) == 64) 7542 lib_fun = cilk_for_64_fndecl; 7543 else 7544 gcc_unreachable (); 7545 7546 gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR); 7547 7548 /* WS_ARGS contains the library function flavor to call: 7549 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the 7550 user-defined grain value. If the user does not define one, then zero 7551 is passed in by the parser. */ 7552 vec_alloc (region->ws_args, 2); 7553 region->ws_args->quick_push (lib_fun); 7554 region->ws_args->quick_push (fd->chunk_size); 7555 } 7556 7557 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing 7558 loop. Given parameters: 7559 7560 for (V = N1; V cond N2; V += STEP) BODY; 7561 7562 where COND is "<" or ">", we generate pseudocode 7563 7564 V = N1; 7565 goto L1; 7566 L0: 7567 BODY; 7568 V += STEP; 7569 L1: 7570 if (V cond N2) goto L0; else goto L2; 7571 L2: 7572 7573 For collapsed loops, given parameters: 7574 collapse(3) 7575 for (V1 = N11; V1 cond1 N12; V1 += STEP1) 7576 for (V2 = N21; V2 cond2 N22; V2 += STEP2) 7577 for (V3 = N31; V3 cond3 N32; V3 += STEP3) 7578 BODY; 7579 7580 we generate pseudocode 7581 7582 if (cond3 is <) 7583 adj = STEP3 - 1; 7584 else 7585 adj = STEP3 + 1; 7586 count3 = (adj + N32 - N31) / STEP3; 7587 if (cond2 is <) 7588 adj = STEP2 - 1; 7589 else 7590 adj = STEP2 + 1; 7591 count2 = (adj + N22 - N21) / STEP2; 7592 if (cond1 is <) 7593 adj = STEP1 - 1; 7594 else 7595 adj = STEP1 + 1; 7596 count1 = (adj + N12 - N11) / STEP1; 7597 count = count1 * count2 * count3; 7598 V = 0; 7599 V1 = N11; 7600 V2 = N21; 7601 V3 = N31; 7602 goto L1; 7603 L0: 7604 BODY; 7605 V += 1; 7606 V3 += STEP3; 7607 V2 += (V3 cond3 N32) ? 0 : STEP2; 7608 V3 = (V3 cond3 N32) ? V3 : N31; 7609 V1 += (V2 cond2 N22) ? 0 : STEP1; 7610 V2 = (V2 cond2 N22) ? V2 : N21; 7611 L1: 7612 if (V < count) goto L0; else goto L2; 7613 L2: 7614 7615 */ 7616 7617 static void 7618 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd) 7619 { 7620 tree type, t; 7621 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb; 7622 gimple_stmt_iterator gsi; 7623 gimple stmt; 7624 gcond *cond_stmt; 7625 bool broken_loop = region->cont == NULL; 7626 edge e, ne; 7627 tree *counts = NULL; 7628 int i; 7629 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7630 OMP_CLAUSE_SAFELEN); 7631 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7632 OMP_CLAUSE__SIMDUID_); 7633 tree n1, n2; 7634 7635 type = TREE_TYPE (fd->loop.v); 7636 entry_bb = region->entry; 7637 cont_bb = region->cont; 7638 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 7639 gcc_assert (broken_loop 7640 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); 7641 l0_bb = FALLTHRU_EDGE (entry_bb)->dest; 7642 if (!broken_loop) 7643 { 7644 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb); 7645 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 7646 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest; 7647 l2_bb = BRANCH_EDGE (entry_bb)->dest; 7648 } 7649 else 7650 { 7651 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL; 7652 l1_bb = split_edge (BRANCH_EDGE (entry_bb)); 7653 l2_bb = single_succ (l1_bb); 7654 } 7655 exit_bb = region->exit; 7656 l2_dom_bb = NULL; 7657 7658 gsi = gsi_last_bb (entry_bb); 7659 7660 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 7661 /* Not needed in SSA form right now. */ 7662 gcc_assert (!gimple_in_ssa_p (cfun)); 7663 if (fd->collapse > 1) 7664 { 7665 int first_zero_iter = -1; 7666 basic_block zero_iter_bb = l2_bb; 7667 7668 counts = XALLOCAVEC (tree, fd->collapse); 7669 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 7670 zero_iter_bb, first_zero_iter, 7671 l2_dom_bb); 7672 } 7673 if (l2_dom_bb == NULL) 7674 l2_dom_bb = l1_bb; 7675 7676 n1 = fd->loop.n1; 7677 n2 = fd->loop.n2; 7678 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 7679 { 7680 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7681 OMP_CLAUSE__LOOPTEMP_); 7682 gcc_assert (innerc); 7683 n1 = OMP_CLAUSE_DECL (innerc); 7684 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 7685 OMP_CLAUSE__LOOPTEMP_); 7686 gcc_assert (innerc); 7687 n2 = OMP_CLAUSE_DECL (innerc); 7688 expand_omp_build_assign (&gsi, fd->loop.v, 7689 fold_convert (type, n1)); 7690 if (fd->collapse > 1) 7691 { 7692 gsi_prev (&gsi); 7693 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1); 7694 gsi_next (&gsi); 7695 } 7696 } 7697 else 7698 { 7699 expand_omp_build_assign (&gsi, fd->loop.v, 7700 fold_convert (type, fd->loop.n1)); 7701 if (fd->collapse > 1) 7702 for (i = 0; i < fd->collapse; i++) 7703 { 7704 tree itype = TREE_TYPE (fd->loops[i].v); 7705 if (POINTER_TYPE_P (itype)) 7706 itype = signed_type_for (itype); 7707 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1); 7708 expand_omp_build_assign (&gsi, fd->loops[i].v, t); 7709 } 7710 } 7711 7712 /* Remove the GIMPLE_OMP_FOR statement. */ 7713 gsi_remove (&gsi, true); 7714 7715 if (!broken_loop) 7716 { 7717 /* Code to control the increment goes in the CONT_BB. */ 7718 gsi = gsi_last_bb (cont_bb); 7719 stmt = gsi_stmt (gsi); 7720 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE); 7721 7722 if (POINTER_TYPE_P (type)) 7723 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step); 7724 else 7725 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step); 7726 expand_omp_build_assign (&gsi, fd->loop.v, t); 7727 7728 if (fd->collapse > 1) 7729 { 7730 i = fd->collapse - 1; 7731 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))) 7732 { 7733 t = fold_convert (sizetype, fd->loops[i].step); 7734 t = fold_build_pointer_plus (fd->loops[i].v, t); 7735 } 7736 else 7737 { 7738 t = fold_convert (TREE_TYPE (fd->loops[i].v), 7739 fd->loops[i].step); 7740 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v), 7741 fd->loops[i].v, t); 7742 } 7743 expand_omp_build_assign (&gsi, fd->loops[i].v, t); 7744 7745 for (i = fd->collapse - 1; i > 0; i--) 7746 { 7747 tree itype = TREE_TYPE (fd->loops[i].v); 7748 tree itype2 = TREE_TYPE (fd->loops[i - 1].v); 7749 if (POINTER_TYPE_P (itype2)) 7750 itype2 = signed_type_for (itype2); 7751 t = build3 (COND_EXPR, itype2, 7752 build2 (fd->loops[i].cond_code, boolean_type_node, 7753 fd->loops[i].v, 7754 fold_convert (itype, fd->loops[i].n2)), 7755 build_int_cst (itype2, 0), 7756 fold_convert (itype2, fd->loops[i - 1].step)); 7757 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v))) 7758 t = fold_build_pointer_plus (fd->loops[i - 1].v, t); 7759 else 7760 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t); 7761 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t); 7762 7763 t = build3 (COND_EXPR, itype, 7764 build2 (fd->loops[i].cond_code, boolean_type_node, 7765 fd->loops[i].v, 7766 fold_convert (itype, fd->loops[i].n2)), 7767 fd->loops[i].v, 7768 fold_convert (itype, fd->loops[i].n1)); 7769 expand_omp_build_assign (&gsi, fd->loops[i].v, t); 7770 } 7771 } 7772 7773 /* Remove GIMPLE_OMP_CONTINUE. */ 7774 gsi_remove (&gsi, true); 7775 } 7776 7777 /* Emit the condition in L1_BB. */ 7778 gsi = gsi_start_bb (l1_bb); 7779 7780 t = fold_convert (type, n2); 7781 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7782 false, GSI_CONTINUE_LINKING); 7783 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t); 7784 cond_stmt = gimple_build_cond_empty (t); 7785 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING); 7786 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p, 7787 NULL, NULL) 7788 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p, 7789 NULL, NULL)) 7790 { 7791 gsi = gsi_for_stmt (cond_stmt); 7792 gimple_regimplify_operands (cond_stmt, &gsi); 7793 } 7794 7795 /* Remove GIMPLE_OMP_RETURN. */ 7796 gsi = gsi_last_bb (exit_bb); 7797 gsi_remove (&gsi, true); 7798 7799 /* Connect the new blocks. */ 7800 remove_edge (FALLTHRU_EDGE (entry_bb)); 7801 7802 if (!broken_loop) 7803 { 7804 remove_edge (BRANCH_EDGE (entry_bb)); 7805 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU); 7806 7807 e = BRANCH_EDGE (l1_bb); 7808 ne = FALLTHRU_EDGE (l1_bb); 7809 e->flags = EDGE_TRUE_VALUE; 7810 } 7811 else 7812 { 7813 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 7814 7815 ne = single_succ_edge (l1_bb); 7816 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE); 7817 7818 } 7819 ne->flags = EDGE_FALSE_VALUE; 7820 e->probability = REG_BR_PROB_BASE * 7 / 8; 7821 ne->probability = REG_BR_PROB_BASE / 8; 7822 7823 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); 7824 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); 7825 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); 7826 7827 if (!broken_loop) 7828 { 7829 struct loop *loop = alloc_loop (); 7830 loop->header = l1_bb; 7831 loop->latch = cont_bb; 7832 add_loop (loop, l1_bb->loop_father); 7833 if (safelen == NULL_TREE) 7834 loop->safelen = INT_MAX; 7835 else 7836 { 7837 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen); 7838 if (TREE_CODE (safelen) != INTEGER_CST) 7839 loop->safelen = 0; 7840 else if (!tree_fits_uhwi_p (safelen) 7841 || tree_to_uhwi (safelen) > INT_MAX) 7842 loop->safelen = INT_MAX; 7843 else 7844 loop->safelen = tree_to_uhwi (safelen); 7845 if (loop->safelen == 1) 7846 loop->safelen = 0; 7847 } 7848 if (simduid) 7849 { 7850 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); 7851 cfun->has_simduid_loops = true; 7852 } 7853 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize 7854 the loop. */ 7855 if ((flag_tree_loop_vectorize 7856 || (!global_options_set.x_flag_tree_loop_vectorize 7857 && !global_options_set.x_flag_tree_vectorize)) 7858 && flag_tree_loop_optimize 7859 && loop->safelen > 1) 7860 { 7861 loop->force_vectorize = true; 7862 cfun->has_force_vectorize_loops = true; 7863 } 7864 } 7865 else if (simduid) 7866 cfun->has_simduid_loops = true; 7867 } 7868 7869 7870 /* Expand the OMP loop defined by REGION. */ 7871 7872 static void 7873 expand_omp_for (struct omp_region *region, gimple inner_stmt) 7874 { 7875 struct omp_for_data fd; 7876 struct omp_for_data_loop *loops; 7877 7878 loops 7879 = (struct omp_for_data_loop *) 7880 alloca (gimple_omp_for_collapse (last_stmt (region->entry)) 7881 * sizeof (struct omp_for_data_loop)); 7882 extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)), 7883 &fd, loops); 7884 region->sched_kind = fd.sched_kind; 7885 7886 gcc_assert (EDGE_COUNT (region->entry->succs) == 2); 7887 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL; 7888 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL; 7889 if (region->cont) 7890 { 7891 gcc_assert (EDGE_COUNT (region->cont->succs) == 2); 7892 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL; 7893 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL; 7894 } 7895 else 7896 /* If there isn't a continue then this is a degerate case where 7897 the introduction of abnormal edges during lowering will prevent 7898 original loops from being detected. Fix that up. */ 7899 loops_state_set (LOOPS_NEED_FIXUP); 7900 7901 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD) 7902 expand_omp_simd (region, &fd); 7903 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR) 7904 expand_cilk_for (region, &fd); 7905 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC 7906 && !fd.have_ordered) 7907 { 7908 if (fd.chunk_size == NULL) 7909 expand_omp_for_static_nochunk (region, &fd, inner_stmt); 7910 else 7911 expand_omp_for_static_chunk (region, &fd, inner_stmt); 7912 } 7913 else 7914 { 7915 int fn_index, start_ix, next_ix; 7916 7917 gcc_assert (gimple_omp_for_kind (fd.for_stmt) 7918 == GF_OMP_FOR_KIND_FOR); 7919 if (fd.chunk_size == NULL 7920 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC) 7921 fd.chunk_size = integer_zero_node; 7922 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO); 7923 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME) 7924 ? 3 : fd.sched_kind; 7925 fn_index += fd.have_ordered * 4; 7926 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index; 7927 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index; 7928 if (fd.iter_type == long_long_unsigned_type_node) 7929 { 7930 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START 7931 - (int)BUILT_IN_GOMP_LOOP_STATIC_START); 7932 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT 7933 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT); 7934 } 7935 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix, 7936 (enum built_in_function) next_ix, inner_stmt); 7937 } 7938 7939 if (gimple_in_ssa_p (cfun)) 7940 update_ssa (TODO_update_ssa_only_virtuals); 7941 } 7942 7943 7944 /* Expand code for an OpenMP sections directive. In pseudo code, we generate 7945 7946 v = GOMP_sections_start (n); 7947 L0: 7948 switch (v) 7949 { 7950 case 0: 7951 goto L2; 7952 case 1: 7953 section 1; 7954 goto L1; 7955 case 2: 7956 ... 7957 case n: 7958 ... 7959 default: 7960 abort (); 7961 } 7962 L1: 7963 v = GOMP_sections_next (); 7964 goto L0; 7965 L2: 7966 reduction; 7967 7968 If this is a combined parallel sections, replace the call to 7969 GOMP_sections_start with call to GOMP_sections_next. */ 7970 7971 static void 7972 expand_omp_sections (struct omp_region *region) 7973 { 7974 tree t, u, vin = NULL, vmain, vnext, l2; 7975 unsigned len; 7976 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb; 7977 gimple_stmt_iterator si, switch_si; 7978 gomp_sections *sections_stmt; 7979 gimple stmt; 7980 gomp_continue *cont; 7981 edge_iterator ei; 7982 edge e; 7983 struct omp_region *inner; 7984 unsigned i, casei; 7985 bool exit_reachable = region->cont != NULL; 7986 7987 gcc_assert (region->exit != NULL); 7988 entry_bb = region->entry; 7989 l0_bb = single_succ (entry_bb); 7990 l1_bb = region->cont; 7991 l2_bb = region->exit; 7992 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb) 7993 l2 = gimple_block_label (l2_bb); 7994 else 7995 { 7996 /* This can happen if there are reductions. */ 7997 len = EDGE_COUNT (l0_bb->succs); 7998 gcc_assert (len > 0); 7999 e = EDGE_SUCC (l0_bb, len - 1); 8000 si = gsi_last_bb (e->dest); 8001 l2 = NULL_TREE; 8002 if (gsi_end_p (si) 8003 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION) 8004 l2 = gimple_block_label (e->dest); 8005 else 8006 FOR_EACH_EDGE (e, ei, l0_bb->succs) 8007 { 8008 si = gsi_last_bb (e->dest); 8009 if (gsi_end_p (si) 8010 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION) 8011 { 8012 l2 = gimple_block_label (e->dest); 8013 break; 8014 } 8015 } 8016 } 8017 if (exit_reachable) 8018 default_bb = create_empty_bb (l1_bb->prev_bb); 8019 else 8020 default_bb = create_empty_bb (l0_bb); 8021 8022 /* We will build a switch() with enough cases for all the 8023 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work 8024 and a default case to abort if something goes wrong. */ 8025 len = EDGE_COUNT (l0_bb->succs); 8026 8027 /* Use vec::quick_push on label_vec throughout, since we know the size 8028 in advance. */ 8029 auto_vec<tree> label_vec (len); 8030 8031 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the 8032 GIMPLE_OMP_SECTIONS statement. */ 8033 si = gsi_last_bb (entry_bb); 8034 sections_stmt = as_a <gomp_sections *> (gsi_stmt (si)); 8035 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS); 8036 vin = gimple_omp_sections_control (sections_stmt); 8037 if (!is_combined_parallel (region)) 8038 { 8039 /* If we are not inside a combined parallel+sections region, 8040 call GOMP_sections_start. */ 8041 t = build_int_cst (unsigned_type_node, len - 1); 8042 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START); 8043 stmt = gimple_build_call (u, 1, t); 8044 } 8045 else 8046 { 8047 /* Otherwise, call GOMP_sections_next. */ 8048 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT); 8049 stmt = gimple_build_call (u, 0); 8050 } 8051 gimple_call_set_lhs (stmt, vin); 8052 gsi_insert_after (&si, stmt, GSI_SAME_STMT); 8053 gsi_remove (&si, true); 8054 8055 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in 8056 L0_BB. */ 8057 switch_si = gsi_last_bb (l0_bb); 8058 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH); 8059 if (exit_reachable) 8060 { 8061 cont = as_a <gomp_continue *> (last_stmt (l1_bb)); 8062 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE); 8063 vmain = gimple_omp_continue_control_use (cont); 8064 vnext = gimple_omp_continue_control_def (cont); 8065 } 8066 else 8067 { 8068 vmain = vin; 8069 vnext = NULL_TREE; 8070 } 8071 8072 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2); 8073 label_vec.quick_push (t); 8074 i = 1; 8075 8076 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */ 8077 for (inner = region->inner, casei = 1; 8078 inner; 8079 inner = inner->next, i++, casei++) 8080 { 8081 basic_block s_entry_bb, s_exit_bb; 8082 8083 /* Skip optional reduction region. */ 8084 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD) 8085 { 8086 --i; 8087 --casei; 8088 continue; 8089 } 8090 8091 s_entry_bb = inner->entry; 8092 s_exit_bb = inner->exit; 8093 8094 t = gimple_block_label (s_entry_bb); 8095 u = build_int_cst (unsigned_type_node, casei); 8096 u = build_case_label (u, NULL, t); 8097 label_vec.quick_push (u); 8098 8099 si = gsi_last_bb (s_entry_bb); 8100 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION); 8101 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si))); 8102 gsi_remove (&si, true); 8103 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU; 8104 8105 if (s_exit_bb == NULL) 8106 continue; 8107 8108 si = gsi_last_bb (s_exit_bb); 8109 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN); 8110 gsi_remove (&si, true); 8111 8112 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU; 8113 } 8114 8115 /* Error handling code goes in DEFAULT_BB. */ 8116 t = gimple_block_label (default_bb); 8117 u = build_case_label (NULL, NULL, t); 8118 make_edge (l0_bb, default_bb, 0); 8119 add_bb_to_loop (default_bb, current_loops->tree_root); 8120 8121 stmt = gimple_build_switch (vmain, u, label_vec); 8122 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT); 8123 gsi_remove (&switch_si, true); 8124 8125 si = gsi_start_bb (default_bb); 8126 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0); 8127 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING); 8128 8129 if (exit_reachable) 8130 { 8131 tree bfn_decl; 8132 8133 /* Code to get the next section goes in L1_BB. */ 8134 si = gsi_last_bb (l1_bb); 8135 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE); 8136 8137 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT); 8138 stmt = gimple_build_call (bfn_decl, 0); 8139 gimple_call_set_lhs (stmt, vnext); 8140 gsi_insert_after (&si, stmt, GSI_SAME_STMT); 8141 gsi_remove (&si, true); 8142 8143 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU; 8144 } 8145 8146 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */ 8147 si = gsi_last_bb (l2_bb); 8148 if (gimple_omp_return_nowait_p (gsi_stmt (si))) 8149 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT); 8150 else if (gimple_omp_return_lhs (gsi_stmt (si))) 8151 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL); 8152 else 8153 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END); 8154 stmt = gimple_build_call (t, 0); 8155 if (gimple_omp_return_lhs (gsi_stmt (si))) 8156 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si))); 8157 gsi_insert_after (&si, stmt, GSI_SAME_STMT); 8158 gsi_remove (&si, true); 8159 8160 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb); 8161 } 8162 8163 8164 /* Expand code for an OpenMP single directive. We've already expanded 8165 much of the code, here we simply place the GOMP_barrier call. */ 8166 8167 static void 8168 expand_omp_single (struct omp_region *region) 8169 { 8170 basic_block entry_bb, exit_bb; 8171 gimple_stmt_iterator si; 8172 8173 entry_bb = region->entry; 8174 exit_bb = region->exit; 8175 8176 si = gsi_last_bb (entry_bb); 8177 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE); 8178 gsi_remove (&si, true); 8179 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 8180 8181 si = gsi_last_bb (exit_bb); 8182 if (!gimple_omp_return_nowait_p (gsi_stmt (si))) 8183 { 8184 tree t = gimple_omp_return_lhs (gsi_stmt (si)); 8185 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT); 8186 } 8187 gsi_remove (&si, true); 8188 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU; 8189 } 8190 8191 8192 /* Generic expansion for OpenMP synchronization directives: master, 8193 ordered and critical. All we need to do here is remove the entry 8194 and exit markers for REGION. */ 8195 8196 static void 8197 expand_omp_synch (struct omp_region *region) 8198 { 8199 basic_block entry_bb, exit_bb; 8200 gimple_stmt_iterator si; 8201 8202 entry_bb = region->entry; 8203 exit_bb = region->exit; 8204 8205 si = gsi_last_bb (entry_bb); 8206 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE 8207 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER 8208 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP 8209 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED 8210 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL 8211 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS); 8212 gsi_remove (&si, true); 8213 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 8214 8215 if (exit_bb) 8216 { 8217 si = gsi_last_bb (exit_bb); 8218 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN); 8219 gsi_remove (&si, true); 8220 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU; 8221 } 8222 } 8223 8224 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic 8225 operation as a normal volatile load. */ 8226 8227 static bool 8228 expand_omp_atomic_load (basic_block load_bb, tree addr, 8229 tree loaded_val, int index) 8230 { 8231 enum built_in_function tmpbase; 8232 gimple_stmt_iterator gsi; 8233 basic_block store_bb; 8234 location_t loc; 8235 gimple stmt; 8236 tree decl, call, type, itype; 8237 8238 gsi = gsi_last_bb (load_bb); 8239 stmt = gsi_stmt (gsi); 8240 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD); 8241 loc = gimple_location (stmt); 8242 8243 /* ??? If the target does not implement atomic_load_optab[mode], and mode 8244 is smaller than word size, then expand_atomic_load assumes that the load 8245 is atomic. We could avoid the builtin entirely in this case. */ 8246 8247 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1); 8248 decl = builtin_decl_explicit (tmpbase); 8249 if (decl == NULL_TREE) 8250 return false; 8251 8252 type = TREE_TYPE (loaded_val); 8253 itype = TREE_TYPE (TREE_TYPE (decl)); 8254 8255 call = build_call_expr_loc (loc, decl, 2, addr, 8256 build_int_cst (NULL, 8257 gimple_omp_atomic_seq_cst_p (stmt) 8258 ? MEMMODEL_SEQ_CST 8259 : MEMMODEL_RELAXED)); 8260 if (!useless_type_conversion_p (type, itype)) 8261 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call); 8262 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call); 8263 8264 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); 8265 gsi_remove (&gsi, true); 8266 8267 store_bb = single_succ (load_bb); 8268 gsi = gsi_last_bb (store_bb); 8269 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE); 8270 gsi_remove (&gsi, true); 8271 8272 if (gimple_in_ssa_p (cfun)) 8273 update_ssa (TODO_update_ssa_no_phi); 8274 8275 return true; 8276 } 8277 8278 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic 8279 operation as a normal volatile store. */ 8280 8281 static bool 8282 expand_omp_atomic_store (basic_block load_bb, tree addr, 8283 tree loaded_val, tree stored_val, int index) 8284 { 8285 enum built_in_function tmpbase; 8286 gimple_stmt_iterator gsi; 8287 basic_block store_bb = single_succ (load_bb); 8288 location_t loc; 8289 gimple stmt; 8290 tree decl, call, type, itype; 8291 machine_mode imode; 8292 bool exchange; 8293 8294 gsi = gsi_last_bb (load_bb); 8295 stmt = gsi_stmt (gsi); 8296 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD); 8297 8298 /* If the load value is needed, then this isn't a store but an exchange. */ 8299 exchange = gimple_omp_atomic_need_value_p (stmt); 8300 8301 gsi = gsi_last_bb (store_bb); 8302 stmt = gsi_stmt (gsi); 8303 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE); 8304 loc = gimple_location (stmt); 8305 8306 /* ??? If the target does not implement atomic_store_optab[mode], and mode 8307 is smaller than word size, then expand_atomic_store assumes that the store 8308 is atomic. We could avoid the builtin entirely in this case. */ 8309 8310 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N); 8311 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1); 8312 decl = builtin_decl_explicit (tmpbase); 8313 if (decl == NULL_TREE) 8314 return false; 8315 8316 type = TREE_TYPE (stored_val); 8317 8318 /* Dig out the type of the function's second argument. */ 8319 itype = TREE_TYPE (decl); 8320 itype = TYPE_ARG_TYPES (itype); 8321 itype = TREE_CHAIN (itype); 8322 itype = TREE_VALUE (itype); 8323 imode = TYPE_MODE (itype); 8324 8325 if (exchange && !can_atomic_exchange_p (imode, true)) 8326 return false; 8327 8328 if (!useless_type_conversion_p (itype, type)) 8329 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val); 8330 call = build_call_expr_loc (loc, decl, 3, addr, stored_val, 8331 build_int_cst (NULL, 8332 gimple_omp_atomic_seq_cst_p (stmt) 8333 ? MEMMODEL_SEQ_CST 8334 : MEMMODEL_RELAXED)); 8335 if (exchange) 8336 { 8337 if (!useless_type_conversion_p (type, itype)) 8338 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call); 8339 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call); 8340 } 8341 8342 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); 8343 gsi_remove (&gsi, true); 8344 8345 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */ 8346 gsi = gsi_last_bb (load_bb); 8347 gsi_remove (&gsi, true); 8348 8349 if (gimple_in_ssa_p (cfun)) 8350 update_ssa (TODO_update_ssa_no_phi); 8351 8352 return true; 8353 } 8354 8355 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic 8356 operation as a __atomic_fetch_op builtin. INDEX is log2 of the 8357 size of the data type, and thus usable to find the index of the builtin 8358 decl. Returns false if the expression is not of the proper form. */ 8359 8360 static bool 8361 expand_omp_atomic_fetch_op (basic_block load_bb, 8362 tree addr, tree loaded_val, 8363 tree stored_val, int index) 8364 { 8365 enum built_in_function oldbase, newbase, tmpbase; 8366 tree decl, itype, call; 8367 tree lhs, rhs; 8368 basic_block store_bb = single_succ (load_bb); 8369 gimple_stmt_iterator gsi; 8370 gimple stmt; 8371 location_t loc; 8372 enum tree_code code; 8373 bool need_old, need_new; 8374 machine_mode imode; 8375 bool seq_cst; 8376 8377 /* We expect to find the following sequences: 8378 8379 load_bb: 8380 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem) 8381 8382 store_bb: 8383 val = tmp OP something; (or: something OP tmp) 8384 GIMPLE_OMP_STORE (val) 8385 8386 ???FIXME: Allow a more flexible sequence. 8387 Perhaps use data flow to pick the statements. 8388 8389 */ 8390 8391 gsi = gsi_after_labels (store_bb); 8392 stmt = gsi_stmt (gsi); 8393 loc = gimple_location (stmt); 8394 if (!is_gimple_assign (stmt)) 8395 return false; 8396 gsi_next (&gsi); 8397 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE) 8398 return false; 8399 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi)); 8400 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb)); 8401 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb)); 8402 gcc_checking_assert (!need_old || !need_new); 8403 8404 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0)) 8405 return false; 8406 8407 /* Check for one of the supported fetch-op operations. */ 8408 code = gimple_assign_rhs_code (stmt); 8409 switch (code) 8410 { 8411 case PLUS_EXPR: 8412 case POINTER_PLUS_EXPR: 8413 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N; 8414 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N; 8415 break; 8416 case MINUS_EXPR: 8417 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N; 8418 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N; 8419 break; 8420 case BIT_AND_EXPR: 8421 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N; 8422 newbase = BUILT_IN_ATOMIC_AND_FETCH_N; 8423 break; 8424 case BIT_IOR_EXPR: 8425 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N; 8426 newbase = BUILT_IN_ATOMIC_OR_FETCH_N; 8427 break; 8428 case BIT_XOR_EXPR: 8429 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N; 8430 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N; 8431 break; 8432 default: 8433 return false; 8434 } 8435 8436 /* Make sure the expression is of the proper form. */ 8437 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0)) 8438 rhs = gimple_assign_rhs2 (stmt); 8439 else if (commutative_tree_code (gimple_assign_rhs_code (stmt)) 8440 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0)) 8441 rhs = gimple_assign_rhs1 (stmt); 8442 else 8443 return false; 8444 8445 tmpbase = ((enum built_in_function) 8446 ((need_new ? newbase : oldbase) + index + 1)); 8447 decl = builtin_decl_explicit (tmpbase); 8448 if (decl == NULL_TREE) 8449 return false; 8450 itype = TREE_TYPE (TREE_TYPE (decl)); 8451 imode = TYPE_MODE (itype); 8452 8453 /* We could test all of the various optabs involved, but the fact of the 8454 matter is that (with the exception of i486 vs i586 and xadd) all targets 8455 that support any atomic operaton optab also implements compare-and-swap. 8456 Let optabs.c take care of expanding any compare-and-swap loop. */ 8457 if (!can_compare_and_swap_p (imode, true)) 8458 return false; 8459 8460 gsi = gsi_last_bb (load_bb); 8461 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD); 8462 8463 /* OpenMP does not imply any barrier-like semantics on its atomic ops. 8464 It only requires that the operation happen atomically. Thus we can 8465 use the RELAXED memory model. */ 8466 call = build_call_expr_loc (loc, decl, 3, addr, 8467 fold_convert_loc (loc, itype, rhs), 8468 build_int_cst (NULL, 8469 seq_cst ? MEMMODEL_SEQ_CST 8470 : MEMMODEL_RELAXED)); 8471 8472 if (need_old || need_new) 8473 { 8474 lhs = need_old ? loaded_val : stored_val; 8475 call = fold_convert_loc (loc, TREE_TYPE (lhs), call); 8476 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call); 8477 } 8478 else 8479 call = fold_convert_loc (loc, void_type_node, call); 8480 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); 8481 gsi_remove (&gsi, true); 8482 8483 gsi = gsi_last_bb (store_bb); 8484 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE); 8485 gsi_remove (&gsi, true); 8486 gsi = gsi_last_bb (store_bb); 8487 gsi_remove (&gsi, true); 8488 8489 if (gimple_in_ssa_p (cfun)) 8490 update_ssa (TODO_update_ssa_no_phi); 8491 8492 return true; 8493 } 8494 8495 /* A subroutine of expand_omp_atomic. Implement the atomic operation as: 8496 8497 oldval = *addr; 8498 repeat: 8499 newval = rhs; // with oldval replacing *addr in rhs 8500 oldval = __sync_val_compare_and_swap (addr, oldval, newval); 8501 if (oldval != newval) 8502 goto repeat; 8503 8504 INDEX is log2 of the size of the data type, and thus usable to find the 8505 index of the builtin decl. */ 8506 8507 static bool 8508 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb, 8509 tree addr, tree loaded_val, tree stored_val, 8510 int index) 8511 { 8512 tree loadedi, storedi, initial, new_storedi, old_vali; 8513 tree type, itype, cmpxchg, iaddr; 8514 gimple_stmt_iterator si; 8515 basic_block loop_header = single_succ (load_bb); 8516 gimple phi, stmt; 8517 edge e; 8518 enum built_in_function fncode; 8519 8520 /* ??? We need a non-pointer interface to __atomic_compare_exchange in 8521 order to use the RELAXED memory model effectively. */ 8522 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N 8523 + index + 1); 8524 cmpxchg = builtin_decl_explicit (fncode); 8525 if (cmpxchg == NULL_TREE) 8526 return false; 8527 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr))); 8528 itype = TREE_TYPE (TREE_TYPE (cmpxchg)); 8529 8530 if (!can_compare_and_swap_p (TYPE_MODE (itype), true)) 8531 return false; 8532 8533 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */ 8534 si = gsi_last_bb (load_bb); 8535 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); 8536 8537 /* For floating-point values, we'll need to view-convert them to integers 8538 so that we can perform the atomic compare and swap. Simplify the 8539 following code by always setting up the "i"ntegral variables. */ 8540 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type)) 8541 { 8542 tree iaddr_val; 8543 8544 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode, 8545 true)); 8546 iaddr_val 8547 = force_gimple_operand_gsi (&si, 8548 fold_convert (TREE_TYPE (iaddr), addr), 8549 false, NULL_TREE, true, GSI_SAME_STMT); 8550 stmt = gimple_build_assign (iaddr, iaddr_val); 8551 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8552 loadedi = create_tmp_var (itype); 8553 if (gimple_in_ssa_p (cfun)) 8554 loadedi = make_ssa_name (loadedi); 8555 } 8556 else 8557 { 8558 iaddr = addr; 8559 loadedi = loaded_val; 8560 } 8561 8562 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1); 8563 tree loaddecl = builtin_decl_explicit (fncode); 8564 if (loaddecl) 8565 initial 8566 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)), 8567 build_call_expr (loaddecl, 2, iaddr, 8568 build_int_cst (NULL_TREE, 8569 MEMMODEL_RELAXED))); 8570 else 8571 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr, 8572 build_int_cst (TREE_TYPE (iaddr), 0)); 8573 8574 initial 8575 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true, 8576 GSI_SAME_STMT); 8577 8578 /* Move the value to the LOADEDI temporary. */ 8579 if (gimple_in_ssa_p (cfun)) 8580 { 8581 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header))); 8582 phi = create_phi_node (loadedi, loop_header); 8583 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)), 8584 initial); 8585 } 8586 else 8587 gsi_insert_before (&si, 8588 gimple_build_assign (loadedi, initial), 8589 GSI_SAME_STMT); 8590 if (loadedi != loaded_val) 8591 { 8592 gimple_stmt_iterator gsi2; 8593 tree x; 8594 8595 x = build1 (VIEW_CONVERT_EXPR, type, loadedi); 8596 gsi2 = gsi_start_bb (loop_header); 8597 if (gimple_in_ssa_p (cfun)) 8598 { 8599 gassign *stmt; 8600 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE, 8601 true, GSI_SAME_STMT); 8602 stmt = gimple_build_assign (loaded_val, x); 8603 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT); 8604 } 8605 else 8606 { 8607 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x); 8608 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE, 8609 true, GSI_SAME_STMT); 8610 } 8611 } 8612 gsi_remove (&si, true); 8613 8614 si = gsi_last_bb (store_bb); 8615 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE); 8616 8617 if (iaddr == addr) 8618 storedi = stored_val; 8619 else 8620 storedi = 8621 force_gimple_operand_gsi (&si, 8622 build1 (VIEW_CONVERT_EXPR, itype, 8623 stored_val), true, NULL_TREE, true, 8624 GSI_SAME_STMT); 8625 8626 /* Build the compare&swap statement. */ 8627 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi); 8628 new_storedi = force_gimple_operand_gsi (&si, 8629 fold_convert (TREE_TYPE (loadedi), 8630 new_storedi), 8631 true, NULL_TREE, 8632 true, GSI_SAME_STMT); 8633 8634 if (gimple_in_ssa_p (cfun)) 8635 old_vali = loadedi; 8636 else 8637 { 8638 old_vali = create_tmp_var (TREE_TYPE (loadedi)); 8639 stmt = gimple_build_assign (old_vali, loadedi); 8640 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8641 8642 stmt = gimple_build_assign (loadedi, new_storedi); 8643 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8644 } 8645 8646 /* Note that we always perform the comparison as an integer, even for 8647 floating point. This allows the atomic operation to properly 8648 succeed even with NaNs and -0.0. */ 8649 stmt = gimple_build_cond_empty 8650 (build2 (NE_EXPR, boolean_type_node, 8651 new_storedi, old_vali)); 8652 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8653 8654 /* Update cfg. */ 8655 e = single_succ_edge (store_bb); 8656 e->flags &= ~EDGE_FALLTHRU; 8657 e->flags |= EDGE_FALSE_VALUE; 8658 8659 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE); 8660 8661 /* Copy the new value to loadedi (we already did that before the condition 8662 if we are not in SSA). */ 8663 if (gimple_in_ssa_p (cfun)) 8664 { 8665 phi = gimple_seq_first_stmt (phi_nodes (loop_header)); 8666 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi); 8667 } 8668 8669 /* Remove GIMPLE_OMP_ATOMIC_STORE. */ 8670 gsi_remove (&si, true); 8671 8672 struct loop *loop = alloc_loop (); 8673 loop->header = loop_header; 8674 loop->latch = store_bb; 8675 add_loop (loop, loop_header->loop_father); 8676 8677 if (gimple_in_ssa_p (cfun)) 8678 update_ssa (TODO_update_ssa_no_phi); 8679 8680 return true; 8681 } 8682 8683 /* A subroutine of expand_omp_atomic. Implement the atomic operation as: 8684 8685 GOMP_atomic_start (); 8686 *addr = rhs; 8687 GOMP_atomic_end (); 8688 8689 The result is not globally atomic, but works so long as all parallel 8690 references are within #pragma omp atomic directives. According to 8691 responses received from omp@openmp.org, appears to be within spec. 8692 Which makes sense, since that's how several other compilers handle 8693 this situation as well. 8694 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're 8695 expanding. STORED_VAL is the operand of the matching 8696 GIMPLE_OMP_ATOMIC_STORE. 8697 8698 We replace 8699 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with 8700 loaded_val = *addr; 8701 8702 and replace 8703 GIMPLE_OMP_ATOMIC_STORE (stored_val) with 8704 *addr = stored_val; 8705 */ 8706 8707 static bool 8708 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb, 8709 tree addr, tree loaded_val, tree stored_val) 8710 { 8711 gimple_stmt_iterator si; 8712 gassign *stmt; 8713 tree t; 8714 8715 si = gsi_last_bb (load_bb); 8716 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); 8717 8718 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START); 8719 t = build_call_expr (t, 0); 8720 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT); 8721 8722 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr)); 8723 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8724 gsi_remove (&si, true); 8725 8726 si = gsi_last_bb (store_bb); 8727 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE); 8728 8729 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)), 8730 stored_val); 8731 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8732 8733 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END); 8734 t = build_call_expr (t, 0); 8735 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT); 8736 gsi_remove (&si, true); 8737 8738 if (gimple_in_ssa_p (cfun)) 8739 update_ssa (TODO_update_ssa_no_phi); 8740 return true; 8741 } 8742 8743 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand 8744 using expand_omp_atomic_fetch_op. If it failed, we try to 8745 call expand_omp_atomic_pipeline, and if it fails too, the 8746 ultimate fallback is wrapping the operation in a mutex 8747 (expand_omp_atomic_mutex). REGION is the atomic region built 8748 by build_omp_regions_1(). */ 8749 8750 static void 8751 expand_omp_atomic (struct omp_region *region) 8752 { 8753 basic_block load_bb = region->entry, store_bb = region->exit; 8754 gomp_atomic_load *load = as_a <gomp_atomic_load *> (last_stmt (load_bb)); 8755 gomp_atomic_store *store = as_a <gomp_atomic_store *> (last_stmt (store_bb)); 8756 tree loaded_val = gimple_omp_atomic_load_lhs (load); 8757 tree addr = gimple_omp_atomic_load_rhs (load); 8758 tree stored_val = gimple_omp_atomic_store_val (store); 8759 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr))); 8760 HOST_WIDE_INT index; 8761 8762 /* Make sure the type is one of the supported sizes. */ 8763 index = tree_to_uhwi (TYPE_SIZE_UNIT (type)); 8764 index = exact_log2 (index); 8765 if (index >= 0 && index <= 4) 8766 { 8767 unsigned int align = TYPE_ALIGN_UNIT (type); 8768 8769 /* __sync builtins require strict data alignment. */ 8770 if (exact_log2 (align) >= index) 8771 { 8772 /* Atomic load. */ 8773 if (loaded_val == stored_val 8774 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT 8775 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT) 8776 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD 8777 && expand_omp_atomic_load (load_bb, addr, loaded_val, index)) 8778 return; 8779 8780 /* Atomic store. */ 8781 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT 8782 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT) 8783 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD 8784 && store_bb == single_succ (load_bb) 8785 && first_stmt (store_bb) == store 8786 && expand_omp_atomic_store (load_bb, addr, loaded_val, 8787 stored_val, index)) 8788 return; 8789 8790 /* When possible, use specialized atomic update functions. */ 8791 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type)) 8792 && store_bb == single_succ (load_bb) 8793 && expand_omp_atomic_fetch_op (load_bb, addr, 8794 loaded_val, stored_val, index)) 8795 return; 8796 8797 /* If we don't have specialized __sync builtins, try and implement 8798 as a compare and swap loop. */ 8799 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr, 8800 loaded_val, stored_val, index)) 8801 return; 8802 } 8803 } 8804 8805 /* The ultimate fallback is wrapping the operation in a mutex. */ 8806 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val); 8807 } 8808 8809 8810 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */ 8811 8812 static void 8813 expand_omp_target (struct omp_region *region) 8814 { 8815 basic_block entry_bb, exit_bb, new_bb; 8816 struct function *child_cfun; 8817 tree child_fn, block, t; 8818 gimple_stmt_iterator gsi; 8819 gomp_target *entry_stmt; 8820 gimple stmt; 8821 edge e; 8822 bool offloaded, data_region; 8823 8824 entry_stmt = as_a <gomp_target *> (last_stmt (region->entry)); 8825 new_bb = region->entry; 8826 8827 offloaded = is_gimple_omp_offloaded (entry_stmt); 8828 switch (gimple_omp_target_kind (entry_stmt)) 8829 { 8830 case GF_OMP_TARGET_KIND_REGION: 8831 case GF_OMP_TARGET_KIND_UPDATE: 8832 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 8833 case GF_OMP_TARGET_KIND_OACC_KERNELS: 8834 case GF_OMP_TARGET_KIND_OACC_UPDATE: 8835 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 8836 data_region = false; 8837 break; 8838 case GF_OMP_TARGET_KIND_DATA: 8839 case GF_OMP_TARGET_KIND_OACC_DATA: 8840 data_region = true; 8841 break; 8842 default: 8843 gcc_unreachable (); 8844 } 8845 8846 child_fn = NULL_TREE; 8847 child_cfun = NULL; 8848 if (offloaded) 8849 { 8850 child_fn = gimple_omp_target_child_fn (entry_stmt); 8851 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 8852 } 8853 8854 /* Supported by expand_omp_taskreg, but not here. */ 8855 if (child_cfun != NULL) 8856 gcc_checking_assert (!child_cfun->cfg); 8857 gcc_checking_assert (!gimple_in_ssa_p (cfun)); 8858 8859 entry_bb = region->entry; 8860 exit_bb = region->exit; 8861 8862 if (offloaded) 8863 { 8864 unsigned srcidx, dstidx, num; 8865 8866 /* If the offloading region needs data sent from the parent 8867 function, then the very first statement (except possible 8868 tree profile counter updates) of the offloading body 8869 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since 8870 &.OMP_DATA_O is passed as an argument to the child function, 8871 we need to replace it with the argument as seen by the child 8872 function. 8873 8874 In most cases, this will end up being the identity assignment 8875 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had 8876 a function call that has been inlined, the original PARM_DECL 8877 .OMP_DATA_I may have been converted into a different local 8878 variable. In which case, we need to keep the assignment. */ 8879 tree data_arg = gimple_omp_target_data_arg (entry_stmt); 8880 if (data_arg) 8881 { 8882 basic_block entry_succ_bb = single_succ (entry_bb); 8883 gimple_stmt_iterator gsi; 8884 tree arg; 8885 gimple tgtcopy_stmt = NULL; 8886 tree sender = TREE_VEC_ELT (data_arg, 0); 8887 8888 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi)) 8889 { 8890 gcc_assert (!gsi_end_p (gsi)); 8891 stmt = gsi_stmt (gsi); 8892 if (gimple_code (stmt) != GIMPLE_ASSIGN) 8893 continue; 8894 8895 if (gimple_num_ops (stmt) == 2) 8896 { 8897 tree arg = gimple_assign_rhs1 (stmt); 8898 8899 /* We're ignoring the subcode because we're 8900 effectively doing a STRIP_NOPS. */ 8901 8902 if (TREE_CODE (arg) == ADDR_EXPR 8903 && TREE_OPERAND (arg, 0) == sender) 8904 { 8905 tgtcopy_stmt = stmt; 8906 break; 8907 } 8908 } 8909 } 8910 8911 gcc_assert (tgtcopy_stmt != NULL); 8912 arg = DECL_ARGUMENTS (child_fn); 8913 8914 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg); 8915 gsi_remove (&gsi, true); 8916 } 8917 8918 /* Declare local variables needed in CHILD_CFUN. */ 8919 block = DECL_INITIAL (child_fn); 8920 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls); 8921 /* The gimplifier could record temporaries in the offloading block 8922 rather than in containing function's local_decls chain, 8923 which would mean cgraph missed finalizing them. Do it now. */ 8924 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t)) 8925 if (TREE_CODE (t) == VAR_DECL 8926 && TREE_STATIC (t) 8927 && !DECL_EXTERNAL (t)) 8928 varpool_node::finalize_decl (t); 8929 DECL_SAVED_TREE (child_fn) = NULL; 8930 /* We'll create a CFG for child_fn, so no gimple body is needed. */ 8931 gimple_set_body (child_fn, NULL); 8932 TREE_USED (block) = 1; 8933 8934 /* Reset DECL_CONTEXT on function arguments. */ 8935 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) 8936 DECL_CONTEXT (t) = child_fn; 8937 8938 /* Split ENTRY_BB at GIMPLE_*, 8939 so that it can be moved to the child function. */ 8940 gsi = gsi_last_bb (entry_bb); 8941 stmt = gsi_stmt (gsi); 8942 gcc_assert (stmt 8943 && gimple_code (stmt) == gimple_code (entry_stmt)); 8944 e = split_block (entry_bb, stmt); 8945 gsi_remove (&gsi, true); 8946 entry_bb = e->dest; 8947 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 8948 8949 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */ 8950 if (exit_bb) 8951 { 8952 gsi = gsi_last_bb (exit_bb); 8953 gcc_assert (!gsi_end_p (gsi) 8954 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); 8955 stmt = gimple_build_return (NULL); 8956 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); 8957 gsi_remove (&gsi, true); 8958 } 8959 8960 /* Move the offloading region into CHILD_CFUN. */ 8961 8962 block = gimple_block (entry_stmt); 8963 8964 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block); 8965 if (exit_bb) 8966 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU; 8967 /* When the OMP expansion process cannot guarantee an up-to-date 8968 loop tree arrange for the child function to fixup loops. */ 8969 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP)) 8970 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP; 8971 8972 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */ 8973 num = vec_safe_length (child_cfun->local_decls); 8974 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++) 8975 { 8976 t = (*child_cfun->local_decls)[srcidx]; 8977 if (DECL_CONTEXT (t) == cfun->decl) 8978 continue; 8979 if (srcidx != dstidx) 8980 (*child_cfun->local_decls)[dstidx] = t; 8981 dstidx++; 8982 } 8983 if (dstidx != num) 8984 vec_safe_truncate (child_cfun->local_decls, dstidx); 8985 8986 /* Inform the callgraph about the new function. */ 8987 child_cfun->curr_properties = cfun->curr_properties; 8988 child_cfun->has_simduid_loops |= cfun->has_simduid_loops; 8989 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops; 8990 cgraph_node::add_new_function (child_fn, true); 8991 8992 #ifdef ENABLE_OFFLOADING 8993 /* Add the new function to the offload table. */ 8994 vec_safe_push (offload_funcs, child_fn); 8995 #endif 8996 8997 /* Fix the callgraph edges for child_cfun. Those for cfun will be 8998 fixed in a following pass. */ 8999 push_cfun (child_cfun); 9000 cgraph_edge::rebuild_edges (); 9001 9002 #ifdef ENABLE_OFFLOADING 9003 /* Prevent IPA from removing child_fn as unreachable, since there are no 9004 refs from the parent function to child_fn in offload LTO mode. */ 9005 struct cgraph_node *node = cgraph_node::get (child_fn); 9006 node->mark_force_output (); 9007 #endif 9008 9009 /* Some EH regions might become dead, see PR34608. If 9010 pass_cleanup_cfg isn't the first pass to happen with the 9011 new child, these dead EH edges might cause problems. 9012 Clean them up now. */ 9013 if (flag_exceptions) 9014 { 9015 basic_block bb; 9016 bool changed = false; 9017 9018 FOR_EACH_BB_FN (bb, cfun) 9019 changed |= gimple_purge_dead_eh_edges (bb); 9020 if (changed) 9021 cleanup_tree_cfg (); 9022 } 9023 pop_cfun (); 9024 } 9025 9026 /* Emit a library call to launch the offloading region, or do data 9027 transfers. */ 9028 tree t1, t2, t3, t4, device, cond, c, clauses; 9029 enum built_in_function start_ix; 9030 location_t clause_loc; 9031 9032 switch (gimple_omp_target_kind (entry_stmt)) 9033 { 9034 case GF_OMP_TARGET_KIND_REGION: 9035 start_ix = BUILT_IN_GOMP_TARGET; 9036 break; 9037 case GF_OMP_TARGET_KIND_DATA: 9038 start_ix = BUILT_IN_GOMP_TARGET_DATA; 9039 break; 9040 case GF_OMP_TARGET_KIND_UPDATE: 9041 start_ix = BUILT_IN_GOMP_TARGET_UPDATE; 9042 break; 9043 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 9044 case GF_OMP_TARGET_KIND_OACC_KERNELS: 9045 start_ix = BUILT_IN_GOACC_PARALLEL; 9046 break; 9047 case GF_OMP_TARGET_KIND_OACC_DATA: 9048 start_ix = BUILT_IN_GOACC_DATA_START; 9049 break; 9050 case GF_OMP_TARGET_KIND_OACC_UPDATE: 9051 start_ix = BUILT_IN_GOACC_UPDATE; 9052 break; 9053 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 9054 start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA; 9055 break; 9056 default: 9057 gcc_unreachable (); 9058 } 9059 9060 clauses = gimple_omp_target_clauses (entry_stmt); 9061 9062 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime 9063 library choose) and there is no conditional. */ 9064 cond = NULL_TREE; 9065 device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV); 9066 9067 c = find_omp_clause (clauses, OMP_CLAUSE_IF); 9068 if (c) 9069 cond = OMP_CLAUSE_IF_EXPR (c); 9070 9071 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE); 9072 if (c) 9073 { 9074 /* Even if we pass it to all library function calls, it is currently only 9075 defined/used for the OpenMP target ones. */ 9076 gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET 9077 || start_ix == BUILT_IN_GOMP_TARGET_DATA 9078 || start_ix == BUILT_IN_GOMP_TARGET_UPDATE); 9079 9080 device = OMP_CLAUSE_DEVICE_ID (c); 9081 clause_loc = OMP_CLAUSE_LOCATION (c); 9082 } 9083 else 9084 clause_loc = gimple_location (entry_stmt); 9085 9086 /* Ensure 'device' is of the correct type. */ 9087 device = fold_convert_loc (clause_loc, integer_type_node, device); 9088 9089 /* If we found the clause 'if (cond)', build 9090 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */ 9091 if (cond) 9092 { 9093 cond = gimple_boolify (cond); 9094 9095 basic_block cond_bb, then_bb, else_bb; 9096 edge e; 9097 tree tmp_var; 9098 9099 tmp_var = create_tmp_var (TREE_TYPE (device)); 9100 if (offloaded) 9101 e = split_block (new_bb, NULL); 9102 else 9103 { 9104 gsi = gsi_last_bb (new_bb); 9105 gsi_prev (&gsi); 9106 e = split_block (new_bb, gsi_stmt (gsi)); 9107 } 9108 cond_bb = e->src; 9109 new_bb = e->dest; 9110 remove_edge (e); 9111 9112 then_bb = create_empty_bb (cond_bb); 9113 else_bb = create_empty_bb (then_bb); 9114 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb); 9115 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb); 9116 9117 stmt = gimple_build_cond_empty (cond); 9118 gsi = gsi_last_bb (cond_bb); 9119 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 9120 9121 gsi = gsi_start_bb (then_bb); 9122 stmt = gimple_build_assign (tmp_var, device); 9123 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 9124 9125 gsi = gsi_start_bb (else_bb); 9126 stmt = gimple_build_assign (tmp_var, 9127 build_int_cst (integer_type_node, 9128 GOMP_DEVICE_HOST_FALLBACK)); 9129 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 9130 9131 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); 9132 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE); 9133 add_bb_to_loop (then_bb, cond_bb->loop_father); 9134 add_bb_to_loop (else_bb, cond_bb->loop_father); 9135 make_edge (then_bb, new_bb, EDGE_FALLTHRU); 9136 make_edge (else_bb, new_bb, EDGE_FALLTHRU); 9137 9138 device = tmp_var; 9139 gsi = gsi_last_bb (new_bb); 9140 } 9141 else 9142 { 9143 gsi = gsi_last_bb (new_bb); 9144 device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE, 9145 true, GSI_SAME_STMT); 9146 } 9147 9148 t = gimple_omp_target_data_arg (entry_stmt); 9149 if (t == NULL) 9150 { 9151 t1 = size_zero_node; 9152 t2 = build_zero_cst (ptr_type_node); 9153 t3 = t2; 9154 t4 = t2; 9155 } 9156 else 9157 { 9158 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1)))); 9159 t1 = size_binop (PLUS_EXPR, t1, size_int (1)); 9160 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0)); 9161 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1)); 9162 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2)); 9163 } 9164 9165 gimple g; 9166 /* The maximum number used by any start_ix, without varargs. */ 9167 auto_vec<tree, 11> args; 9168 args.quick_push (device); 9169 if (offloaded) 9170 args.quick_push (build_fold_addr_expr (child_fn)); 9171 switch (start_ix) 9172 { 9173 case BUILT_IN_GOMP_TARGET: 9174 case BUILT_IN_GOMP_TARGET_DATA: 9175 case BUILT_IN_GOMP_TARGET_UPDATE: 9176 /* This const void * is part of the current ABI, but we're not actually 9177 using it. */ 9178 args.quick_push (build_zero_cst (ptr_type_node)); 9179 break; 9180 case BUILT_IN_GOACC_DATA_START: 9181 case BUILT_IN_GOACC_ENTER_EXIT_DATA: 9182 case BUILT_IN_GOACC_PARALLEL: 9183 case BUILT_IN_GOACC_UPDATE: 9184 break; 9185 default: 9186 gcc_unreachable (); 9187 } 9188 args.quick_push (t1); 9189 args.quick_push (t2); 9190 args.quick_push (t3); 9191 args.quick_push (t4); 9192 switch (start_ix) 9193 { 9194 case BUILT_IN_GOACC_DATA_START: 9195 case BUILT_IN_GOMP_TARGET: 9196 case BUILT_IN_GOMP_TARGET_DATA: 9197 case BUILT_IN_GOMP_TARGET_UPDATE: 9198 break; 9199 case BUILT_IN_GOACC_PARALLEL: 9200 { 9201 tree t_num_gangs, t_num_workers, t_vector_length; 9202 9203 /* Default values for num_gangs, num_workers, and vector_length. */ 9204 t_num_gangs = t_num_workers = t_vector_length 9205 = fold_convert_loc (gimple_location (entry_stmt), 9206 integer_type_node, integer_one_node); 9207 /* ..., but if present, use the value specified by the respective 9208 clause, making sure that are of the correct type. */ 9209 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS); 9210 if (c) 9211 t_num_gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9212 integer_type_node, 9213 OMP_CLAUSE_NUM_GANGS_EXPR (c)); 9214 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_WORKERS); 9215 if (c) 9216 t_num_workers = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9217 integer_type_node, 9218 OMP_CLAUSE_NUM_WORKERS_EXPR (c)); 9219 c = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH); 9220 if (c) 9221 t_vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9222 integer_type_node, 9223 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c)); 9224 args.quick_push (t_num_gangs); 9225 args.quick_push (t_num_workers); 9226 args.quick_push (t_vector_length); 9227 } 9228 /* FALLTHRU */ 9229 case BUILT_IN_GOACC_ENTER_EXIT_DATA: 9230 case BUILT_IN_GOACC_UPDATE: 9231 { 9232 tree t_async; 9233 int t_wait_idx; 9234 9235 /* Default values for t_async. */ 9236 t_async = fold_convert_loc (gimple_location (entry_stmt), 9237 integer_type_node, 9238 build_int_cst (integer_type_node, 9239 GOMP_ASYNC_SYNC)); 9240 /* ..., but if present, use the value specified by the respective 9241 clause, making sure that is of the correct type. */ 9242 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC); 9243 if (c) 9244 t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9245 integer_type_node, 9246 OMP_CLAUSE_ASYNC_EXPR (c)); 9247 9248 args.quick_push (t_async); 9249 /* Save the index, and... */ 9250 t_wait_idx = args.length (); 9251 /* ... push a default value. */ 9252 args.quick_push (fold_convert_loc (gimple_location (entry_stmt), 9253 integer_type_node, 9254 integer_zero_node)); 9255 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT); 9256 if (c) 9257 { 9258 int n = 0; 9259 9260 for (; c; c = OMP_CLAUSE_CHAIN (c)) 9261 { 9262 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT) 9263 { 9264 args.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9265 integer_type_node, 9266 OMP_CLAUSE_WAIT_EXPR (c))); 9267 n++; 9268 } 9269 } 9270 9271 /* Now that we know the number, replace the default value. */ 9272 args.ordered_remove (t_wait_idx); 9273 args.quick_insert (t_wait_idx, 9274 fold_convert_loc (gimple_location (entry_stmt), 9275 integer_type_node, 9276 build_int_cst (integer_type_node, n))); 9277 } 9278 } 9279 break; 9280 default: 9281 gcc_unreachable (); 9282 } 9283 9284 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args); 9285 gimple_set_location (g, gimple_location (entry_stmt)); 9286 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 9287 if (!offloaded) 9288 { 9289 g = gsi_stmt (gsi); 9290 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET); 9291 gsi_remove (&gsi, true); 9292 } 9293 if (data_region 9294 && region->exit) 9295 { 9296 gsi = gsi_last_bb (region->exit); 9297 g = gsi_stmt (gsi); 9298 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN); 9299 gsi_remove (&gsi, true); 9300 } 9301 } 9302 9303 9304 /* Expand the parallel region tree rooted at REGION. Expansion 9305 proceeds in depth-first order. Innermost regions are expanded 9306 first. This way, parallel regions that require a new function to 9307 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any 9308 internal dependencies in their body. */ 9309 9310 static void 9311 expand_omp (struct omp_region *region) 9312 { 9313 while (region) 9314 { 9315 location_t saved_location; 9316 gimple inner_stmt = NULL; 9317 9318 /* First, determine whether this is a combined parallel+workshare 9319 region. */ 9320 if (region->type == GIMPLE_OMP_PARALLEL) 9321 determine_parallel_type (region); 9322 9323 if (region->type == GIMPLE_OMP_FOR 9324 && gimple_omp_for_combined_p (last_stmt (region->entry))) 9325 inner_stmt = last_stmt (region->inner->entry); 9326 9327 if (region->inner) 9328 expand_omp (region->inner); 9329 9330 saved_location = input_location; 9331 if (gimple_has_location (last_stmt (region->entry))) 9332 input_location = gimple_location (last_stmt (region->entry)); 9333 9334 switch (region->type) 9335 { 9336 case GIMPLE_OMP_PARALLEL: 9337 case GIMPLE_OMP_TASK: 9338 expand_omp_taskreg (region); 9339 break; 9340 9341 case GIMPLE_OMP_FOR: 9342 expand_omp_for (region, inner_stmt); 9343 break; 9344 9345 case GIMPLE_OMP_SECTIONS: 9346 expand_omp_sections (region); 9347 break; 9348 9349 case GIMPLE_OMP_SECTION: 9350 /* Individual omp sections are handled together with their 9351 parent GIMPLE_OMP_SECTIONS region. */ 9352 break; 9353 9354 case GIMPLE_OMP_SINGLE: 9355 expand_omp_single (region); 9356 break; 9357 9358 case GIMPLE_OMP_MASTER: 9359 case GIMPLE_OMP_TASKGROUP: 9360 case GIMPLE_OMP_ORDERED: 9361 case GIMPLE_OMP_CRITICAL: 9362 case GIMPLE_OMP_TEAMS: 9363 expand_omp_synch (region); 9364 break; 9365 9366 case GIMPLE_OMP_ATOMIC_LOAD: 9367 expand_omp_atomic (region); 9368 break; 9369 9370 case GIMPLE_OMP_TARGET: 9371 expand_omp_target (region); 9372 break; 9373 9374 default: 9375 gcc_unreachable (); 9376 } 9377 9378 input_location = saved_location; 9379 region = region->next; 9380 } 9381 } 9382 9383 9384 /* Helper for build_omp_regions. Scan the dominator tree starting at 9385 block BB. PARENT is the region that contains BB. If SINGLE_TREE is 9386 true, the function ends once a single tree is built (otherwise, whole 9387 forest of OMP constructs may be built). */ 9388 9389 static void 9390 build_omp_regions_1 (basic_block bb, struct omp_region *parent, 9391 bool single_tree) 9392 { 9393 gimple_stmt_iterator gsi; 9394 gimple stmt; 9395 basic_block son; 9396 9397 gsi = gsi_last_bb (bb); 9398 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi))) 9399 { 9400 struct omp_region *region; 9401 enum gimple_code code; 9402 9403 stmt = gsi_stmt (gsi); 9404 code = gimple_code (stmt); 9405 if (code == GIMPLE_OMP_RETURN) 9406 { 9407 /* STMT is the return point out of region PARENT. Mark it 9408 as the exit point and make PARENT the immediately 9409 enclosing region. */ 9410 gcc_assert (parent); 9411 region = parent; 9412 region->exit = bb; 9413 parent = parent->outer; 9414 } 9415 else if (code == GIMPLE_OMP_ATOMIC_STORE) 9416 { 9417 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to 9418 GIMPLE_OMP_RETURN, but matches with 9419 GIMPLE_OMP_ATOMIC_LOAD. */ 9420 gcc_assert (parent); 9421 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD); 9422 region = parent; 9423 region->exit = bb; 9424 parent = parent->outer; 9425 } 9426 else if (code == GIMPLE_OMP_CONTINUE) 9427 { 9428 gcc_assert (parent); 9429 parent->cont = bb; 9430 } 9431 else if (code == GIMPLE_OMP_SECTIONS_SWITCH) 9432 { 9433 /* GIMPLE_OMP_SECTIONS_SWITCH is part of 9434 GIMPLE_OMP_SECTIONS, and we do nothing for it. */ 9435 } 9436 else 9437 { 9438 region = new_omp_region (bb, code, parent); 9439 /* Otherwise... */ 9440 if (code == GIMPLE_OMP_TARGET) 9441 { 9442 switch (gimple_omp_target_kind (stmt)) 9443 { 9444 case GF_OMP_TARGET_KIND_REGION: 9445 case GF_OMP_TARGET_KIND_DATA: 9446 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 9447 case GF_OMP_TARGET_KIND_OACC_KERNELS: 9448 case GF_OMP_TARGET_KIND_OACC_DATA: 9449 break; 9450 case GF_OMP_TARGET_KIND_UPDATE: 9451 case GF_OMP_TARGET_KIND_OACC_UPDATE: 9452 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 9453 /* ..., other than for those stand-alone directives... */ 9454 region = NULL; 9455 break; 9456 default: 9457 gcc_unreachable (); 9458 } 9459 } 9460 /* ..., this directive becomes the parent for a new region. */ 9461 if (region) 9462 parent = region; 9463 } 9464 } 9465 9466 if (single_tree && !parent) 9467 return; 9468 9469 for (son = first_dom_son (CDI_DOMINATORS, bb); 9470 son; 9471 son = next_dom_son (CDI_DOMINATORS, son)) 9472 build_omp_regions_1 (son, parent, single_tree); 9473 } 9474 9475 /* Builds the tree of OMP regions rooted at ROOT, storing it to 9476 root_omp_region. */ 9477 9478 static void 9479 build_omp_regions_root (basic_block root) 9480 { 9481 gcc_assert (root_omp_region == NULL); 9482 build_omp_regions_1 (root, NULL, true); 9483 gcc_assert (root_omp_region != NULL); 9484 } 9485 9486 /* Expands omp construct (and its subconstructs) starting in HEAD. */ 9487 9488 void 9489 omp_expand_local (basic_block head) 9490 { 9491 build_omp_regions_root (head); 9492 if (dump_file && (dump_flags & TDF_DETAILS)) 9493 { 9494 fprintf (dump_file, "\nOMP region tree\n\n"); 9495 dump_omp_region (dump_file, root_omp_region, 0); 9496 fprintf (dump_file, "\n"); 9497 } 9498 9499 remove_exit_barriers (root_omp_region); 9500 expand_omp (root_omp_region); 9501 9502 free_omp_regions (); 9503 } 9504 9505 /* Scan the CFG and build a tree of OMP regions. Return the root of 9506 the OMP region tree. */ 9507 9508 static void 9509 build_omp_regions (void) 9510 { 9511 gcc_assert (root_omp_region == NULL); 9512 calculate_dominance_info (CDI_DOMINATORS); 9513 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false); 9514 } 9515 9516 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */ 9517 9518 static unsigned int 9519 execute_expand_omp (void) 9520 { 9521 build_omp_regions (); 9522 9523 if (!root_omp_region) 9524 return 0; 9525 9526 if (dump_file) 9527 { 9528 fprintf (dump_file, "\nOMP region tree\n\n"); 9529 dump_omp_region (dump_file, root_omp_region, 0); 9530 fprintf (dump_file, "\n"); 9531 } 9532 9533 remove_exit_barriers (root_omp_region); 9534 9535 expand_omp (root_omp_region); 9536 9537 cleanup_tree_cfg (); 9538 9539 free_omp_regions (); 9540 9541 return 0; 9542 } 9543 9544 /* OMP expansion -- the default pass, run before creation of SSA form. */ 9545 9546 namespace { 9547 9548 const pass_data pass_data_expand_omp = 9549 { 9550 GIMPLE_PASS, /* type */ 9551 "ompexp", /* name */ 9552 OPTGROUP_NONE, /* optinfo_flags */ 9553 TV_NONE, /* tv_id */ 9554 PROP_gimple_any, /* properties_required */ 9555 PROP_gimple_eomp, /* properties_provided */ 9556 0, /* properties_destroyed */ 9557 0, /* todo_flags_start */ 9558 0, /* todo_flags_finish */ 9559 }; 9560 9561 class pass_expand_omp : public gimple_opt_pass 9562 { 9563 public: 9564 pass_expand_omp (gcc::context *ctxt) 9565 : gimple_opt_pass (pass_data_expand_omp, ctxt) 9566 {} 9567 9568 /* opt_pass methods: */ 9569 virtual unsigned int execute (function *) 9570 { 9571 bool gate = ((flag_cilkplus != 0 || flag_openacc != 0 || flag_openmp != 0 9572 || flag_openmp_simd != 0) 9573 && !seen_error ()); 9574 9575 /* This pass always runs, to provide PROP_gimple_eomp. 9576 But often, there is nothing to do. */ 9577 if (!gate) 9578 return 0; 9579 9580 return execute_expand_omp (); 9581 } 9582 9583 }; // class pass_expand_omp 9584 9585 } // anon namespace 9586 9587 gimple_opt_pass * 9588 make_pass_expand_omp (gcc::context *ctxt) 9589 { 9590 return new pass_expand_omp (ctxt); 9591 } 9592 9593 namespace { 9594 9595 const pass_data pass_data_expand_omp_ssa = 9596 { 9597 GIMPLE_PASS, /* type */ 9598 "ompexpssa", /* name */ 9599 OPTGROUP_NONE, /* optinfo_flags */ 9600 TV_NONE, /* tv_id */ 9601 PROP_cfg | PROP_ssa, /* properties_required */ 9602 PROP_gimple_eomp, /* properties_provided */ 9603 0, /* properties_destroyed */ 9604 0, /* todo_flags_start */ 9605 TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */ 9606 }; 9607 9608 class pass_expand_omp_ssa : public gimple_opt_pass 9609 { 9610 public: 9611 pass_expand_omp_ssa (gcc::context *ctxt) 9612 : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt) 9613 {} 9614 9615 /* opt_pass methods: */ 9616 virtual bool gate (function *fun) 9617 { 9618 return !(fun->curr_properties & PROP_gimple_eomp); 9619 } 9620 virtual unsigned int execute (function *) { return execute_expand_omp (); } 9621 9622 }; // class pass_expand_omp_ssa 9623 9624 } // anon namespace 9625 9626 gimple_opt_pass * 9627 make_pass_expand_omp_ssa (gcc::context *ctxt) 9628 { 9629 return new pass_expand_omp_ssa (ctxt); 9630 } 9631 9632 /* Routines to lower OMP directives into OMP-GIMPLE. */ 9633 9634 /* Helper function to preform, potentially COMPLEX_TYPE, operation and 9635 convert it to gimple. */ 9636 static void 9637 oacc_gimple_assign (tree dest, tree_code op, tree src, gimple_seq *seq) 9638 { 9639 gimple stmt; 9640 9641 if (TREE_CODE (TREE_TYPE (dest)) != COMPLEX_TYPE) 9642 { 9643 stmt = gimple_build_assign (dest, op, dest, src); 9644 gimple_seq_add_stmt (seq, stmt); 9645 return; 9646 } 9647 9648 tree t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9649 tree rdest = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest); 9650 gimplify_assign (t, rdest, seq); 9651 rdest = t; 9652 9653 t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9654 tree idest = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest); 9655 gimplify_assign (t, idest, seq); 9656 idest = t; 9657 9658 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src))); 9659 tree rsrc = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src); 9660 gimplify_assign (t, rsrc, seq); 9661 rsrc = t; 9662 9663 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src))); 9664 tree isrc = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src); 9665 gimplify_assign (t, isrc, seq); 9666 isrc = t; 9667 9668 tree r = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9669 tree i = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9670 tree result; 9671 9672 if (op == PLUS_EXPR) 9673 { 9674 stmt = gimple_build_assign (r, op, rdest, rsrc); 9675 gimple_seq_add_stmt (seq, stmt); 9676 9677 stmt = gimple_build_assign (i, op, idest, isrc); 9678 gimple_seq_add_stmt (seq, stmt); 9679 } 9680 else if (op == MULT_EXPR) 9681 { 9682 /* Let x = a + ib = dest, y = c + id = src. 9683 x * y = (ac - bd) + i(ad + bc) */ 9684 tree ac = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9685 tree bd = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9686 tree ad = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9687 tree bc = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9688 9689 stmt = gimple_build_assign (ac, MULT_EXPR, rdest, rsrc); 9690 gimple_seq_add_stmt (seq, stmt); 9691 9692 stmt = gimple_build_assign (bd, MULT_EXPR, idest, isrc); 9693 gimple_seq_add_stmt (seq, stmt); 9694 9695 stmt = gimple_build_assign (r, MINUS_EXPR, ac, bd); 9696 gimple_seq_add_stmt (seq, stmt); 9697 9698 stmt = gimple_build_assign (ad, MULT_EXPR, rdest, isrc); 9699 gimple_seq_add_stmt (seq, stmt); 9700 9701 stmt = gimple_build_assign (bd, MULT_EXPR, idest, rsrc); 9702 gimple_seq_add_stmt (seq, stmt); 9703 9704 stmt = gimple_build_assign (i, PLUS_EXPR, ad, bc); 9705 gimple_seq_add_stmt (seq, stmt); 9706 } 9707 else 9708 gcc_unreachable (); 9709 9710 result = build2 (COMPLEX_EXPR, TREE_TYPE (dest), r, i); 9711 gimplify_assign (dest, result, seq); 9712 } 9713 9714 /* Helper function to initialize local data for the reduction arrays. 9715 The reduction arrays need to be placed inside the calling function 9716 for accelerators, or else the host won't be able to preform the final 9717 reduction. */ 9718 9719 static void 9720 oacc_initialize_reduction_data (tree clauses, tree nthreads, 9721 gimple_seq *stmt_seqp, omp_context *ctx) 9722 { 9723 tree c, t, oc; 9724 gimple stmt; 9725 omp_context *octx; 9726 9727 /* Find the innermost OpenACC parallel context. */ 9728 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET 9729 && (gimple_omp_target_kind (ctx->stmt) 9730 == GF_OMP_TARGET_KIND_OACC_PARALLEL)) 9731 octx = ctx; 9732 else 9733 octx = ctx->outer; 9734 gcc_checking_assert (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET 9735 && (gimple_omp_target_kind (octx->stmt) 9736 == GF_OMP_TARGET_KIND_OACC_PARALLEL)); 9737 9738 /* Extract the clauses. */ 9739 oc = gimple_omp_target_clauses (octx->stmt); 9740 9741 /* Find the last outer clause. */ 9742 for (; oc && OMP_CLAUSE_CHAIN (oc); oc = OMP_CLAUSE_CHAIN (oc)) 9743 ; 9744 9745 /* Allocate arrays for each reduction variable. */ 9746 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 9747 { 9748 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) 9749 continue; 9750 9751 tree var = OMP_CLAUSE_DECL (c); 9752 tree type = get_base_type (var); 9753 tree array = lookup_oacc_reduction (oacc_get_reduction_array_id (var), 9754 ctx); 9755 tree size, call; 9756 9757 /* Calculate size of the reduction array. */ 9758 t = create_tmp_var (TREE_TYPE (nthreads)); 9759 stmt = gimple_build_assign (t, MULT_EXPR, nthreads, 9760 fold_convert (TREE_TYPE (nthreads), 9761 TYPE_SIZE_UNIT (type))); 9762 gimple_seq_add_stmt (stmt_seqp, stmt); 9763 9764 size = create_tmp_var (sizetype); 9765 gimplify_assign (size, fold_build1 (NOP_EXPR, sizetype, t), stmt_seqp); 9766 9767 /* Now allocate memory for it. */ 9768 call = unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA)); 9769 stmt = gimple_build_call (call, 1, size); 9770 gimple_call_set_lhs (stmt, array); 9771 gimple_seq_add_stmt (stmt_seqp, stmt); 9772 9773 /* Map this array into the accelerator. */ 9774 9775 /* Add the reduction array to the list of clauses. */ 9776 tree x = array; 9777 t = build_omp_clause (gimple_location (ctx->stmt), OMP_CLAUSE_MAP); 9778 OMP_CLAUSE_SET_MAP_KIND (t, GOMP_MAP_FORCE_FROM); 9779 OMP_CLAUSE_DECL (t) = x; 9780 OMP_CLAUSE_CHAIN (t) = NULL; 9781 if (oc) 9782 OMP_CLAUSE_CHAIN (oc) = t; 9783 else 9784 gimple_omp_target_set_clauses (as_a <gomp_target *> (octx->stmt), t); 9785 OMP_CLAUSE_SIZE (t) = size; 9786 oc = t; 9787 } 9788 } 9789 9790 /* Helper function to process the array of partial reductions. Nthreads 9791 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS 9792 cannot be used here, because nthreads on the host may be different than 9793 on the accelerator. */ 9794 9795 static void 9796 oacc_finalize_reduction_data (tree clauses, tree nthreads, 9797 gimple_seq *stmt_seqp, omp_context *ctx) 9798 { 9799 tree c, x, var, array, loop_header, loop_body, loop_exit, type; 9800 gimple stmt; 9801 9802 /* Create for loop. 9803 9804 let var = the original reduction variable 9805 let array = reduction variable array 9806 9807 for (i = 0; i < nthreads; i++) 9808 var op= array[i] 9809 */ 9810 9811 loop_header = create_artificial_label (UNKNOWN_LOCATION); 9812 loop_body = create_artificial_label (UNKNOWN_LOCATION); 9813 loop_exit = create_artificial_label (UNKNOWN_LOCATION); 9814 9815 /* Create and initialize an index variable. */ 9816 tree ix = create_tmp_var (sizetype); 9817 gimplify_assign (ix, fold_build1 (NOP_EXPR, sizetype, integer_zero_node), 9818 stmt_seqp); 9819 9820 /* Insert the loop header label here. */ 9821 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_header)); 9822 9823 /* Exit loop if ix >= nthreads. */ 9824 x = create_tmp_var (sizetype); 9825 gimplify_assign (x, fold_build1 (NOP_EXPR, sizetype, nthreads), stmt_seqp); 9826 stmt = gimple_build_cond (GE_EXPR, ix, x, loop_exit, loop_body); 9827 gimple_seq_add_stmt (stmt_seqp, stmt); 9828 9829 /* Insert the loop body label here. */ 9830 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_body)); 9831 9832 /* Collapse each reduction array, one element at a time. */ 9833 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 9834 { 9835 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) 9836 continue; 9837 9838 tree_code reduction_code = OMP_CLAUSE_REDUCTION_CODE (c); 9839 9840 /* reduction(-:var) sums up the partial results, so it acts 9841 identically to reduction(+:var). */ 9842 if (reduction_code == MINUS_EXPR) 9843 reduction_code = PLUS_EXPR; 9844 9845 /* Set up reduction variable var. */ 9846 var = OMP_CLAUSE_DECL (c); 9847 type = get_base_type (var); 9848 array = lookup_oacc_reduction (oacc_get_reduction_array_id 9849 (OMP_CLAUSE_DECL (c)), ctx); 9850 9851 /* Calculate the array offset. */ 9852 tree offset = create_tmp_var (sizetype); 9853 gimplify_assign (offset, TYPE_SIZE_UNIT (type), stmt_seqp); 9854 stmt = gimple_build_assign (offset, MULT_EXPR, offset, ix); 9855 gimple_seq_add_stmt (stmt_seqp, stmt); 9856 9857 tree ptr = create_tmp_var (TREE_TYPE (array)); 9858 stmt = gimple_build_assign (ptr, POINTER_PLUS_EXPR, array, offset); 9859 gimple_seq_add_stmt (stmt_seqp, stmt); 9860 9861 /* Extract array[ix] into mem. */ 9862 tree mem = create_tmp_var (type); 9863 gimplify_assign (mem, build_simple_mem_ref (ptr), stmt_seqp); 9864 9865 /* Find the original reduction variable. */ 9866 if (is_reference (var)) 9867 var = build_simple_mem_ref (var); 9868 9869 tree t = create_tmp_var (type); 9870 9871 x = lang_hooks.decls.omp_clause_assign_op (c, t, var); 9872 gimplify_and_add (unshare_expr(x), stmt_seqp); 9873 9874 /* var = var op mem */ 9875 switch (OMP_CLAUSE_REDUCTION_CODE (c)) 9876 { 9877 case TRUTH_ANDIF_EXPR: 9878 case TRUTH_ORIF_EXPR: 9879 t = fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c), integer_type_node, 9880 t, mem); 9881 gimplify_and_add (t, stmt_seqp); 9882 break; 9883 default: 9884 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */ 9885 oacc_gimple_assign (t, OMP_CLAUSE_REDUCTION_CODE (c), mem, 9886 stmt_seqp); 9887 } 9888 9889 t = fold_build1 (NOP_EXPR, TREE_TYPE (var), t); 9890 x = lang_hooks.decls.omp_clause_assign_op (c, var, t); 9891 gimplify_and_add (unshare_expr(x), stmt_seqp); 9892 } 9893 9894 /* Increment the induction variable. */ 9895 tree one = fold_build1 (NOP_EXPR, sizetype, integer_one_node); 9896 stmt = gimple_build_assign (ix, PLUS_EXPR, ix, one); 9897 gimple_seq_add_stmt (stmt_seqp, stmt); 9898 9899 /* Go back to the top of the loop. */ 9900 gimple_seq_add_stmt (stmt_seqp, gimple_build_goto (loop_header)); 9901 9902 /* Place the loop exit label here. */ 9903 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_exit)); 9904 } 9905 9906 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and 9907 scan that for reductions. */ 9908 9909 static void 9910 oacc_process_reduction_data (gimple_seq *body, gimple_seq *in_stmt_seqp, 9911 gimple_seq *out_stmt_seqp, omp_context *ctx) 9912 { 9913 gimple_stmt_iterator gsi; 9914 gimple_seq inner = NULL; 9915 9916 /* A collapse clause may have inserted a new bind block. */ 9917 gsi = gsi_start (*body); 9918 while (!gsi_end_p (gsi)) 9919 { 9920 gimple stmt = gsi_stmt (gsi); 9921 if (gbind *bind_stmt = dyn_cast <gbind *> (stmt)) 9922 { 9923 inner = gimple_bind_body (bind_stmt); 9924 body = &inner; 9925 gsi = gsi_start (*body); 9926 } 9927 else if (dyn_cast <gomp_for *> (stmt)) 9928 break; 9929 else 9930 gsi_next (&gsi); 9931 } 9932 9933 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) 9934 { 9935 tree clauses, nthreads, t, c, acc_device, acc_device_host, call, 9936 enter, exit; 9937 bool reduction_found = false; 9938 9939 gimple stmt = gsi_stmt (gsi); 9940 9941 switch (gimple_code (stmt)) 9942 { 9943 case GIMPLE_OMP_FOR: 9944 clauses = gimple_omp_for_clauses (stmt); 9945 9946 /* Search for a reduction clause. */ 9947 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 9948 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) 9949 { 9950 reduction_found = true; 9951 break; 9952 } 9953 9954 if (!reduction_found) 9955 break; 9956 9957 ctx = maybe_lookup_ctx (stmt); 9958 t = NULL_TREE; 9959 9960 /* Extract the number of threads. */ 9961 nthreads = create_tmp_var (sizetype); 9962 t = oacc_max_threads (ctx); 9963 gimplify_assign (nthreads, t, in_stmt_seqp); 9964 9965 /* Determine if this is kernel will be executed on the host. */ 9966 call = builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE); 9967 acc_device = create_tmp_var (integer_type_node, ".acc_device_type"); 9968 stmt = gimple_build_call (call, 0); 9969 gimple_call_set_lhs (stmt, acc_device); 9970 gimple_seq_add_stmt (in_stmt_seqp, stmt); 9971 9972 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */ 9973 acc_device_host = create_tmp_var (integer_type_node, 9974 ".acc_device_host"); 9975 gimplify_assign (acc_device_host, 9976 build_int_cst (integer_type_node, 9977 GOMP_DEVICE_HOST), 9978 in_stmt_seqp); 9979 9980 enter = create_artificial_label (UNKNOWN_LOCATION); 9981 exit = create_artificial_label (UNKNOWN_LOCATION); 9982 9983 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host, 9984 enter, exit); 9985 gimple_seq_add_stmt (in_stmt_seqp, stmt); 9986 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter)); 9987 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype, 9988 integer_one_node), 9989 in_stmt_seqp); 9990 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit)); 9991 9992 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */ 9993 gimplify_assign (acc_device_host, 9994 build_int_cst (integer_type_node, 9995 GOMP_DEVICE_HOST_NONSHM), 9996 in_stmt_seqp); 9997 9998 enter = create_artificial_label (UNKNOWN_LOCATION); 9999 exit = create_artificial_label (UNKNOWN_LOCATION); 10000 10001 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host, 10002 enter, exit); 10003 gimple_seq_add_stmt (in_stmt_seqp, stmt); 10004 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter)); 10005 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype, 10006 integer_one_node), 10007 in_stmt_seqp); 10008 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit)); 10009 10010 oacc_initialize_reduction_data (clauses, nthreads, in_stmt_seqp, 10011 ctx); 10012 oacc_finalize_reduction_data (clauses, nthreads, out_stmt_seqp, ctx); 10013 break; 10014 default: 10015 // Scan for other directives which support reduction here. 10016 break; 10017 } 10018 } 10019 } 10020 10021 /* If ctx is a worksharing context inside of a cancellable parallel 10022 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN 10023 and conditional branch to parallel's cancel_label to handle 10024 cancellation in the implicit barrier. */ 10025 10026 static void 10027 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body) 10028 { 10029 gimple omp_return = gimple_seq_last_stmt (*body); 10030 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN); 10031 if (gimple_omp_return_nowait_p (omp_return)) 10032 return; 10033 if (ctx->outer 10034 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL 10035 && ctx->outer->cancellable) 10036 { 10037 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL); 10038 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl)); 10039 tree lhs = create_tmp_var (c_bool_type); 10040 gimple_omp_return_set_lhs (omp_return, lhs); 10041 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION); 10042 gimple g = gimple_build_cond (NE_EXPR, lhs, 10043 fold_convert (c_bool_type, 10044 boolean_false_node), 10045 ctx->outer->cancel_label, fallthru_label); 10046 gimple_seq_add_stmt (body, g); 10047 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label)); 10048 } 10049 } 10050 10051 /* Lower the OpenMP sections directive in the current statement in GSI_P. 10052 CTX is the enclosing OMP context for the current statement. */ 10053 10054 static void 10055 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10056 { 10057 tree block, control; 10058 gimple_stmt_iterator tgsi; 10059 gomp_sections *stmt; 10060 gimple t; 10061 gbind *new_stmt, *bind; 10062 gimple_seq ilist, dlist, olist, new_body; 10063 10064 stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p)); 10065 10066 push_gimplify_context (); 10067 10068 dlist = NULL; 10069 ilist = NULL; 10070 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt), 10071 &ilist, &dlist, ctx, NULL); 10072 10073 new_body = gimple_omp_body (stmt); 10074 gimple_omp_set_body (stmt, NULL); 10075 tgsi = gsi_start (new_body); 10076 for (; !gsi_end_p (tgsi); gsi_next (&tgsi)) 10077 { 10078 omp_context *sctx; 10079 gimple sec_start; 10080 10081 sec_start = gsi_stmt (tgsi); 10082 sctx = maybe_lookup_ctx (sec_start); 10083 gcc_assert (sctx); 10084 10085 lower_omp (gimple_omp_body_ptr (sec_start), sctx); 10086 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start), 10087 GSI_CONTINUE_LINKING); 10088 gimple_omp_set_body (sec_start, NULL); 10089 10090 if (gsi_one_before_end_p (tgsi)) 10091 { 10092 gimple_seq l = NULL; 10093 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL, 10094 &l, ctx); 10095 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING); 10096 gimple_omp_section_set_last (sec_start); 10097 } 10098 10099 gsi_insert_after (&tgsi, gimple_build_omp_return (false), 10100 GSI_CONTINUE_LINKING); 10101 } 10102 10103 block = make_node (BLOCK); 10104 bind = gimple_build_bind (NULL, new_body, block); 10105 10106 olist = NULL; 10107 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx); 10108 10109 block = make_node (BLOCK); 10110 new_stmt = gimple_build_bind (NULL, NULL, block); 10111 gsi_replace (gsi_p, new_stmt, true); 10112 10113 pop_gimplify_context (new_stmt); 10114 gimple_bind_append_vars (new_stmt, ctx->block_vars); 10115 BLOCK_VARS (block) = gimple_bind_vars (bind); 10116 if (BLOCK_VARS (block)) 10117 TREE_USED (block) = 1; 10118 10119 new_body = NULL; 10120 gimple_seq_add_seq (&new_body, ilist); 10121 gimple_seq_add_stmt (&new_body, stmt); 10122 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ()); 10123 gimple_seq_add_stmt (&new_body, bind); 10124 10125 control = create_tmp_var (unsigned_type_node, ".section"); 10126 t = gimple_build_omp_continue (control, control); 10127 gimple_omp_sections_set_control (stmt, control); 10128 gimple_seq_add_stmt (&new_body, t); 10129 10130 gimple_seq_add_seq (&new_body, olist); 10131 if (ctx->cancellable) 10132 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label)); 10133 gimple_seq_add_seq (&new_body, dlist); 10134 10135 new_body = maybe_catch_exception (new_body); 10136 10137 t = gimple_build_omp_return 10138 (!!find_omp_clause (gimple_omp_sections_clauses (stmt), 10139 OMP_CLAUSE_NOWAIT)); 10140 gimple_seq_add_stmt (&new_body, t); 10141 maybe_add_implicit_barrier_cancel (ctx, &new_body); 10142 10143 gimple_bind_set_body (new_stmt, new_body); 10144 } 10145 10146 10147 /* A subroutine of lower_omp_single. Expand the simple form of 10148 a GIMPLE_OMP_SINGLE, without a copyprivate clause: 10149 10150 if (GOMP_single_start ()) 10151 BODY; 10152 [ GOMP_barrier (); ] -> unless 'nowait' is present. 10153 10154 FIXME. It may be better to delay expanding the logic of this until 10155 pass_expand_omp. The expanded logic may make the job more difficult 10156 to a synchronization analysis pass. */ 10157 10158 static void 10159 lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p) 10160 { 10161 location_t loc = gimple_location (single_stmt); 10162 tree tlabel = create_artificial_label (loc); 10163 tree flabel = create_artificial_label (loc); 10164 gimple call, cond; 10165 tree lhs, decl; 10166 10167 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START); 10168 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl))); 10169 call = gimple_build_call (decl, 0); 10170 gimple_call_set_lhs (call, lhs); 10171 gimple_seq_add_stmt (pre_p, call); 10172 10173 cond = gimple_build_cond (EQ_EXPR, lhs, 10174 fold_convert_loc (loc, TREE_TYPE (lhs), 10175 boolean_true_node), 10176 tlabel, flabel); 10177 gimple_seq_add_stmt (pre_p, cond); 10178 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel)); 10179 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt)); 10180 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel)); 10181 } 10182 10183 10184 /* A subroutine of lower_omp_single. Expand the simple form of 10185 a GIMPLE_OMP_SINGLE, with a copyprivate clause: 10186 10187 #pragma omp single copyprivate (a, b, c) 10188 10189 Create a new structure to hold copies of 'a', 'b' and 'c' and emit: 10190 10191 { 10192 if ((copyout_p = GOMP_single_copy_start ()) == NULL) 10193 { 10194 BODY; 10195 copyout.a = a; 10196 copyout.b = b; 10197 copyout.c = c; 10198 GOMP_single_copy_end (©out); 10199 } 10200 else 10201 { 10202 a = copyout_p->a; 10203 b = copyout_p->b; 10204 c = copyout_p->c; 10205 } 10206 GOMP_barrier (); 10207 } 10208 10209 FIXME. It may be better to delay expanding the logic of this until 10210 pass_expand_omp. The expanded logic may make the job more difficult 10211 to a synchronization analysis pass. */ 10212 10213 static void 10214 lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p, 10215 omp_context *ctx) 10216 { 10217 tree ptr_type, t, l0, l1, l2, bfn_decl; 10218 gimple_seq copyin_seq; 10219 location_t loc = gimple_location (single_stmt); 10220 10221 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o"); 10222 10223 ptr_type = build_pointer_type (ctx->record_type); 10224 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i"); 10225 10226 l0 = create_artificial_label (loc); 10227 l1 = create_artificial_label (loc); 10228 l2 = create_artificial_label (loc); 10229 10230 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START); 10231 t = build_call_expr_loc (loc, bfn_decl, 0); 10232 t = fold_convert_loc (loc, ptr_type, t); 10233 gimplify_assign (ctx->receiver_decl, t, pre_p); 10234 10235 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl, 10236 build_int_cst (ptr_type, 0)); 10237 t = build3 (COND_EXPR, void_type_node, t, 10238 build_and_jump (&l0), build_and_jump (&l1)); 10239 gimplify_and_add (t, pre_p); 10240 10241 gimple_seq_add_stmt (pre_p, gimple_build_label (l0)); 10242 10243 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt)); 10244 10245 copyin_seq = NULL; 10246 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p, 10247 ©in_seq, ctx); 10248 10249 t = build_fold_addr_expr_loc (loc, ctx->sender_decl); 10250 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END); 10251 t = build_call_expr_loc (loc, bfn_decl, 1, t); 10252 gimplify_and_add (t, pre_p); 10253 10254 t = build_and_jump (&l2); 10255 gimplify_and_add (t, pre_p); 10256 10257 gimple_seq_add_stmt (pre_p, gimple_build_label (l1)); 10258 10259 gimple_seq_add_seq (pre_p, copyin_seq); 10260 10261 gimple_seq_add_stmt (pre_p, gimple_build_label (l2)); 10262 } 10263 10264 10265 /* Expand code for an OpenMP single directive. */ 10266 10267 static void 10268 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10269 { 10270 tree block; 10271 gimple t; 10272 gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p)); 10273 gbind *bind; 10274 gimple_seq bind_body, bind_body_tail = NULL, dlist; 10275 10276 push_gimplify_context (); 10277 10278 block = make_node (BLOCK); 10279 bind = gimple_build_bind (NULL, NULL, block); 10280 gsi_replace (gsi_p, bind, true); 10281 bind_body = NULL; 10282 dlist = NULL; 10283 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt), 10284 &bind_body, &dlist, ctx, NULL); 10285 lower_omp (gimple_omp_body_ptr (single_stmt), ctx); 10286 10287 gimple_seq_add_stmt (&bind_body, single_stmt); 10288 10289 if (ctx->record_type) 10290 lower_omp_single_copy (single_stmt, &bind_body, ctx); 10291 else 10292 lower_omp_single_simple (single_stmt, &bind_body); 10293 10294 gimple_omp_set_body (single_stmt, NULL); 10295 10296 gimple_seq_add_seq (&bind_body, dlist); 10297 10298 bind_body = maybe_catch_exception (bind_body); 10299 10300 t = gimple_build_omp_return 10301 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt), 10302 OMP_CLAUSE_NOWAIT)); 10303 gimple_seq_add_stmt (&bind_body_tail, t); 10304 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail); 10305 if (ctx->record_type) 10306 { 10307 gimple_stmt_iterator gsi = gsi_start (bind_body_tail); 10308 tree clobber = build_constructor (ctx->record_type, NULL); 10309 TREE_THIS_VOLATILE (clobber) = 1; 10310 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl, 10311 clobber), GSI_SAME_STMT); 10312 } 10313 gimple_seq_add_seq (&bind_body, bind_body_tail); 10314 gimple_bind_set_body (bind, bind_body); 10315 10316 pop_gimplify_context (bind); 10317 10318 gimple_bind_append_vars (bind, ctx->block_vars); 10319 BLOCK_VARS (block) = ctx->block_vars; 10320 if (BLOCK_VARS (block)) 10321 TREE_USED (block) = 1; 10322 } 10323 10324 10325 /* Expand code for an OpenMP master directive. */ 10326 10327 static void 10328 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10329 { 10330 tree block, lab = NULL, x, bfn_decl; 10331 gimple stmt = gsi_stmt (*gsi_p); 10332 gbind *bind; 10333 location_t loc = gimple_location (stmt); 10334 gimple_seq tseq; 10335 10336 push_gimplify_context (); 10337 10338 block = make_node (BLOCK); 10339 bind = gimple_build_bind (NULL, NULL, block); 10340 gsi_replace (gsi_p, bind, true); 10341 gimple_bind_add_stmt (bind, stmt); 10342 10343 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 10344 x = build_call_expr_loc (loc, bfn_decl, 0); 10345 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node); 10346 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab)); 10347 tseq = NULL; 10348 gimplify_and_add (x, &tseq); 10349 gimple_bind_add_seq (bind, tseq); 10350 10351 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10352 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); 10353 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10354 gimple_omp_set_body (stmt, NULL); 10355 10356 gimple_bind_add_stmt (bind, gimple_build_label (lab)); 10357 10358 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10359 10360 pop_gimplify_context (bind); 10361 10362 gimple_bind_append_vars (bind, ctx->block_vars); 10363 BLOCK_VARS (block) = ctx->block_vars; 10364 } 10365 10366 10367 /* Expand code for an OpenMP taskgroup directive. */ 10368 10369 static void 10370 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10371 { 10372 gimple stmt = gsi_stmt (*gsi_p); 10373 gcall *x; 10374 gbind *bind; 10375 tree block = make_node (BLOCK); 10376 10377 bind = gimple_build_bind (NULL, NULL, block); 10378 gsi_replace (gsi_p, bind, true); 10379 gimple_bind_add_stmt (bind, stmt); 10380 10381 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START), 10382 0); 10383 gimple_bind_add_stmt (bind, x); 10384 10385 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10386 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10387 gimple_omp_set_body (stmt, NULL); 10388 10389 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10390 10391 gimple_bind_append_vars (bind, ctx->block_vars); 10392 BLOCK_VARS (block) = ctx->block_vars; 10393 } 10394 10395 10396 /* Expand code for an OpenMP ordered directive. */ 10397 10398 static void 10399 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10400 { 10401 tree block; 10402 gimple stmt = gsi_stmt (*gsi_p); 10403 gcall *x; 10404 gbind *bind; 10405 10406 push_gimplify_context (); 10407 10408 block = make_node (BLOCK); 10409 bind = gimple_build_bind (NULL, NULL, block); 10410 gsi_replace (gsi_p, bind, true); 10411 gimple_bind_add_stmt (bind, stmt); 10412 10413 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START), 10414 0); 10415 gimple_bind_add_stmt (bind, x); 10416 10417 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10418 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); 10419 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10420 gimple_omp_set_body (stmt, NULL); 10421 10422 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0); 10423 gimple_bind_add_stmt (bind, x); 10424 10425 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10426 10427 pop_gimplify_context (bind); 10428 10429 gimple_bind_append_vars (bind, ctx->block_vars); 10430 BLOCK_VARS (block) = gimple_bind_vars (bind); 10431 } 10432 10433 10434 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple 10435 substitution of a couple of function calls. But in the NAMED case, 10436 requires that languages coordinate a symbol name. It is therefore 10437 best put here in common code. */ 10438 10439 static GTY(()) hash_map<tree, tree> *critical_name_mutexes; 10440 10441 static void 10442 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10443 { 10444 tree block; 10445 tree name, lock, unlock; 10446 gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p)); 10447 gbind *bind; 10448 location_t loc = gimple_location (stmt); 10449 gimple_seq tbody; 10450 10451 name = gimple_omp_critical_name (stmt); 10452 if (name) 10453 { 10454 tree decl; 10455 10456 if (!critical_name_mutexes) 10457 critical_name_mutexes = hash_map<tree, tree>::create_ggc (10); 10458 10459 tree *n = critical_name_mutexes->get (name); 10460 if (n == NULL) 10461 { 10462 char *new_str; 10463 10464 decl = create_tmp_var_raw (ptr_type_node); 10465 10466 new_str = ACONCAT ((".gomp_critical_user_", 10467 IDENTIFIER_POINTER (name), NULL)); 10468 DECL_NAME (decl) = get_identifier (new_str); 10469 TREE_PUBLIC (decl) = 1; 10470 TREE_STATIC (decl) = 1; 10471 DECL_COMMON (decl) = 1; 10472 DECL_ARTIFICIAL (decl) = 1; 10473 DECL_IGNORED_P (decl) = 1; 10474 10475 varpool_node::finalize_decl (decl); 10476 10477 critical_name_mutexes->put (name, decl); 10478 } 10479 else 10480 decl = *n; 10481 10482 /* If '#pragma omp critical' is inside offloaded region or 10483 inside function marked as offloadable, the symbol must be 10484 marked as offloadable too. */ 10485 omp_context *octx; 10486 if (cgraph_node::get (current_function_decl)->offloadable) 10487 varpool_node::get_create (decl)->offloadable = 1; 10488 else 10489 for (octx = ctx->outer; octx; octx = octx->outer) 10490 if (is_gimple_omp_offloaded (octx->stmt)) 10491 { 10492 varpool_node::get_create (decl)->offloadable = 1; 10493 break; 10494 } 10495 10496 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START); 10497 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl)); 10498 10499 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END); 10500 unlock = build_call_expr_loc (loc, unlock, 1, 10501 build_fold_addr_expr_loc (loc, decl)); 10502 } 10503 else 10504 { 10505 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START); 10506 lock = build_call_expr_loc (loc, lock, 0); 10507 10508 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END); 10509 unlock = build_call_expr_loc (loc, unlock, 0); 10510 } 10511 10512 push_gimplify_context (); 10513 10514 block = make_node (BLOCK); 10515 bind = gimple_build_bind (NULL, NULL, block); 10516 gsi_replace (gsi_p, bind, true); 10517 gimple_bind_add_stmt (bind, stmt); 10518 10519 tbody = gimple_bind_body (bind); 10520 gimplify_and_add (lock, &tbody); 10521 gimple_bind_set_body (bind, tbody); 10522 10523 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10524 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); 10525 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10526 gimple_omp_set_body (stmt, NULL); 10527 10528 tbody = gimple_bind_body (bind); 10529 gimplify_and_add (unlock, &tbody); 10530 gimple_bind_set_body (bind, tbody); 10531 10532 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10533 10534 pop_gimplify_context (bind); 10535 gimple_bind_append_vars (bind, ctx->block_vars); 10536 BLOCK_VARS (block) = gimple_bind_vars (bind); 10537 } 10538 10539 10540 /* A subroutine of lower_omp_for. Generate code to emit the predicate 10541 for a lastprivate clause. Given a loop control predicate of (V 10542 cond N2), we gate the clause on (!(V cond N2)). The lowered form 10543 is appended to *DLIST, iterator initialization is appended to 10544 *BODY_P. */ 10545 10546 static void 10547 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p, 10548 gimple_seq *dlist, struct omp_context *ctx) 10549 { 10550 tree clauses, cond, vinit; 10551 enum tree_code cond_code; 10552 gimple_seq stmts; 10553 10554 cond_code = fd->loop.cond_code; 10555 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR; 10556 10557 /* When possible, use a strict equality expression. This can let VRP 10558 type optimizations deduce the value and remove a copy. */ 10559 if (tree_fits_shwi_p (fd->loop.step)) 10560 { 10561 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step); 10562 if (step == 1 || step == -1) 10563 cond_code = EQ_EXPR; 10564 } 10565 10566 tree n2 = fd->loop.n2; 10567 if (fd->collapse > 1 10568 && TREE_CODE (n2) != INTEGER_CST 10569 && gimple_omp_for_combined_into_p (fd->for_stmt) 10570 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR) 10571 { 10572 gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt); 10573 if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR) 10574 { 10575 struct omp_for_data outer_fd; 10576 extract_omp_for_data (gfor, &outer_fd, NULL); 10577 n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2); 10578 } 10579 } 10580 cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2); 10581 10582 clauses = gimple_omp_for_clauses (fd->for_stmt); 10583 stmts = NULL; 10584 lower_lastprivate_clauses (clauses, cond, &stmts, ctx); 10585 if (!gimple_seq_empty_p (stmts)) 10586 { 10587 gimple_seq_add_seq (&stmts, *dlist); 10588 *dlist = stmts; 10589 10590 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */ 10591 vinit = fd->loop.n1; 10592 if (cond_code == EQ_EXPR 10593 && tree_fits_shwi_p (fd->loop.n2) 10594 && ! integer_zerop (fd->loop.n2)) 10595 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0); 10596 else 10597 vinit = unshare_expr (vinit); 10598 10599 /* Initialize the iterator variable, so that threads that don't execute 10600 any iterations don't execute the lastprivate clauses by accident. */ 10601 gimplify_assign (fd->loop.v, vinit, body_p); 10602 } 10603 } 10604 10605 10606 /* Lower code for an OMP loop directive. */ 10607 10608 static void 10609 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10610 { 10611 tree *rhs_p, block; 10612 struct omp_for_data fd, *fdp = NULL; 10613 gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p)); 10614 gbind *new_stmt; 10615 gimple_seq omp_for_body, body, dlist; 10616 size_t i; 10617 10618 push_gimplify_context (); 10619 10620 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx); 10621 10622 block = make_node (BLOCK); 10623 new_stmt = gimple_build_bind (NULL, NULL, block); 10624 /* Replace at gsi right away, so that 'stmt' is no member 10625 of a sequence anymore as we're going to add to to a different 10626 one below. */ 10627 gsi_replace (gsi_p, new_stmt, true); 10628 10629 /* Move declaration of temporaries in the loop body before we make 10630 it go away. */ 10631 omp_for_body = gimple_omp_body (stmt); 10632 if (!gimple_seq_empty_p (omp_for_body) 10633 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND) 10634 { 10635 gbind *inner_bind 10636 = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body)); 10637 tree vars = gimple_bind_vars (inner_bind); 10638 gimple_bind_append_vars (new_stmt, vars); 10639 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't 10640 keep them on the inner_bind and it's block. */ 10641 gimple_bind_set_vars (inner_bind, NULL_TREE); 10642 if (gimple_bind_block (inner_bind)) 10643 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE; 10644 } 10645 10646 if (gimple_omp_for_combined_into_p (stmt)) 10647 { 10648 extract_omp_for_data (stmt, &fd, NULL); 10649 fdp = &fd; 10650 10651 /* We need two temporaries with fd.loop.v type (istart/iend) 10652 and then (fd.collapse - 1) temporaries with the same 10653 type for count2 ... countN-1 vars if not constant. */ 10654 size_t count = 2; 10655 tree type = fd.iter_type; 10656 if (fd.collapse > 1 10657 && TREE_CODE (fd.loop.n2) != INTEGER_CST) 10658 count += fd.collapse - 1; 10659 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR; 10660 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt); 10661 tree clauses = *pc; 10662 if (parallel_for) 10663 outerc 10664 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt), 10665 OMP_CLAUSE__LOOPTEMP_); 10666 for (i = 0; i < count; i++) 10667 { 10668 tree temp; 10669 if (parallel_for) 10670 { 10671 gcc_assert (outerc); 10672 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer); 10673 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc), 10674 OMP_CLAUSE__LOOPTEMP_); 10675 } 10676 else 10677 { 10678 temp = create_tmp_var (type); 10679 insert_decl_map (&ctx->outer->cb, temp, temp); 10680 } 10681 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_); 10682 OMP_CLAUSE_DECL (*pc) = temp; 10683 pc = &OMP_CLAUSE_CHAIN (*pc); 10684 } 10685 *pc = clauses; 10686 } 10687 10688 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */ 10689 dlist = NULL; 10690 body = NULL; 10691 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx, 10692 fdp); 10693 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt)); 10694 10695 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10696 10697 /* Lower the header expressions. At this point, we can assume that 10698 the header is of the form: 10699 10700 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3) 10701 10702 We just need to make sure that VAL1, VAL2 and VAL3 are lowered 10703 using the .omp_data_s mapping, if needed. */ 10704 for (i = 0; i < gimple_omp_for_collapse (stmt); i++) 10705 { 10706 rhs_p = gimple_omp_for_initial_ptr (stmt, i); 10707 if (!is_gimple_min_invariant (*rhs_p)) 10708 *rhs_p = get_formal_tmp_var (*rhs_p, &body); 10709 10710 rhs_p = gimple_omp_for_final_ptr (stmt, i); 10711 if (!is_gimple_min_invariant (*rhs_p)) 10712 *rhs_p = get_formal_tmp_var (*rhs_p, &body); 10713 10714 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1); 10715 if (!is_gimple_min_invariant (*rhs_p)) 10716 *rhs_p = get_formal_tmp_var (*rhs_p, &body); 10717 } 10718 10719 /* Once lowered, extract the bounds and clauses. */ 10720 extract_omp_for_data (stmt, &fd, NULL); 10721 10722 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx); 10723 10724 gimple_seq_add_stmt (&body, stmt); 10725 gimple_seq_add_seq (&body, gimple_omp_body (stmt)); 10726 10727 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v, 10728 fd.loop.v)); 10729 10730 /* After the loop, add exit clauses. */ 10731 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx); 10732 10733 if (ctx->cancellable) 10734 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label)); 10735 10736 gimple_seq_add_seq (&body, dlist); 10737 10738 body = maybe_catch_exception (body); 10739 10740 /* Region exit marker goes at the end of the loop body. */ 10741 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait)); 10742 maybe_add_implicit_barrier_cancel (ctx, &body); 10743 pop_gimplify_context (new_stmt); 10744 10745 gimple_bind_append_vars (new_stmt, ctx->block_vars); 10746 BLOCK_VARS (block) = gimple_bind_vars (new_stmt); 10747 if (BLOCK_VARS (block)) 10748 TREE_USED (block) = 1; 10749 10750 gimple_bind_set_body (new_stmt, body); 10751 gimple_omp_set_body (stmt, NULL); 10752 gimple_omp_for_set_pre_body (stmt, NULL); 10753 } 10754 10755 /* Callback for walk_stmts. Check if the current statement only contains 10756 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */ 10757 10758 static tree 10759 check_combined_parallel (gimple_stmt_iterator *gsi_p, 10760 bool *handled_ops_p, 10761 struct walk_stmt_info *wi) 10762 { 10763 int *info = (int *) wi->info; 10764 gimple stmt = gsi_stmt (*gsi_p); 10765 10766 *handled_ops_p = true; 10767 switch (gimple_code (stmt)) 10768 { 10769 WALK_SUBSTMTS; 10770 10771 case GIMPLE_OMP_FOR: 10772 case GIMPLE_OMP_SECTIONS: 10773 *info = *info == 0 ? 1 : -1; 10774 break; 10775 default: 10776 *info = -1; 10777 break; 10778 } 10779 return NULL; 10780 } 10781 10782 struct omp_taskcopy_context 10783 { 10784 /* This field must be at the beginning, as we do "inheritance": Some 10785 callback functions for tree-inline.c (e.g., omp_copy_decl) 10786 receive a copy_body_data pointer that is up-casted to an 10787 omp_context pointer. */ 10788 copy_body_data cb; 10789 omp_context *ctx; 10790 }; 10791 10792 static tree 10793 task_copyfn_copy_decl (tree var, copy_body_data *cb) 10794 { 10795 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb; 10796 10797 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var)) 10798 return create_tmp_var (TREE_TYPE (var)); 10799 10800 return var; 10801 } 10802 10803 static tree 10804 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type) 10805 { 10806 tree name, new_fields = NULL, type, f; 10807 10808 type = lang_hooks.types.make_type (RECORD_TYPE); 10809 name = DECL_NAME (TYPE_NAME (orig_type)); 10810 name = build_decl (gimple_location (tcctx->ctx->stmt), 10811 TYPE_DECL, name, type); 10812 TYPE_NAME (type) = name; 10813 10814 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f)) 10815 { 10816 tree new_f = copy_node (f); 10817 DECL_CONTEXT (new_f) = type; 10818 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb); 10819 TREE_CHAIN (new_f) = new_fields; 10820 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL); 10821 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL); 10822 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, 10823 &tcctx->cb, NULL); 10824 new_fields = new_f; 10825 tcctx->cb.decl_map->put (f, new_f); 10826 } 10827 TYPE_FIELDS (type) = nreverse (new_fields); 10828 layout_type (type); 10829 return type; 10830 } 10831 10832 /* Create task copyfn. */ 10833 10834 static void 10835 create_task_copyfn (gomp_task *task_stmt, omp_context *ctx) 10836 { 10837 struct function *child_cfun; 10838 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl; 10839 tree record_type, srecord_type, bind, list; 10840 bool record_needs_remap = false, srecord_needs_remap = false; 10841 splay_tree_node n; 10842 struct omp_taskcopy_context tcctx; 10843 location_t loc = gimple_location (task_stmt); 10844 10845 child_fn = gimple_omp_task_copy_fn (task_stmt); 10846 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 10847 gcc_assert (child_cfun->cfg == NULL); 10848 DECL_SAVED_TREE (child_fn) = alloc_stmt_list (); 10849 10850 /* Reset DECL_CONTEXT on function arguments. */ 10851 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) 10852 DECL_CONTEXT (t) = child_fn; 10853 10854 /* Populate the function. */ 10855 push_gimplify_context (); 10856 push_cfun (child_cfun); 10857 10858 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL); 10859 TREE_SIDE_EFFECTS (bind) = 1; 10860 list = NULL; 10861 DECL_SAVED_TREE (child_fn) = bind; 10862 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt); 10863 10864 /* Remap src and dst argument types if needed. */ 10865 record_type = ctx->record_type; 10866 srecord_type = ctx->srecord_type; 10867 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) 10868 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) 10869 { 10870 record_needs_remap = true; 10871 break; 10872 } 10873 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f)) 10874 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) 10875 { 10876 srecord_needs_remap = true; 10877 break; 10878 } 10879 10880 if (record_needs_remap || srecord_needs_remap) 10881 { 10882 memset (&tcctx, '\0', sizeof (tcctx)); 10883 tcctx.cb.src_fn = ctx->cb.src_fn; 10884 tcctx.cb.dst_fn = child_fn; 10885 tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn); 10886 gcc_checking_assert (tcctx.cb.src_node); 10887 tcctx.cb.dst_node = tcctx.cb.src_node; 10888 tcctx.cb.src_cfun = ctx->cb.src_cfun; 10889 tcctx.cb.copy_decl = task_copyfn_copy_decl; 10890 tcctx.cb.eh_lp_nr = 0; 10891 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE; 10892 tcctx.cb.decl_map = new hash_map<tree, tree>; 10893 tcctx.ctx = ctx; 10894 10895 if (record_needs_remap) 10896 record_type = task_copyfn_remap_type (&tcctx, record_type); 10897 if (srecord_needs_remap) 10898 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type); 10899 } 10900 else 10901 tcctx.cb.decl_map = NULL; 10902 10903 arg = DECL_ARGUMENTS (child_fn); 10904 TREE_TYPE (arg) = build_pointer_type (record_type); 10905 sarg = DECL_CHAIN (arg); 10906 TREE_TYPE (sarg) = build_pointer_type (srecord_type); 10907 10908 /* First pass: initialize temporaries used in record_type and srecord_type 10909 sizes and field offsets. */ 10910 if (tcctx.cb.decl_map) 10911 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) 10912 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) 10913 { 10914 tree *p; 10915 10916 decl = OMP_CLAUSE_DECL (c); 10917 p = tcctx.cb.decl_map->get (decl); 10918 if (p == NULL) 10919 continue; 10920 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10921 sf = (tree) n->value; 10922 sf = *tcctx.cb.decl_map->get (sf); 10923 src = build_simple_mem_ref_loc (loc, sarg); 10924 src = omp_build_component_ref (src, sf); 10925 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src); 10926 append_to_statement_list (t, &list); 10927 } 10928 10929 /* Second pass: copy shared var pointers and copy construct non-VLA 10930 firstprivate vars. */ 10931 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) 10932 switch (OMP_CLAUSE_CODE (c)) 10933 { 10934 case OMP_CLAUSE_SHARED: 10935 decl = OMP_CLAUSE_DECL (c); 10936 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 10937 if (n == NULL) 10938 break; 10939 f = (tree) n->value; 10940 if (tcctx.cb.decl_map) 10941 f = *tcctx.cb.decl_map->get (f); 10942 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10943 sf = (tree) n->value; 10944 if (tcctx.cb.decl_map) 10945 sf = *tcctx.cb.decl_map->get (sf); 10946 src = build_simple_mem_ref_loc (loc, sarg); 10947 src = omp_build_component_ref (src, sf); 10948 dst = build_simple_mem_ref_loc (loc, arg); 10949 dst = omp_build_component_ref (dst, f); 10950 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); 10951 append_to_statement_list (t, &list); 10952 break; 10953 case OMP_CLAUSE_FIRSTPRIVATE: 10954 decl = OMP_CLAUSE_DECL (c); 10955 if (is_variable_sized (decl)) 10956 break; 10957 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 10958 if (n == NULL) 10959 break; 10960 f = (tree) n->value; 10961 if (tcctx.cb.decl_map) 10962 f = *tcctx.cb.decl_map->get (f); 10963 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10964 if (n != NULL) 10965 { 10966 sf = (tree) n->value; 10967 if (tcctx.cb.decl_map) 10968 sf = *tcctx.cb.decl_map->get (sf); 10969 src = build_simple_mem_ref_loc (loc, sarg); 10970 src = omp_build_component_ref (src, sf); 10971 if (use_pointer_for_field (decl, NULL) || is_reference (decl)) 10972 src = build_simple_mem_ref_loc (loc, src); 10973 } 10974 else 10975 src = decl; 10976 dst = build_simple_mem_ref_loc (loc, arg); 10977 dst = omp_build_component_ref (dst, f); 10978 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); 10979 append_to_statement_list (t, &list); 10980 break; 10981 case OMP_CLAUSE_PRIVATE: 10982 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 10983 break; 10984 decl = OMP_CLAUSE_DECL (c); 10985 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 10986 f = (tree) n->value; 10987 if (tcctx.cb.decl_map) 10988 f = *tcctx.cb.decl_map->get (f); 10989 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10990 if (n != NULL) 10991 { 10992 sf = (tree) n->value; 10993 if (tcctx.cb.decl_map) 10994 sf = *tcctx.cb.decl_map->get (sf); 10995 src = build_simple_mem_ref_loc (loc, sarg); 10996 src = omp_build_component_ref (src, sf); 10997 if (use_pointer_for_field (decl, NULL)) 10998 src = build_simple_mem_ref_loc (loc, src); 10999 } 11000 else 11001 src = decl; 11002 dst = build_simple_mem_ref_loc (loc, arg); 11003 dst = omp_build_component_ref (dst, f); 11004 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); 11005 append_to_statement_list (t, &list); 11006 break; 11007 default: 11008 break; 11009 } 11010 11011 /* Last pass: handle VLA firstprivates. */ 11012 if (tcctx.cb.decl_map) 11013 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) 11014 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) 11015 { 11016 tree ind, ptr, df; 11017 11018 decl = OMP_CLAUSE_DECL (c); 11019 if (!is_variable_sized (decl)) 11020 continue; 11021 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 11022 if (n == NULL) 11023 continue; 11024 f = (tree) n->value; 11025 f = *tcctx.cb.decl_map->get (f); 11026 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl)); 11027 ind = DECL_VALUE_EXPR (decl); 11028 gcc_assert (TREE_CODE (ind) == INDIRECT_REF); 11029 gcc_assert (DECL_P (TREE_OPERAND (ind, 0))); 11030 n = splay_tree_lookup (ctx->sfield_map, 11031 (splay_tree_key) TREE_OPERAND (ind, 0)); 11032 sf = (tree) n->value; 11033 sf = *tcctx.cb.decl_map->get (sf); 11034 src = build_simple_mem_ref_loc (loc, sarg); 11035 src = omp_build_component_ref (src, sf); 11036 src = build_simple_mem_ref_loc (loc, src); 11037 dst = build_simple_mem_ref_loc (loc, arg); 11038 dst = omp_build_component_ref (dst, f); 11039 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); 11040 append_to_statement_list (t, &list); 11041 n = splay_tree_lookup (ctx->field_map, 11042 (splay_tree_key) TREE_OPERAND (ind, 0)); 11043 df = (tree) n->value; 11044 df = *tcctx.cb.decl_map->get (df); 11045 ptr = build_simple_mem_ref_loc (loc, arg); 11046 ptr = omp_build_component_ref (ptr, df); 11047 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr, 11048 build_fold_addr_expr_loc (loc, dst)); 11049 append_to_statement_list (t, &list); 11050 } 11051 11052 t = build1 (RETURN_EXPR, void_type_node, NULL); 11053 append_to_statement_list (t, &list); 11054 11055 if (tcctx.cb.decl_map) 11056 delete tcctx.cb.decl_map; 11057 pop_gimplify_context (NULL); 11058 BIND_EXPR_BODY (bind) = list; 11059 pop_cfun (); 11060 } 11061 11062 static void 11063 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq) 11064 { 11065 tree c, clauses; 11066 gimple g; 11067 size_t n_in = 0, n_out = 0, idx = 2, i; 11068 11069 clauses = find_omp_clause (gimple_omp_task_clauses (stmt), 11070 OMP_CLAUSE_DEPEND); 11071 gcc_assert (clauses); 11072 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 11073 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND) 11074 switch (OMP_CLAUSE_DEPEND_KIND (c)) 11075 { 11076 case OMP_CLAUSE_DEPEND_IN: 11077 n_in++; 11078 break; 11079 case OMP_CLAUSE_DEPEND_OUT: 11080 case OMP_CLAUSE_DEPEND_INOUT: 11081 n_out++; 11082 break; 11083 default: 11084 gcc_unreachable (); 11085 } 11086 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2); 11087 tree array = create_tmp_var (type); 11088 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE, 11089 NULL_TREE); 11090 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out)); 11091 gimple_seq_add_stmt (iseq, g); 11092 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE, 11093 NULL_TREE); 11094 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out)); 11095 gimple_seq_add_stmt (iseq, g); 11096 for (i = 0; i < 2; i++) 11097 { 11098 if ((i ? n_in : n_out) == 0) 11099 continue; 11100 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 11101 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND 11102 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i)) 11103 { 11104 tree t = OMP_CLAUSE_DECL (c); 11105 t = fold_convert (ptr_type_node, t); 11106 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue); 11107 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++), 11108 NULL_TREE, NULL_TREE); 11109 g = gimple_build_assign (r, t); 11110 gimple_seq_add_stmt (iseq, g); 11111 } 11112 } 11113 tree *p = gimple_omp_task_clauses_ptr (stmt); 11114 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND); 11115 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array); 11116 OMP_CLAUSE_CHAIN (c) = *p; 11117 *p = c; 11118 tree clobber = build_constructor (type, NULL); 11119 TREE_THIS_VOLATILE (clobber) = 1; 11120 g = gimple_build_assign (array, clobber); 11121 gimple_seq_add_stmt (oseq, g); 11122 } 11123 11124 /* Lower the OpenMP parallel or task directive in the current statement 11125 in GSI_P. CTX holds context information for the directive. */ 11126 11127 static void 11128 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11129 { 11130 tree clauses; 11131 tree child_fn, t; 11132 gimple stmt = gsi_stmt (*gsi_p); 11133 gbind *par_bind, *bind, *dep_bind = NULL; 11134 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body; 11135 location_t loc = gimple_location (stmt); 11136 11137 clauses = gimple_omp_taskreg_clauses (stmt); 11138 par_bind 11139 = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt))); 11140 par_body = gimple_bind_body (par_bind); 11141 child_fn = ctx->cb.dst_fn; 11142 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL 11143 && !gimple_omp_parallel_combined_p (stmt)) 11144 { 11145 struct walk_stmt_info wi; 11146 int ws_num = 0; 11147 11148 memset (&wi, 0, sizeof (wi)); 11149 wi.info = &ws_num; 11150 wi.val_only = true; 11151 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi); 11152 if (ws_num == 1) 11153 gimple_omp_parallel_set_combined_p (stmt, true); 11154 } 11155 gimple_seq dep_ilist = NULL; 11156 gimple_seq dep_olist = NULL; 11157 if (gimple_code (stmt) == GIMPLE_OMP_TASK 11158 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND)) 11159 { 11160 push_gimplify_context (); 11161 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); 11162 lower_depend_clauses (stmt, &dep_ilist, &dep_olist); 11163 } 11164 11165 if (ctx->srecord_type) 11166 create_task_copyfn (as_a <gomp_task *> (stmt), ctx); 11167 11168 push_gimplify_context (); 11169 11170 par_olist = NULL; 11171 par_ilist = NULL; 11172 par_rlist = NULL; 11173 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL); 11174 lower_omp (&par_body, ctx); 11175 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL) 11176 lower_reduction_clauses (clauses, &par_rlist, ctx); 11177 11178 /* Declare all the variables created by mapping and the variables 11179 declared in the scope of the parallel body. */ 11180 record_vars_into (ctx->block_vars, child_fn); 11181 record_vars_into (gimple_bind_vars (par_bind), child_fn); 11182 11183 if (ctx->record_type) 11184 { 11185 ctx->sender_decl 11186 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type 11187 : ctx->record_type, ".omp_data_o"); 11188 DECL_NAMELESS (ctx->sender_decl) = 1; 11189 TREE_ADDRESSABLE (ctx->sender_decl) = 1; 11190 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl); 11191 } 11192 11193 olist = NULL; 11194 ilist = NULL; 11195 lower_send_clauses (clauses, &ilist, &olist, ctx); 11196 lower_send_shared_vars (&ilist, &olist, ctx); 11197 11198 if (ctx->record_type) 11199 { 11200 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL); 11201 TREE_THIS_VOLATILE (clobber) = 1; 11202 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, 11203 clobber)); 11204 } 11205 11206 /* Once all the expansions are done, sequence all the different 11207 fragments inside gimple_omp_body. */ 11208 11209 new_body = NULL; 11210 11211 if (ctx->record_type) 11212 { 11213 t = build_fold_addr_expr_loc (loc, ctx->sender_decl); 11214 /* fixup_child_record_type might have changed receiver_decl's type. */ 11215 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); 11216 gimple_seq_add_stmt (&new_body, 11217 gimple_build_assign (ctx->receiver_decl, t)); 11218 } 11219 11220 gimple_seq_add_seq (&new_body, par_ilist); 11221 gimple_seq_add_seq (&new_body, par_body); 11222 gimple_seq_add_seq (&new_body, par_rlist); 11223 if (ctx->cancellable) 11224 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label)); 11225 gimple_seq_add_seq (&new_body, par_olist); 11226 new_body = maybe_catch_exception (new_body); 11227 if (gimple_code (stmt) == GIMPLE_OMP_TASK) 11228 gimple_seq_add_stmt (&new_body, 11229 gimple_build_omp_continue (integer_zero_node, 11230 integer_zero_node)); 11231 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); 11232 gimple_omp_set_body (stmt, new_body); 11233 11234 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind)); 11235 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true); 11236 gimple_bind_add_seq (bind, ilist); 11237 gimple_bind_add_stmt (bind, stmt); 11238 gimple_bind_add_seq (bind, olist); 11239 11240 pop_gimplify_context (NULL); 11241 11242 if (dep_bind) 11243 { 11244 gimple_bind_add_seq (dep_bind, dep_ilist); 11245 gimple_bind_add_stmt (dep_bind, bind); 11246 gimple_bind_add_seq (dep_bind, dep_olist); 11247 pop_gimplify_context (dep_bind); 11248 } 11249 } 11250 11251 /* Lower the GIMPLE_OMP_TARGET in the current statement 11252 in GSI_P. CTX holds context information for the directive. */ 11253 11254 static void 11255 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11256 { 11257 tree clauses; 11258 tree child_fn, t, c; 11259 gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p)); 11260 gbind *tgt_bind, *bind; 11261 gimple_seq tgt_body, olist, ilist, orlist, irlist, new_body; 11262 location_t loc = gimple_location (stmt); 11263 bool offloaded, data_region; 11264 unsigned int map_cnt = 0; 11265 11266 offloaded = is_gimple_omp_offloaded (stmt); 11267 switch (gimple_omp_target_kind (stmt)) 11268 { 11269 case GF_OMP_TARGET_KIND_REGION: 11270 case GF_OMP_TARGET_KIND_UPDATE: 11271 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 11272 case GF_OMP_TARGET_KIND_OACC_KERNELS: 11273 case GF_OMP_TARGET_KIND_OACC_UPDATE: 11274 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 11275 data_region = false; 11276 break; 11277 case GF_OMP_TARGET_KIND_DATA: 11278 case GF_OMP_TARGET_KIND_OACC_DATA: 11279 data_region = true; 11280 break; 11281 default: 11282 gcc_unreachable (); 11283 } 11284 11285 clauses = gimple_omp_target_clauses (stmt); 11286 11287 tgt_bind = NULL; 11288 tgt_body = NULL; 11289 if (offloaded) 11290 { 11291 tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt)); 11292 tgt_body = gimple_bind_body (tgt_bind); 11293 } 11294 else if (data_region) 11295 tgt_body = gimple_omp_body (stmt); 11296 child_fn = ctx->cb.dst_fn; 11297 11298 push_gimplify_context (); 11299 11300 irlist = NULL; 11301 orlist = NULL; 11302 if (offloaded 11303 && is_gimple_omp_oacc (stmt)) 11304 oacc_process_reduction_data (&tgt_body, &irlist, &orlist, ctx); 11305 11306 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 11307 switch (OMP_CLAUSE_CODE (c)) 11308 { 11309 tree var, x; 11310 11311 default: 11312 break; 11313 case OMP_CLAUSE_MAP: 11314 #ifdef ENABLE_CHECKING 11315 /* First check what we're prepared to handle in the following. */ 11316 switch (OMP_CLAUSE_MAP_KIND (c)) 11317 { 11318 case GOMP_MAP_ALLOC: 11319 case GOMP_MAP_TO: 11320 case GOMP_MAP_FROM: 11321 case GOMP_MAP_TOFROM: 11322 case GOMP_MAP_POINTER: 11323 case GOMP_MAP_TO_PSET: 11324 break; 11325 case GOMP_MAP_FORCE_ALLOC: 11326 case GOMP_MAP_FORCE_TO: 11327 case GOMP_MAP_FORCE_FROM: 11328 case GOMP_MAP_FORCE_TOFROM: 11329 case GOMP_MAP_FORCE_PRESENT: 11330 case GOMP_MAP_FORCE_DEALLOC: 11331 case GOMP_MAP_FORCE_DEVICEPTR: 11332 gcc_assert (is_gimple_omp_oacc (stmt)); 11333 break; 11334 default: 11335 gcc_unreachable (); 11336 } 11337 #endif 11338 /* FALLTHRU */ 11339 case OMP_CLAUSE_TO: 11340 case OMP_CLAUSE_FROM: 11341 var = OMP_CLAUSE_DECL (c); 11342 if (!DECL_P (var)) 11343 { 11344 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP 11345 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) 11346 map_cnt++; 11347 continue; 11348 } 11349 11350 if (DECL_SIZE (var) 11351 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST) 11352 { 11353 tree var2 = DECL_VALUE_EXPR (var); 11354 gcc_assert (TREE_CODE (var2) == INDIRECT_REF); 11355 var2 = TREE_OPERAND (var2, 0); 11356 gcc_assert (DECL_P (var2)); 11357 var = var2; 11358 } 11359 11360 if (!maybe_lookup_field (var, ctx)) 11361 continue; 11362 11363 if (offloaded) 11364 { 11365 x = build_receiver_ref (var, true, ctx); 11366 tree new_var = lookup_decl (var, ctx); 11367 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 11368 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) 11369 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) 11370 x = build_simple_mem_ref (x); 11371 SET_DECL_VALUE_EXPR (new_var, x); 11372 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 11373 } 11374 map_cnt++; 11375 } 11376 11377 if (offloaded) 11378 { 11379 target_nesting_level++; 11380 lower_omp (&tgt_body, ctx); 11381 target_nesting_level--; 11382 } 11383 else if (data_region) 11384 lower_omp (&tgt_body, ctx); 11385 11386 if (offloaded) 11387 { 11388 /* Declare all the variables created by mapping and the variables 11389 declared in the scope of the target body. */ 11390 record_vars_into (ctx->block_vars, child_fn); 11391 record_vars_into (gimple_bind_vars (tgt_bind), child_fn); 11392 } 11393 11394 olist = NULL; 11395 ilist = NULL; 11396 if (ctx->record_type) 11397 { 11398 ctx->sender_decl 11399 = create_tmp_var (ctx->record_type, ".omp_data_arr"); 11400 DECL_NAMELESS (ctx->sender_decl) = 1; 11401 TREE_ADDRESSABLE (ctx->sender_decl) = 1; 11402 t = make_tree_vec (3); 11403 TREE_VEC_ELT (t, 0) = ctx->sender_decl; 11404 TREE_VEC_ELT (t, 1) 11405 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt), 11406 ".omp_data_sizes"); 11407 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1; 11408 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1; 11409 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1; 11410 tree tkind_type; 11411 int talign_shift; 11412 if (is_gimple_omp_oacc (stmt)) 11413 { 11414 tkind_type = short_unsigned_type_node; 11415 talign_shift = 8; 11416 } 11417 else 11418 { 11419 tkind_type = unsigned_char_type_node; 11420 talign_shift = 3; 11421 } 11422 TREE_VEC_ELT (t, 2) 11423 = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt), 11424 ".omp_data_kinds"); 11425 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1; 11426 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1; 11427 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1; 11428 gimple_omp_target_set_data_arg (stmt, t); 11429 11430 vec<constructor_elt, va_gc> *vsize; 11431 vec<constructor_elt, va_gc> *vkind; 11432 vec_alloc (vsize, map_cnt); 11433 vec_alloc (vkind, map_cnt); 11434 unsigned int map_idx = 0; 11435 11436 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 11437 switch (OMP_CLAUSE_CODE (c)) 11438 { 11439 tree ovar, nc; 11440 11441 default: 11442 break; 11443 case OMP_CLAUSE_MAP: 11444 case OMP_CLAUSE_TO: 11445 case OMP_CLAUSE_FROM: 11446 nc = c; 11447 ovar = OMP_CLAUSE_DECL (c); 11448 if (!DECL_P (ovar)) 11449 { 11450 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 11451 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) 11452 { 11453 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c)) 11454 == get_base_address (ovar)); 11455 nc = OMP_CLAUSE_CHAIN (c); 11456 ovar = OMP_CLAUSE_DECL (nc); 11457 } 11458 else 11459 { 11460 tree x = build_sender_ref (ovar, ctx); 11461 tree v 11462 = build_fold_addr_expr_with_type (ovar, ptr_type_node); 11463 gimplify_assign (x, v, &ilist); 11464 nc = NULL_TREE; 11465 } 11466 } 11467 else 11468 { 11469 if (DECL_SIZE (ovar) 11470 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST) 11471 { 11472 tree ovar2 = DECL_VALUE_EXPR (ovar); 11473 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF); 11474 ovar2 = TREE_OPERAND (ovar2, 0); 11475 gcc_assert (DECL_P (ovar2)); 11476 ovar = ovar2; 11477 } 11478 if (!maybe_lookup_field (ovar, ctx)) 11479 continue; 11480 } 11481 11482 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar)); 11483 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign) 11484 talign = DECL_ALIGN_UNIT (ovar); 11485 if (nc) 11486 { 11487 tree var = lookup_decl_in_outer_ctx (ovar, ctx); 11488 tree x = build_sender_ref (ovar, ctx); 11489 if (maybe_lookup_oacc_reduction (var, ctx)) 11490 { 11491 gcc_checking_assert (offloaded 11492 && is_gimple_omp_oacc (stmt)); 11493 gimplify_assign (x, var, &ilist); 11494 } 11495 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 11496 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 11497 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) 11498 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE) 11499 { 11500 gcc_assert (offloaded); 11501 tree avar 11502 = create_tmp_var (TREE_TYPE (TREE_TYPE (x))); 11503 mark_addressable (avar); 11504 gimplify_assign (avar, build_fold_addr_expr (var), &ilist); 11505 talign = DECL_ALIGN_UNIT (avar); 11506 avar = build_fold_addr_expr (avar); 11507 gimplify_assign (x, avar, &ilist); 11508 } 11509 else if (is_gimple_reg (var)) 11510 { 11511 gcc_assert (offloaded); 11512 tree avar = create_tmp_var (TREE_TYPE (var)); 11513 mark_addressable (avar); 11514 enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c); 11515 if (GOMP_MAP_COPY_TO_P (map_kind) 11516 || map_kind == GOMP_MAP_POINTER 11517 || map_kind == GOMP_MAP_TO_PSET 11518 || map_kind == GOMP_MAP_FORCE_DEVICEPTR) 11519 gimplify_assign (avar, var, &ilist); 11520 avar = build_fold_addr_expr (avar); 11521 gimplify_assign (x, avar, &ilist); 11522 if ((GOMP_MAP_COPY_FROM_P (map_kind) 11523 || map_kind == GOMP_MAP_FORCE_DEVICEPTR) 11524 && !TYPE_READONLY (TREE_TYPE (var))) 11525 { 11526 x = build_sender_ref (ovar, ctx); 11527 x = build_simple_mem_ref (x); 11528 gimplify_assign (var, x, &olist); 11529 } 11530 } 11531 else 11532 { 11533 var = build_fold_addr_expr (var); 11534 gimplify_assign (x, var, &ilist); 11535 } 11536 } 11537 tree s = OMP_CLAUSE_SIZE (c); 11538 if (s == NULL_TREE) 11539 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar)); 11540 s = fold_convert (size_type_node, s); 11541 tree purpose = size_int (map_idx++); 11542 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s); 11543 if (TREE_CODE (s) != INTEGER_CST) 11544 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0; 11545 11546 unsigned HOST_WIDE_INT tkind; 11547 switch (OMP_CLAUSE_CODE (c)) 11548 { 11549 case OMP_CLAUSE_MAP: 11550 tkind = OMP_CLAUSE_MAP_KIND (c); 11551 break; 11552 case OMP_CLAUSE_TO: 11553 tkind = GOMP_MAP_TO; 11554 break; 11555 case OMP_CLAUSE_FROM: 11556 tkind = GOMP_MAP_FROM; 11557 break; 11558 default: 11559 gcc_unreachable (); 11560 } 11561 gcc_checking_assert (tkind 11562 < (HOST_WIDE_INT_C (1U) << talign_shift)); 11563 talign = ceil_log2 (talign); 11564 tkind |= talign << talign_shift; 11565 gcc_checking_assert (tkind 11566 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); 11567 CONSTRUCTOR_APPEND_ELT (vkind, purpose, 11568 build_int_cstu (tkind_type, tkind)); 11569 if (nc && nc != c) 11570 c = nc; 11571 } 11572 11573 gcc_assert (map_idx == map_cnt); 11574 11575 DECL_INITIAL (TREE_VEC_ELT (t, 1)) 11576 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize); 11577 DECL_INITIAL (TREE_VEC_ELT (t, 2)) 11578 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind); 11579 if (!TREE_STATIC (TREE_VEC_ELT (t, 1))) 11580 { 11581 gimple_seq initlist = NULL; 11582 force_gimple_operand (build1 (DECL_EXPR, void_type_node, 11583 TREE_VEC_ELT (t, 1)), 11584 &initlist, true, NULL_TREE); 11585 gimple_seq_add_seq (&ilist, initlist); 11586 11587 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), 11588 NULL); 11589 TREE_THIS_VOLATILE (clobber) = 1; 11590 gimple_seq_add_stmt (&olist, 11591 gimple_build_assign (TREE_VEC_ELT (t, 1), 11592 clobber)); 11593 } 11594 11595 tree clobber = build_constructor (ctx->record_type, NULL); 11596 TREE_THIS_VOLATILE (clobber) = 1; 11597 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, 11598 clobber)); 11599 } 11600 11601 /* Once all the expansions are done, sequence all the different 11602 fragments inside gimple_omp_body. */ 11603 11604 new_body = NULL; 11605 11606 if (offloaded 11607 && ctx->record_type) 11608 { 11609 t = build_fold_addr_expr_loc (loc, ctx->sender_decl); 11610 /* fixup_child_record_type might have changed receiver_decl's type. */ 11611 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); 11612 gimple_seq_add_stmt (&new_body, 11613 gimple_build_assign (ctx->receiver_decl, t)); 11614 } 11615 11616 if (offloaded) 11617 { 11618 gimple_seq_add_seq (&new_body, tgt_body); 11619 new_body = maybe_catch_exception (new_body); 11620 } 11621 else if (data_region) 11622 new_body = tgt_body; 11623 if (offloaded || data_region) 11624 { 11625 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); 11626 gimple_omp_set_body (stmt, new_body); 11627 } 11628 11629 bind = gimple_build_bind (NULL, NULL, 11630 tgt_bind ? gimple_bind_block (tgt_bind) 11631 : NULL_TREE); 11632 gsi_replace (gsi_p, bind, true); 11633 gimple_bind_add_seq (bind, irlist); 11634 gimple_bind_add_seq (bind, ilist); 11635 gimple_bind_add_stmt (bind, stmt); 11636 gimple_bind_add_seq (bind, olist); 11637 gimple_bind_add_seq (bind, orlist); 11638 11639 pop_gimplify_context (NULL); 11640 } 11641 11642 /* Expand code for an OpenMP teams directive. */ 11643 11644 static void 11645 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11646 { 11647 gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p)); 11648 push_gimplify_context (); 11649 11650 tree block = make_node (BLOCK); 11651 gbind *bind = gimple_build_bind (NULL, NULL, block); 11652 gsi_replace (gsi_p, bind, true); 11653 gimple_seq bind_body = NULL; 11654 gimple_seq dlist = NULL; 11655 gimple_seq olist = NULL; 11656 11657 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt), 11658 OMP_CLAUSE_NUM_TEAMS); 11659 if (num_teams == NULL_TREE) 11660 num_teams = build_int_cst (unsigned_type_node, 0); 11661 else 11662 { 11663 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams); 11664 num_teams = fold_convert (unsigned_type_node, num_teams); 11665 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue); 11666 } 11667 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt), 11668 OMP_CLAUSE_THREAD_LIMIT); 11669 if (thread_limit == NULL_TREE) 11670 thread_limit = build_int_cst (unsigned_type_node, 0); 11671 else 11672 { 11673 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit); 11674 thread_limit = fold_convert (unsigned_type_node, thread_limit); 11675 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val, 11676 fb_rvalue); 11677 } 11678 11679 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt), 11680 &bind_body, &dlist, ctx, NULL); 11681 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx); 11682 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx); 11683 gimple_seq_add_stmt (&bind_body, teams_stmt); 11684 11685 location_t loc = gimple_location (teams_stmt); 11686 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS); 11687 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit); 11688 gimple_set_location (call, loc); 11689 gimple_seq_add_stmt (&bind_body, call); 11690 11691 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt)); 11692 gimple_omp_set_body (teams_stmt, NULL); 11693 gimple_seq_add_seq (&bind_body, olist); 11694 gimple_seq_add_seq (&bind_body, dlist); 11695 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true)); 11696 gimple_bind_set_body (bind, bind_body); 11697 11698 pop_gimplify_context (bind); 11699 11700 gimple_bind_append_vars (bind, ctx->block_vars); 11701 BLOCK_VARS (block) = ctx->block_vars; 11702 if (BLOCK_VARS (block)) 11703 TREE_USED (block) = 1; 11704 } 11705 11706 11707 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be 11708 regimplified. If DATA is non-NULL, lower_omp_1 is outside 11709 of OMP context, but with task_shared_vars set. */ 11710 11711 static tree 11712 lower_omp_regimplify_p (tree *tp, int *walk_subtrees, 11713 void *data) 11714 { 11715 tree t = *tp; 11716 11717 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ 11718 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t)) 11719 return t; 11720 11721 if (task_shared_vars 11722 && DECL_P (t) 11723 && bitmap_bit_p (task_shared_vars, DECL_UID (t))) 11724 return t; 11725 11726 /* If a global variable has been privatized, TREE_CONSTANT on 11727 ADDR_EXPR might be wrong. */ 11728 if (data == NULL && TREE_CODE (t) == ADDR_EXPR) 11729 recompute_tree_invariant_for_addr_expr (t); 11730 11731 *walk_subtrees = !TYPE_P (t) && !DECL_P (t); 11732 return NULL_TREE; 11733 } 11734 11735 static void 11736 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11737 { 11738 gimple stmt = gsi_stmt (*gsi_p); 11739 struct walk_stmt_info wi; 11740 gcall *call_stmt; 11741 11742 if (gimple_has_location (stmt)) 11743 input_location = gimple_location (stmt); 11744 11745 if (task_shared_vars) 11746 memset (&wi, '\0', sizeof (wi)); 11747 11748 /* If we have issued syntax errors, avoid doing any heavy lifting. 11749 Just replace the OMP directives with a NOP to avoid 11750 confusing RTL expansion. */ 11751 if (seen_error () && is_gimple_omp (stmt)) 11752 { 11753 gsi_replace (gsi_p, gimple_build_nop (), true); 11754 return; 11755 } 11756 11757 switch (gimple_code (stmt)) 11758 { 11759 case GIMPLE_COND: 11760 { 11761 gcond *cond_stmt = as_a <gcond *> (stmt); 11762 if ((ctx || task_shared_vars) 11763 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 11764 lower_omp_regimplify_p, 11765 ctx ? NULL : &wi, NULL) 11766 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 11767 lower_omp_regimplify_p, 11768 ctx ? NULL : &wi, NULL))) 11769 gimple_regimplify_operands (cond_stmt, gsi_p); 11770 } 11771 break; 11772 case GIMPLE_CATCH: 11773 lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx); 11774 break; 11775 case GIMPLE_EH_FILTER: 11776 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx); 11777 break; 11778 case GIMPLE_TRY: 11779 lower_omp (gimple_try_eval_ptr (stmt), ctx); 11780 lower_omp (gimple_try_cleanup_ptr (stmt), ctx); 11781 break; 11782 case GIMPLE_TRANSACTION: 11783 lower_omp (gimple_transaction_body_ptr ( 11784 as_a <gtransaction *> (stmt)), 11785 ctx); 11786 break; 11787 case GIMPLE_BIND: 11788 lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx); 11789 break; 11790 case GIMPLE_OMP_PARALLEL: 11791 case GIMPLE_OMP_TASK: 11792 ctx = maybe_lookup_ctx (stmt); 11793 gcc_assert (ctx); 11794 if (ctx->cancellable) 11795 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); 11796 lower_omp_taskreg (gsi_p, ctx); 11797 break; 11798 case GIMPLE_OMP_FOR: 11799 ctx = maybe_lookup_ctx (stmt); 11800 gcc_assert (ctx); 11801 if (ctx->cancellable) 11802 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); 11803 lower_omp_for (gsi_p, ctx); 11804 break; 11805 case GIMPLE_OMP_SECTIONS: 11806 ctx = maybe_lookup_ctx (stmt); 11807 gcc_assert (ctx); 11808 if (ctx->cancellable) 11809 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); 11810 lower_omp_sections (gsi_p, ctx); 11811 break; 11812 case GIMPLE_OMP_SINGLE: 11813 ctx = maybe_lookup_ctx (stmt); 11814 gcc_assert (ctx); 11815 lower_omp_single (gsi_p, ctx); 11816 break; 11817 case GIMPLE_OMP_MASTER: 11818 ctx = maybe_lookup_ctx (stmt); 11819 gcc_assert (ctx); 11820 lower_omp_master (gsi_p, ctx); 11821 break; 11822 case GIMPLE_OMP_TASKGROUP: 11823 ctx = maybe_lookup_ctx (stmt); 11824 gcc_assert (ctx); 11825 lower_omp_taskgroup (gsi_p, ctx); 11826 break; 11827 case GIMPLE_OMP_ORDERED: 11828 ctx = maybe_lookup_ctx (stmt); 11829 gcc_assert (ctx); 11830 lower_omp_ordered (gsi_p, ctx); 11831 break; 11832 case GIMPLE_OMP_CRITICAL: 11833 ctx = maybe_lookup_ctx (stmt); 11834 gcc_assert (ctx); 11835 lower_omp_critical (gsi_p, ctx); 11836 break; 11837 case GIMPLE_OMP_ATOMIC_LOAD: 11838 if ((ctx || task_shared_vars) 11839 && walk_tree (gimple_omp_atomic_load_rhs_ptr ( 11840 as_a <gomp_atomic_load *> (stmt)), 11841 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL)) 11842 gimple_regimplify_operands (stmt, gsi_p); 11843 break; 11844 case GIMPLE_OMP_TARGET: 11845 ctx = maybe_lookup_ctx (stmt); 11846 gcc_assert (ctx); 11847 lower_omp_target (gsi_p, ctx); 11848 break; 11849 case GIMPLE_OMP_TEAMS: 11850 ctx = maybe_lookup_ctx (stmt); 11851 gcc_assert (ctx); 11852 lower_omp_teams (gsi_p, ctx); 11853 break; 11854 case GIMPLE_CALL: 11855 tree fndecl; 11856 call_stmt = as_a <gcall *> (stmt); 11857 fndecl = gimple_call_fndecl (call_stmt); 11858 if (fndecl 11859 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 11860 switch (DECL_FUNCTION_CODE (fndecl)) 11861 { 11862 case BUILT_IN_GOMP_BARRIER: 11863 if (ctx == NULL) 11864 break; 11865 /* FALLTHRU */ 11866 case BUILT_IN_GOMP_CANCEL: 11867 case BUILT_IN_GOMP_CANCELLATION_POINT: 11868 omp_context *cctx; 11869 cctx = ctx; 11870 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION) 11871 cctx = cctx->outer; 11872 gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE); 11873 if (!cctx->cancellable) 11874 { 11875 if (DECL_FUNCTION_CODE (fndecl) 11876 == BUILT_IN_GOMP_CANCELLATION_POINT) 11877 { 11878 stmt = gimple_build_nop (); 11879 gsi_replace (gsi_p, stmt, false); 11880 } 11881 break; 11882 } 11883 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER) 11884 { 11885 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL); 11886 gimple_call_set_fndecl (call_stmt, fndecl); 11887 gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl)); 11888 } 11889 tree lhs; 11890 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl))); 11891 gimple_call_set_lhs (call_stmt, lhs); 11892 tree fallthru_label; 11893 fallthru_label = create_artificial_label (UNKNOWN_LOCATION); 11894 gimple g; 11895 g = gimple_build_label (fallthru_label); 11896 gsi_insert_after (gsi_p, g, GSI_SAME_STMT); 11897 g = gimple_build_cond (NE_EXPR, lhs, 11898 fold_convert (TREE_TYPE (lhs), 11899 boolean_false_node), 11900 cctx->cancel_label, fallthru_label); 11901 gsi_insert_after (gsi_p, g, GSI_SAME_STMT); 11902 break; 11903 default: 11904 break; 11905 } 11906 /* FALLTHRU */ 11907 default: 11908 if ((ctx || task_shared_vars) 11909 && walk_gimple_op (stmt, lower_omp_regimplify_p, 11910 ctx ? NULL : &wi)) 11911 { 11912 /* Just remove clobbers, this should happen only if we have 11913 "privatized" local addressable variables in SIMD regions, 11914 the clobber isn't needed in that case and gimplifying address 11915 of the ARRAY_REF into a pointer and creating MEM_REF based 11916 clobber would create worse code than we get with the clobber 11917 dropped. */ 11918 if (gimple_clobber_p (stmt)) 11919 { 11920 gsi_replace (gsi_p, gimple_build_nop (), true); 11921 break; 11922 } 11923 gimple_regimplify_operands (stmt, gsi_p); 11924 } 11925 break; 11926 } 11927 } 11928 11929 static void 11930 lower_omp (gimple_seq *body, omp_context *ctx) 11931 { 11932 location_t saved_location = input_location; 11933 gimple_stmt_iterator gsi; 11934 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) 11935 lower_omp_1 (&gsi, ctx); 11936 /* During gimplification, we haven't folded statments inside offloading 11937 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */ 11938 if (target_nesting_level || taskreg_nesting_level) 11939 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) 11940 fold_stmt (&gsi); 11941 input_location = saved_location; 11942 } 11943 11944 /* Main entry point. */ 11945 11946 static unsigned int 11947 execute_lower_omp (void) 11948 { 11949 gimple_seq body; 11950 int i; 11951 omp_context *ctx; 11952 11953 /* This pass always runs, to provide PROP_gimple_lomp. 11954 But often, there is nothing to do. */ 11955 if (flag_cilkplus == 0 && flag_openacc == 0 && flag_openmp == 0 11956 && flag_openmp_simd == 0) 11957 return 0; 11958 11959 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0, 11960 delete_omp_context); 11961 11962 body = gimple_body (current_function_decl); 11963 scan_omp (&body, NULL); 11964 gcc_assert (taskreg_nesting_level == 0); 11965 FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx) 11966 finish_taskreg_scan (ctx); 11967 taskreg_contexts.release (); 11968 11969 if (all_contexts->root) 11970 { 11971 if (task_shared_vars) 11972 push_gimplify_context (); 11973 lower_omp (&body, NULL); 11974 if (task_shared_vars) 11975 pop_gimplify_context (NULL); 11976 } 11977 11978 if (all_contexts) 11979 { 11980 splay_tree_delete (all_contexts); 11981 all_contexts = NULL; 11982 } 11983 BITMAP_FREE (task_shared_vars); 11984 return 0; 11985 } 11986 11987 namespace { 11988 11989 const pass_data pass_data_lower_omp = 11990 { 11991 GIMPLE_PASS, /* type */ 11992 "omplower", /* name */ 11993 OPTGROUP_NONE, /* optinfo_flags */ 11994 TV_NONE, /* tv_id */ 11995 PROP_gimple_any, /* properties_required */ 11996 PROP_gimple_lomp, /* properties_provided */ 11997 0, /* properties_destroyed */ 11998 0, /* todo_flags_start */ 11999 0, /* todo_flags_finish */ 12000 }; 12001 12002 class pass_lower_omp : public gimple_opt_pass 12003 { 12004 public: 12005 pass_lower_omp (gcc::context *ctxt) 12006 : gimple_opt_pass (pass_data_lower_omp, ctxt) 12007 {} 12008 12009 /* opt_pass methods: */ 12010 virtual unsigned int execute (function *) { return execute_lower_omp (); } 12011 12012 }; // class pass_lower_omp 12013 12014 } // anon namespace 12015 12016 gimple_opt_pass * 12017 make_pass_lower_omp (gcc::context *ctxt) 12018 { 12019 return new pass_lower_omp (ctxt); 12020 } 12021 12022 /* The following is a utility to diagnose structured block violations. 12023 It is not part of the "omplower" pass, as that's invoked too late. It 12024 should be invoked by the respective front ends after gimplification. */ 12025 12026 static splay_tree all_labels; 12027 12028 /* Check for mismatched contexts and generate an error if needed. Return 12029 true if an error is detected. */ 12030 12031 static bool 12032 diagnose_sb_0 (gimple_stmt_iterator *gsi_p, 12033 gimple branch_ctx, gimple label_ctx) 12034 { 12035 gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx)); 12036 gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx)); 12037 12038 if (label_ctx == branch_ctx) 12039 return false; 12040 12041 const char* kind = NULL; 12042 12043 if (flag_cilkplus) 12044 { 12045 if ((branch_ctx 12046 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR 12047 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD) 12048 || (label_ctx 12049 && gimple_code (label_ctx) == GIMPLE_OMP_FOR 12050 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD)) 12051 kind = "Cilk Plus"; 12052 } 12053 if (flag_openacc) 12054 { 12055 if ((branch_ctx && is_gimple_omp_oacc (branch_ctx)) 12056 || (label_ctx && is_gimple_omp_oacc (label_ctx))) 12057 { 12058 gcc_checking_assert (kind == NULL); 12059 kind = "OpenACC"; 12060 } 12061 } 12062 if (kind == NULL) 12063 { 12064 gcc_checking_assert (flag_openmp || flag_openmp_simd); 12065 kind = "OpenMP"; 12066 } 12067 12068 /* 12069 Previously we kept track of the label's entire context in diagnose_sb_[12] 12070 so we could traverse it and issue a correct "exit" or "enter" error 12071 message upon a structured block violation. 12072 12073 We built the context by building a list with tree_cons'ing, but there is 12074 no easy counterpart in gimple tuples. It seems like far too much work 12075 for issuing exit/enter error messages. If someone really misses the 12076 distinct error message... patches welcome. 12077 */ 12078 12079 #if 0 12080 /* Try to avoid confusing the user by producing and error message 12081 with correct "exit" or "enter" verbiage. We prefer "exit" 12082 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */ 12083 if (branch_ctx == NULL) 12084 exit_p = false; 12085 else 12086 { 12087 while (label_ctx) 12088 { 12089 if (TREE_VALUE (label_ctx) == branch_ctx) 12090 { 12091 exit_p = false; 12092 break; 12093 } 12094 label_ctx = TREE_CHAIN (label_ctx); 12095 } 12096 } 12097 12098 if (exit_p) 12099 error ("invalid exit from %s structured block", kind); 12100 else 12101 error ("invalid entry to %s structured block", kind); 12102 #endif 12103 12104 /* If it's obvious we have an invalid entry, be specific about the error. */ 12105 if (branch_ctx == NULL) 12106 error ("invalid entry to %s structured block", kind); 12107 else 12108 { 12109 /* Otherwise, be vague and lazy, but efficient. */ 12110 error ("invalid branch to/from %s structured block", kind); 12111 } 12112 12113 gsi_replace (gsi_p, gimple_build_nop (), false); 12114 return true; 12115 } 12116 12117 /* Pass 1: Create a minimal tree of structured blocks, and record 12118 where each label is found. */ 12119 12120 static tree 12121 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, 12122 struct walk_stmt_info *wi) 12123 { 12124 gimple context = (gimple) wi->info; 12125 gimple inner_context; 12126 gimple stmt = gsi_stmt (*gsi_p); 12127 12128 *handled_ops_p = true; 12129 12130 switch (gimple_code (stmt)) 12131 { 12132 WALK_SUBSTMTS; 12133 12134 case GIMPLE_OMP_PARALLEL: 12135 case GIMPLE_OMP_TASK: 12136 case GIMPLE_OMP_SECTIONS: 12137 case GIMPLE_OMP_SINGLE: 12138 case GIMPLE_OMP_SECTION: 12139 case GIMPLE_OMP_MASTER: 12140 case GIMPLE_OMP_ORDERED: 12141 case GIMPLE_OMP_CRITICAL: 12142 case GIMPLE_OMP_TARGET: 12143 case GIMPLE_OMP_TEAMS: 12144 case GIMPLE_OMP_TASKGROUP: 12145 /* The minimal context here is just the current OMP construct. */ 12146 inner_context = stmt; 12147 wi->info = inner_context; 12148 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi); 12149 wi->info = context; 12150 break; 12151 12152 case GIMPLE_OMP_FOR: 12153 inner_context = stmt; 12154 wi->info = inner_context; 12155 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to 12156 walk them. */ 12157 walk_gimple_seq (gimple_omp_for_pre_body (stmt), 12158 diagnose_sb_1, NULL, wi); 12159 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi); 12160 wi->info = context; 12161 break; 12162 12163 case GIMPLE_LABEL: 12164 splay_tree_insert (all_labels, 12165 (splay_tree_key) gimple_label_label ( 12166 as_a <glabel *> (stmt)), 12167 (splay_tree_value) context); 12168 break; 12169 12170 default: 12171 break; 12172 } 12173 12174 return NULL_TREE; 12175 } 12176 12177 /* Pass 2: Check each branch and see if its context differs from that of 12178 the destination label's context. */ 12179 12180 static tree 12181 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, 12182 struct walk_stmt_info *wi) 12183 { 12184 gimple context = (gimple) wi->info; 12185 splay_tree_node n; 12186 gimple stmt = gsi_stmt (*gsi_p); 12187 12188 *handled_ops_p = true; 12189 12190 switch (gimple_code (stmt)) 12191 { 12192 WALK_SUBSTMTS; 12193 12194 case GIMPLE_OMP_PARALLEL: 12195 case GIMPLE_OMP_TASK: 12196 case GIMPLE_OMP_SECTIONS: 12197 case GIMPLE_OMP_SINGLE: 12198 case GIMPLE_OMP_SECTION: 12199 case GIMPLE_OMP_MASTER: 12200 case GIMPLE_OMP_ORDERED: 12201 case GIMPLE_OMP_CRITICAL: 12202 case GIMPLE_OMP_TARGET: 12203 case GIMPLE_OMP_TEAMS: 12204 case GIMPLE_OMP_TASKGROUP: 12205 wi->info = stmt; 12206 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi); 12207 wi->info = context; 12208 break; 12209 12210 case GIMPLE_OMP_FOR: 12211 wi->info = stmt; 12212 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to 12213 walk them. */ 12214 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt), 12215 diagnose_sb_2, NULL, wi); 12216 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi); 12217 wi->info = context; 12218 break; 12219 12220 case GIMPLE_COND: 12221 { 12222 gcond *cond_stmt = as_a <gcond *> (stmt); 12223 tree lab = gimple_cond_true_label (cond_stmt); 12224 if (lab) 12225 { 12226 n = splay_tree_lookup (all_labels, 12227 (splay_tree_key) lab); 12228 diagnose_sb_0 (gsi_p, context, 12229 n ? (gimple) n->value : NULL); 12230 } 12231 lab = gimple_cond_false_label (cond_stmt); 12232 if (lab) 12233 { 12234 n = splay_tree_lookup (all_labels, 12235 (splay_tree_key) lab); 12236 diagnose_sb_0 (gsi_p, context, 12237 n ? (gimple) n->value : NULL); 12238 } 12239 } 12240 break; 12241 12242 case GIMPLE_GOTO: 12243 { 12244 tree lab = gimple_goto_dest (stmt); 12245 if (TREE_CODE (lab) != LABEL_DECL) 12246 break; 12247 12248 n = splay_tree_lookup (all_labels, (splay_tree_key) lab); 12249 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL); 12250 } 12251 break; 12252 12253 case GIMPLE_SWITCH: 12254 { 12255 gswitch *switch_stmt = as_a <gswitch *> (stmt); 12256 unsigned int i; 12257 for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i) 12258 { 12259 tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i)); 12260 n = splay_tree_lookup (all_labels, (splay_tree_key) lab); 12261 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value)) 12262 break; 12263 } 12264 } 12265 break; 12266 12267 case GIMPLE_RETURN: 12268 diagnose_sb_0 (gsi_p, context, NULL); 12269 break; 12270 12271 default: 12272 break; 12273 } 12274 12275 return NULL_TREE; 12276 } 12277 12278 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant 12279 GIMPLE_* codes. */ 12280 bool 12281 make_gimple_omp_edges (basic_block bb, struct omp_region **region, 12282 int *region_idx) 12283 { 12284 gimple last = last_stmt (bb); 12285 enum gimple_code code = gimple_code (last); 12286 struct omp_region *cur_region = *region; 12287 bool fallthru = false; 12288 12289 switch (code) 12290 { 12291 case GIMPLE_OMP_PARALLEL: 12292 case GIMPLE_OMP_TASK: 12293 case GIMPLE_OMP_FOR: 12294 case GIMPLE_OMP_SINGLE: 12295 case GIMPLE_OMP_TEAMS: 12296 case GIMPLE_OMP_MASTER: 12297 case GIMPLE_OMP_TASKGROUP: 12298 case GIMPLE_OMP_ORDERED: 12299 case GIMPLE_OMP_CRITICAL: 12300 case GIMPLE_OMP_SECTION: 12301 cur_region = new_omp_region (bb, code, cur_region); 12302 fallthru = true; 12303 break; 12304 12305 case GIMPLE_OMP_TARGET: 12306 cur_region = new_omp_region (bb, code, cur_region); 12307 fallthru = true; 12308 switch (gimple_omp_target_kind (last)) 12309 { 12310 case GF_OMP_TARGET_KIND_REGION: 12311 case GF_OMP_TARGET_KIND_DATA: 12312 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 12313 case GF_OMP_TARGET_KIND_OACC_KERNELS: 12314 case GF_OMP_TARGET_KIND_OACC_DATA: 12315 break; 12316 case GF_OMP_TARGET_KIND_UPDATE: 12317 case GF_OMP_TARGET_KIND_OACC_UPDATE: 12318 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 12319 cur_region = cur_region->outer; 12320 break; 12321 default: 12322 gcc_unreachable (); 12323 } 12324 break; 12325 12326 case GIMPLE_OMP_SECTIONS: 12327 cur_region = new_omp_region (bb, code, cur_region); 12328 fallthru = true; 12329 break; 12330 12331 case GIMPLE_OMP_SECTIONS_SWITCH: 12332 fallthru = false; 12333 break; 12334 12335 case GIMPLE_OMP_ATOMIC_LOAD: 12336 case GIMPLE_OMP_ATOMIC_STORE: 12337 fallthru = true; 12338 break; 12339 12340 case GIMPLE_OMP_RETURN: 12341 /* In the case of a GIMPLE_OMP_SECTION, the edge will go 12342 somewhere other than the next block. This will be 12343 created later. */ 12344 cur_region->exit = bb; 12345 if (cur_region->type == GIMPLE_OMP_TASK) 12346 /* Add an edge corresponding to not scheduling the task 12347 immediately. */ 12348 make_edge (cur_region->entry, bb, EDGE_ABNORMAL); 12349 fallthru = cur_region->type != GIMPLE_OMP_SECTION; 12350 cur_region = cur_region->outer; 12351 break; 12352 12353 case GIMPLE_OMP_CONTINUE: 12354 cur_region->cont = bb; 12355 switch (cur_region->type) 12356 { 12357 case GIMPLE_OMP_FOR: 12358 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE 12359 succs edges as abnormal to prevent splitting 12360 them. */ 12361 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL; 12362 /* Make the loopback edge. */ 12363 make_edge (bb, single_succ (cur_region->entry), 12364 EDGE_ABNORMAL); 12365 12366 /* Create an edge from GIMPLE_OMP_FOR to exit, which 12367 corresponds to the case that the body of the loop 12368 is not executed at all. */ 12369 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL); 12370 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL); 12371 fallthru = false; 12372 break; 12373 12374 case GIMPLE_OMP_SECTIONS: 12375 /* Wire up the edges into and out of the nested sections. */ 12376 { 12377 basic_block switch_bb = single_succ (cur_region->entry); 12378 12379 struct omp_region *i; 12380 for (i = cur_region->inner; i ; i = i->next) 12381 { 12382 gcc_assert (i->type == GIMPLE_OMP_SECTION); 12383 make_edge (switch_bb, i->entry, 0); 12384 make_edge (i->exit, bb, EDGE_FALLTHRU); 12385 } 12386 12387 /* Make the loopback edge to the block with 12388 GIMPLE_OMP_SECTIONS_SWITCH. */ 12389 make_edge (bb, switch_bb, 0); 12390 12391 /* Make the edge from the switch to exit. */ 12392 make_edge (switch_bb, bb->next_bb, 0); 12393 fallthru = false; 12394 } 12395 break; 12396 12397 case GIMPLE_OMP_TASK: 12398 fallthru = true; 12399 break; 12400 12401 default: 12402 gcc_unreachable (); 12403 } 12404 break; 12405 12406 default: 12407 gcc_unreachable (); 12408 } 12409 12410 if (*region != cur_region) 12411 { 12412 *region = cur_region; 12413 if (cur_region) 12414 *region_idx = cur_region->entry->index; 12415 else 12416 *region_idx = 0; 12417 } 12418 12419 return fallthru; 12420 } 12421 12422 static unsigned int 12423 diagnose_omp_structured_block_errors (void) 12424 { 12425 struct walk_stmt_info wi; 12426 gimple_seq body = gimple_body (current_function_decl); 12427 12428 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0); 12429 12430 memset (&wi, 0, sizeof (wi)); 12431 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi); 12432 12433 memset (&wi, 0, sizeof (wi)); 12434 wi.want_locations = true; 12435 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi); 12436 12437 gimple_set_body (current_function_decl, body); 12438 12439 splay_tree_delete (all_labels); 12440 all_labels = NULL; 12441 12442 return 0; 12443 } 12444 12445 namespace { 12446 12447 const pass_data pass_data_diagnose_omp_blocks = 12448 { 12449 GIMPLE_PASS, /* type */ 12450 "*diagnose_omp_blocks", /* name */ 12451 OPTGROUP_NONE, /* optinfo_flags */ 12452 TV_NONE, /* tv_id */ 12453 PROP_gimple_any, /* properties_required */ 12454 0, /* properties_provided */ 12455 0, /* properties_destroyed */ 12456 0, /* todo_flags_start */ 12457 0, /* todo_flags_finish */ 12458 }; 12459 12460 class pass_diagnose_omp_blocks : public gimple_opt_pass 12461 { 12462 public: 12463 pass_diagnose_omp_blocks (gcc::context *ctxt) 12464 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt) 12465 {} 12466 12467 /* opt_pass methods: */ 12468 virtual bool gate (function *) 12469 { 12470 return flag_cilkplus || flag_openacc || flag_openmp || flag_openmp_simd; 12471 } 12472 virtual unsigned int execute (function *) 12473 { 12474 return diagnose_omp_structured_block_errors (); 12475 } 12476 12477 }; // class pass_diagnose_omp_blocks 12478 12479 } // anon namespace 12480 12481 gimple_opt_pass * 12482 make_pass_diagnose_omp_blocks (gcc::context *ctxt) 12483 { 12484 return new pass_diagnose_omp_blocks (ctxt); 12485 } 12486 12487 /* SIMD clone supporting code. */ 12488 12489 /* Allocate a fresh `simd_clone' and return it. NARGS is the number 12490 of arguments to reserve space for. */ 12491 12492 static struct cgraph_simd_clone * 12493 simd_clone_struct_alloc (int nargs) 12494 { 12495 struct cgraph_simd_clone *clone_info; 12496 size_t len = (sizeof (struct cgraph_simd_clone) 12497 + nargs * sizeof (struct cgraph_simd_clone_arg)); 12498 clone_info = (struct cgraph_simd_clone *) 12499 ggc_internal_cleared_alloc (len); 12500 return clone_info; 12501 } 12502 12503 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */ 12504 12505 static inline void 12506 simd_clone_struct_copy (struct cgraph_simd_clone *to, 12507 struct cgraph_simd_clone *from) 12508 { 12509 memcpy (to, from, (sizeof (struct cgraph_simd_clone) 12510 + ((from->nargs - from->inbranch) 12511 * sizeof (struct cgraph_simd_clone_arg)))); 12512 } 12513 12514 /* Return vector of parameter types of function FNDECL. This uses 12515 TYPE_ARG_TYPES if available, otherwise falls back to types of 12516 DECL_ARGUMENTS types. */ 12517 12518 vec<tree> 12519 simd_clone_vector_of_formal_parm_types (tree fndecl) 12520 { 12521 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl))) 12522 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl)); 12523 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl); 12524 unsigned int i; 12525 tree arg; 12526 FOR_EACH_VEC_ELT (args, i, arg) 12527 args[i] = TREE_TYPE (args[i]); 12528 return args; 12529 } 12530 12531 /* Given a simd function in NODE, extract the simd specific 12532 information from the OMP clauses passed in CLAUSES, and return 12533 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED 12534 is set to TRUE if the `inbranch' or `notinbranch' clause specified, 12535 otherwise set to FALSE. */ 12536 12537 static struct cgraph_simd_clone * 12538 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses, 12539 bool *inbranch_specified) 12540 { 12541 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl); 12542 tree t; 12543 int n; 12544 *inbranch_specified = false; 12545 12546 n = args.length (); 12547 if (n > 0 && args.last () == void_type_node) 12548 n--; 12549 12550 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to 12551 be cloned have a distinctive artificial label in addition to "omp 12552 declare simd". */ 12553 bool cilk_clone 12554 = (flag_cilkplus 12555 && lookup_attribute ("cilk simd function", 12556 DECL_ATTRIBUTES (node->decl))); 12557 12558 /* Allocate one more than needed just in case this is an in-branch 12559 clone which will require a mask argument. */ 12560 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1); 12561 clone_info->nargs = n; 12562 clone_info->cilk_elemental = cilk_clone; 12563 12564 if (!clauses) 12565 { 12566 args.release (); 12567 return clone_info; 12568 } 12569 clauses = TREE_VALUE (clauses); 12570 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE) 12571 return clone_info; 12572 12573 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t)) 12574 { 12575 switch (OMP_CLAUSE_CODE (t)) 12576 { 12577 case OMP_CLAUSE_INBRANCH: 12578 clone_info->inbranch = 1; 12579 *inbranch_specified = true; 12580 break; 12581 case OMP_CLAUSE_NOTINBRANCH: 12582 clone_info->inbranch = 0; 12583 *inbranch_specified = true; 12584 break; 12585 case OMP_CLAUSE_SIMDLEN: 12586 clone_info->simdlen 12587 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t)); 12588 break; 12589 case OMP_CLAUSE_LINEAR: 12590 { 12591 tree decl = OMP_CLAUSE_DECL (t); 12592 tree step = OMP_CLAUSE_LINEAR_STEP (t); 12593 int argno = TREE_INT_CST_LOW (decl); 12594 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t)) 12595 { 12596 clone_info->args[argno].arg_type 12597 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP; 12598 clone_info->args[argno].linear_step = tree_to_shwi (step); 12599 gcc_assert (clone_info->args[argno].linear_step >= 0 12600 && clone_info->args[argno].linear_step < n); 12601 } 12602 else 12603 { 12604 if (POINTER_TYPE_P (args[argno])) 12605 step = fold_convert (ssizetype, step); 12606 if (!tree_fits_shwi_p (step)) 12607 { 12608 warning_at (OMP_CLAUSE_LOCATION (t), 0, 12609 "ignoring large linear step"); 12610 args.release (); 12611 return NULL; 12612 } 12613 else if (integer_zerop (step)) 12614 { 12615 warning_at (OMP_CLAUSE_LOCATION (t), 0, 12616 "ignoring zero linear step"); 12617 args.release (); 12618 return NULL; 12619 } 12620 else 12621 { 12622 clone_info->args[argno].arg_type 12623 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP; 12624 clone_info->args[argno].linear_step = tree_to_shwi (step); 12625 } 12626 } 12627 break; 12628 } 12629 case OMP_CLAUSE_UNIFORM: 12630 { 12631 tree decl = OMP_CLAUSE_DECL (t); 12632 int argno = tree_to_uhwi (decl); 12633 clone_info->args[argno].arg_type 12634 = SIMD_CLONE_ARG_TYPE_UNIFORM; 12635 break; 12636 } 12637 case OMP_CLAUSE_ALIGNED: 12638 { 12639 tree decl = OMP_CLAUSE_DECL (t); 12640 int argno = tree_to_uhwi (decl); 12641 clone_info->args[argno].alignment 12642 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t)); 12643 break; 12644 } 12645 default: 12646 break; 12647 } 12648 } 12649 args.release (); 12650 return clone_info; 12651 } 12652 12653 /* Given a SIMD clone in NODE, calculate the characteristic data 12654 type and return the coresponding type. The characteristic data 12655 type is computed as described in the Intel Vector ABI. */ 12656 12657 static tree 12658 simd_clone_compute_base_data_type (struct cgraph_node *node, 12659 struct cgraph_simd_clone *clone_info) 12660 { 12661 tree type = integer_type_node; 12662 tree fndecl = node->decl; 12663 12664 /* a) For non-void function, the characteristic data type is the 12665 return type. */ 12666 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE) 12667 type = TREE_TYPE (TREE_TYPE (fndecl)); 12668 12669 /* b) If the function has any non-uniform, non-linear parameters, 12670 then the characteristic data type is the type of the first 12671 such parameter. */ 12672 else 12673 { 12674 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl); 12675 for (unsigned int i = 0; i < clone_info->nargs; ++i) 12676 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) 12677 { 12678 type = map[i]; 12679 break; 12680 } 12681 map.release (); 12682 } 12683 12684 /* c) If the characteristic data type determined by a) or b) above 12685 is struct, union, or class type which is pass-by-value (except 12686 for the type that maps to the built-in complex data type), the 12687 characteristic data type is int. */ 12688 if (RECORD_OR_UNION_TYPE_P (type) 12689 && !aggregate_value_p (type, NULL) 12690 && TREE_CODE (type) != COMPLEX_TYPE) 12691 return integer_type_node; 12692 12693 /* d) If none of the above three classes is applicable, the 12694 characteristic data type is int. */ 12695 12696 return type; 12697 12698 /* e) For Intel Xeon Phi native and offload compilation, if the 12699 resulting characteristic data type is 8-bit or 16-bit integer 12700 data type, the characteristic data type is int. */ 12701 /* Well, we don't handle Xeon Phi yet. */ 12702 } 12703 12704 static tree 12705 simd_clone_mangle (struct cgraph_node *node, 12706 struct cgraph_simd_clone *clone_info) 12707 { 12708 char vecsize_mangle = clone_info->vecsize_mangle; 12709 char mask = clone_info->inbranch ? 'M' : 'N'; 12710 unsigned int simdlen = clone_info->simdlen; 12711 unsigned int n; 12712 pretty_printer pp; 12713 12714 gcc_assert (vecsize_mangle && simdlen); 12715 12716 pp_string (&pp, "_ZGV"); 12717 pp_character (&pp, vecsize_mangle); 12718 pp_character (&pp, mask); 12719 pp_decimal_int (&pp, simdlen); 12720 12721 for (n = 0; n < clone_info->nargs; ++n) 12722 { 12723 struct cgraph_simd_clone_arg arg = clone_info->args[n]; 12724 12725 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM) 12726 pp_character (&pp, 'u'); 12727 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) 12728 { 12729 gcc_assert (arg.linear_step != 0); 12730 pp_character (&pp, 'l'); 12731 if (arg.linear_step > 1) 12732 pp_unsigned_wide_integer (&pp, arg.linear_step); 12733 else if (arg.linear_step < 0) 12734 { 12735 pp_character (&pp, 'n'); 12736 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT) 12737 arg.linear_step)); 12738 } 12739 } 12740 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP) 12741 { 12742 pp_character (&pp, 's'); 12743 pp_unsigned_wide_integer (&pp, arg.linear_step); 12744 } 12745 else 12746 pp_character (&pp, 'v'); 12747 if (arg.alignment) 12748 { 12749 pp_character (&pp, 'a'); 12750 pp_decimal_int (&pp, arg.alignment); 12751 } 12752 } 12753 12754 pp_underscore (&pp); 12755 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)); 12756 if (*str == '*') 12757 ++str; 12758 pp_string (&pp, str); 12759 str = pp_formatted_text (&pp); 12760 12761 /* If there already is a SIMD clone with the same mangled name, don't 12762 add another one. This can happen e.g. for 12763 #pragma omp declare simd 12764 #pragma omp declare simd simdlen(8) 12765 int foo (int, int); 12766 if the simdlen is assumed to be 8 for the first one, etc. */ 12767 for (struct cgraph_node *clone = node->simd_clones; clone; 12768 clone = clone->simdclone->next_clone) 12769 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)), 12770 str) == 0) 12771 return NULL_TREE; 12772 12773 return get_identifier (str); 12774 } 12775 12776 /* Create a simd clone of OLD_NODE and return it. */ 12777 12778 static struct cgraph_node * 12779 simd_clone_create (struct cgraph_node *old_node) 12780 { 12781 struct cgraph_node *new_node; 12782 if (old_node->definition) 12783 { 12784 if (!old_node->has_gimple_body_p ()) 12785 return NULL; 12786 old_node->get_body (); 12787 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL, 12788 false, NULL, NULL, 12789 "simdclone"); 12790 } 12791 else 12792 { 12793 tree old_decl = old_node->decl; 12794 tree new_decl = copy_node (old_node->decl); 12795 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone"); 12796 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); 12797 SET_DECL_RTL (new_decl, NULL); 12798 DECL_STATIC_CONSTRUCTOR (new_decl) = 0; 12799 DECL_STATIC_DESTRUCTOR (new_decl) = 0; 12800 new_node = old_node->create_version_clone (new_decl, vNULL, NULL); 12801 if (old_node->in_other_partition) 12802 new_node->in_other_partition = 1; 12803 } 12804 if (new_node == NULL) 12805 return new_node; 12806 12807 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl); 12808 12809 /* The function cgraph_function_versioning () will force the new 12810 symbol local. Undo this, and inherit external visability from 12811 the old node. */ 12812 new_node->local.local = old_node->local.local; 12813 new_node->externally_visible = old_node->externally_visible; 12814 12815 return new_node; 12816 } 12817 12818 /* Adjust the return type of the given function to its appropriate 12819 vector counterpart. Returns a simd array to be used throughout the 12820 function as a return value. */ 12821 12822 static tree 12823 simd_clone_adjust_return_type (struct cgraph_node *node) 12824 { 12825 tree fndecl = node->decl; 12826 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl)); 12827 unsigned int veclen; 12828 tree t; 12829 12830 /* Adjust the function return type. */ 12831 if (orig_rettype == void_type_node) 12832 return NULL_TREE; 12833 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl)); 12834 t = TREE_TYPE (TREE_TYPE (fndecl)); 12835 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t)) 12836 veclen = node->simdclone->vecsize_int; 12837 else 12838 veclen = node->simdclone->vecsize_float; 12839 veclen /= GET_MODE_BITSIZE (TYPE_MODE (t)); 12840 if (veclen > node->simdclone->simdlen) 12841 veclen = node->simdclone->simdlen; 12842 if (POINTER_TYPE_P (t)) 12843 t = pointer_sized_int_node; 12844 if (veclen == node->simdclone->simdlen) 12845 t = build_vector_type (t, node->simdclone->simdlen); 12846 else 12847 { 12848 t = build_vector_type (t, veclen); 12849 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen); 12850 } 12851 TREE_TYPE (TREE_TYPE (fndecl)) = t; 12852 if (!node->definition) 12853 return NULL_TREE; 12854 12855 t = DECL_RESULT (fndecl); 12856 /* Adjust the DECL_RESULT. */ 12857 gcc_assert (TREE_TYPE (t) != void_type_node); 12858 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl)); 12859 relayout_decl (t); 12860 12861 tree atype = build_array_type_nelts (orig_rettype, 12862 node->simdclone->simdlen); 12863 if (veclen != node->simdclone->simdlen) 12864 return build1 (VIEW_CONVERT_EXPR, atype, t); 12865 12866 /* Set up a SIMD array to use as the return value. */ 12867 tree retval = create_tmp_var_raw (atype, "retval"); 12868 gimple_add_tmp_var (retval); 12869 return retval; 12870 } 12871 12872 /* Each vector argument has a corresponding array to be used locally 12873 as part of the eventual loop. Create such temporary array and 12874 return it. 12875 12876 PREFIX is the prefix to be used for the temporary. 12877 12878 TYPE is the inner element type. 12879 12880 SIMDLEN is the number of elements. */ 12881 12882 static tree 12883 create_tmp_simd_array (const char *prefix, tree type, int simdlen) 12884 { 12885 tree atype = build_array_type_nelts (type, simdlen); 12886 tree avar = create_tmp_var_raw (atype, prefix); 12887 gimple_add_tmp_var (avar); 12888 return avar; 12889 } 12890 12891 /* Modify the function argument types to their corresponding vector 12892 counterparts if appropriate. Also, create one array for each simd 12893 argument to be used locally when using the function arguments as 12894 part of the loop. 12895 12896 NODE is the function whose arguments are to be adjusted. 12897 12898 Returns an adjustment vector that will be filled describing how the 12899 argument types will be adjusted. */ 12900 12901 static ipa_parm_adjustment_vec 12902 simd_clone_adjust_argument_types (struct cgraph_node *node) 12903 { 12904 vec<tree> args; 12905 ipa_parm_adjustment_vec adjustments; 12906 12907 if (node->definition) 12908 args = ipa_get_vector_of_formal_parms (node->decl); 12909 else 12910 args = simd_clone_vector_of_formal_parm_types (node->decl); 12911 adjustments.create (args.length ()); 12912 unsigned i, j, veclen; 12913 struct ipa_parm_adjustment adj; 12914 for (i = 0; i < node->simdclone->nargs; ++i) 12915 { 12916 memset (&adj, 0, sizeof (adj)); 12917 tree parm = args[i]; 12918 tree parm_type = node->definition ? TREE_TYPE (parm) : parm; 12919 adj.base_index = i; 12920 adj.base = parm; 12921 12922 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE; 12923 node->simdclone->args[i].orig_type = parm_type; 12924 12925 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR) 12926 { 12927 /* No adjustment necessary for scalar arguments. */ 12928 adj.op = IPA_PARM_OP_COPY; 12929 } 12930 else 12931 { 12932 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type)) 12933 veclen = node->simdclone->vecsize_int; 12934 else 12935 veclen = node->simdclone->vecsize_float; 12936 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type)); 12937 if (veclen > node->simdclone->simdlen) 12938 veclen = node->simdclone->simdlen; 12939 adj.arg_prefix = "simd"; 12940 if (POINTER_TYPE_P (parm_type)) 12941 adj.type = build_vector_type (pointer_sized_int_node, veclen); 12942 else 12943 adj.type = build_vector_type (parm_type, veclen); 12944 node->simdclone->args[i].vector_type = adj.type; 12945 for (j = veclen; j < node->simdclone->simdlen; j += veclen) 12946 { 12947 adjustments.safe_push (adj); 12948 if (j == veclen) 12949 { 12950 memset (&adj, 0, sizeof (adj)); 12951 adj.op = IPA_PARM_OP_NEW; 12952 adj.arg_prefix = "simd"; 12953 adj.base_index = i; 12954 adj.type = node->simdclone->args[i].vector_type; 12955 } 12956 } 12957 12958 if (node->definition) 12959 node->simdclone->args[i].simd_array 12960 = create_tmp_simd_array (DECL_NAME (parm) 12961 ? IDENTIFIER_POINTER (DECL_NAME (parm)) 12962 : NULL, 12963 parm_type, node->simdclone->simdlen); 12964 } 12965 adjustments.safe_push (adj); 12966 } 12967 12968 if (node->simdclone->inbranch) 12969 { 12970 tree base_type 12971 = simd_clone_compute_base_data_type (node->simdclone->origin, 12972 node->simdclone); 12973 12974 memset (&adj, 0, sizeof (adj)); 12975 adj.op = IPA_PARM_OP_NEW; 12976 adj.arg_prefix = "mask"; 12977 12978 adj.base_index = i; 12979 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type)) 12980 veclen = node->simdclone->vecsize_int; 12981 else 12982 veclen = node->simdclone->vecsize_float; 12983 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type)); 12984 if (veclen > node->simdclone->simdlen) 12985 veclen = node->simdclone->simdlen; 12986 if (POINTER_TYPE_P (base_type)) 12987 adj.type = build_vector_type (pointer_sized_int_node, veclen); 12988 else 12989 adj.type = build_vector_type (base_type, veclen); 12990 adjustments.safe_push (adj); 12991 12992 for (j = veclen; j < node->simdclone->simdlen; j += veclen) 12993 adjustments.safe_push (adj); 12994 12995 /* We have previously allocated one extra entry for the mask. Use 12996 it and fill it. */ 12997 struct cgraph_simd_clone *sc = node->simdclone; 12998 sc->nargs++; 12999 if (node->definition) 13000 { 13001 sc->args[i].orig_arg 13002 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type); 13003 sc->args[i].simd_array 13004 = create_tmp_simd_array ("mask", base_type, sc->simdlen); 13005 } 13006 sc->args[i].orig_type = base_type; 13007 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK; 13008 } 13009 13010 if (node->definition) 13011 ipa_modify_formal_parameters (node->decl, adjustments); 13012 else 13013 { 13014 tree new_arg_types = NULL_TREE, new_reversed; 13015 bool last_parm_void = false; 13016 if (args.length () > 0 && args.last () == void_type_node) 13017 last_parm_void = true; 13018 13019 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl))); 13020 j = adjustments.length (); 13021 for (i = 0; i < j; i++) 13022 { 13023 struct ipa_parm_adjustment *adj = &adjustments[i]; 13024 tree ptype; 13025 if (adj->op == IPA_PARM_OP_COPY) 13026 ptype = args[adj->base_index]; 13027 else 13028 ptype = adj->type; 13029 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types); 13030 } 13031 new_reversed = nreverse (new_arg_types); 13032 if (last_parm_void) 13033 { 13034 if (new_reversed) 13035 TREE_CHAIN (new_arg_types) = void_list_node; 13036 else 13037 new_reversed = void_list_node; 13038 } 13039 13040 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl)); 13041 TYPE_ARG_TYPES (new_type) = new_reversed; 13042 TREE_TYPE (node->decl) = new_type; 13043 13044 adjustments.release (); 13045 } 13046 args.release (); 13047 return adjustments; 13048 } 13049 13050 /* Initialize and copy the function arguments in NODE to their 13051 corresponding local simd arrays. Returns a fresh gimple_seq with 13052 the instruction sequence generated. */ 13053 13054 static gimple_seq 13055 simd_clone_init_simd_arrays (struct cgraph_node *node, 13056 ipa_parm_adjustment_vec adjustments) 13057 { 13058 gimple_seq seq = NULL; 13059 unsigned i = 0, j = 0, k; 13060 13061 for (tree arg = DECL_ARGUMENTS (node->decl); 13062 arg; 13063 arg = DECL_CHAIN (arg), i++, j++) 13064 { 13065 if (adjustments[j].op == IPA_PARM_OP_COPY) 13066 continue; 13067 13068 node->simdclone->args[i].vector_arg = arg; 13069 13070 tree array = node->simdclone->args[i].simd_array; 13071 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen) 13072 { 13073 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 13074 tree ptr = build_fold_addr_expr (array); 13075 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 13076 build_int_cst (ptype, 0)); 13077 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 13078 gimplify_and_add (t, &seq); 13079 } 13080 else 13081 { 13082 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)); 13083 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 13084 for (k = 0; k < node->simdclone->simdlen; k += simdlen) 13085 { 13086 tree ptr = build_fold_addr_expr (array); 13087 int elemsize; 13088 if (k) 13089 { 13090 arg = DECL_CHAIN (arg); 13091 j++; 13092 } 13093 elemsize 13094 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg)))); 13095 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 13096 build_int_cst (ptype, k * elemsize)); 13097 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 13098 gimplify_and_add (t, &seq); 13099 } 13100 } 13101 } 13102 return seq; 13103 } 13104 13105 /* Callback info for ipa_simd_modify_stmt_ops below. */ 13106 13107 struct modify_stmt_info { 13108 ipa_parm_adjustment_vec adjustments; 13109 gimple stmt; 13110 /* True if the parent statement was modified by 13111 ipa_simd_modify_stmt_ops. */ 13112 bool modified; 13113 }; 13114 13115 /* Callback for walk_gimple_op. 13116 13117 Adjust operands from a given statement as specified in the 13118 adjustments vector in the callback data. */ 13119 13120 static tree 13121 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data) 13122 { 13123 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 13124 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info; 13125 tree *orig_tp = tp; 13126 if (TREE_CODE (*tp) == ADDR_EXPR) 13127 tp = &TREE_OPERAND (*tp, 0); 13128 struct ipa_parm_adjustment *cand = NULL; 13129 if (TREE_CODE (*tp) == PARM_DECL) 13130 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true); 13131 else 13132 { 13133 if (TYPE_P (*tp)) 13134 *walk_subtrees = 0; 13135 } 13136 13137 tree repl = NULL_TREE; 13138 if (cand) 13139 repl = unshare_expr (cand->new_decl); 13140 else 13141 { 13142 if (tp != orig_tp) 13143 { 13144 *walk_subtrees = 0; 13145 bool modified = info->modified; 13146 info->modified = false; 13147 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset); 13148 if (!info->modified) 13149 { 13150 info->modified = modified; 13151 return NULL_TREE; 13152 } 13153 info->modified = modified; 13154 repl = *tp; 13155 } 13156 else 13157 return NULL_TREE; 13158 } 13159 13160 if (tp != orig_tp) 13161 { 13162 repl = build_fold_addr_expr (repl); 13163 gimple stmt; 13164 if (is_gimple_debug (info->stmt)) 13165 { 13166 tree vexpr = make_node (DEBUG_EXPR_DECL); 13167 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL); 13168 DECL_ARTIFICIAL (vexpr) = 1; 13169 TREE_TYPE (vexpr) = TREE_TYPE (repl); 13170 DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl)); 13171 repl = vexpr; 13172 } 13173 else 13174 { 13175 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl); 13176 repl = gimple_assign_lhs (stmt); 13177 } 13178 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt); 13179 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 13180 *orig_tp = repl; 13181 } 13182 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl))) 13183 { 13184 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl); 13185 *tp = vce; 13186 } 13187 else 13188 *tp = repl; 13189 13190 info->modified = true; 13191 return NULL_TREE; 13192 } 13193 13194 /* Traverse the function body and perform all modifications as 13195 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be 13196 modified such that the replacement/reduction value will now be an 13197 offset into the corresponding simd_array. 13198 13199 This function will replace all function argument uses with their 13200 corresponding simd array elements, and ajust the return values 13201 accordingly. */ 13202 13203 static void 13204 ipa_simd_modify_function_body (struct cgraph_node *node, 13205 ipa_parm_adjustment_vec adjustments, 13206 tree retval_array, tree iter) 13207 { 13208 basic_block bb; 13209 unsigned int i, j, l; 13210 13211 /* Re-use the adjustments array, but this time use it to replace 13212 every function argument use to an offset into the corresponding 13213 simd_array. */ 13214 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j) 13215 { 13216 if (!node->simdclone->args[i].vector_arg) 13217 continue; 13218 13219 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg); 13220 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg); 13221 adjustments[j].new_decl 13222 = build4 (ARRAY_REF, 13223 basetype, 13224 node->simdclone->args[i].simd_array, 13225 iter, 13226 NULL_TREE, NULL_TREE); 13227 if (adjustments[j].op == IPA_PARM_OP_NONE 13228 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen) 13229 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1; 13230 } 13231 13232 l = adjustments.length (); 13233 for (i = 1; i < num_ssa_names; i++) 13234 { 13235 tree name = ssa_name (i); 13236 if (name 13237 && SSA_NAME_VAR (name) 13238 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL) 13239 { 13240 for (j = 0; j < l; j++) 13241 if (SSA_NAME_VAR (name) == adjustments[j].base 13242 && adjustments[j].new_decl) 13243 { 13244 tree base_var; 13245 if (adjustments[j].new_ssa_base == NULL_TREE) 13246 { 13247 base_var 13248 = copy_var_decl (adjustments[j].base, 13249 DECL_NAME (adjustments[j].base), 13250 TREE_TYPE (adjustments[j].base)); 13251 adjustments[j].new_ssa_base = base_var; 13252 } 13253 else 13254 base_var = adjustments[j].new_ssa_base; 13255 if (SSA_NAME_IS_DEFAULT_DEF (name)) 13256 { 13257 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 13258 gimple_stmt_iterator gsi = gsi_after_labels (bb); 13259 tree new_decl = unshare_expr (adjustments[j].new_decl); 13260 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE); 13261 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 13262 SSA_NAME_IS_DEFAULT_DEF (name) = 0; 13263 gimple stmt = gimple_build_assign (name, new_decl); 13264 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 13265 } 13266 else 13267 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 13268 } 13269 } 13270 } 13271 13272 struct modify_stmt_info info; 13273 info.adjustments = adjustments; 13274 13275 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) 13276 { 13277 gimple_stmt_iterator gsi; 13278 13279 gsi = gsi_start_bb (bb); 13280 while (!gsi_end_p (gsi)) 13281 { 13282 gimple stmt = gsi_stmt (gsi); 13283 info.stmt = stmt; 13284 struct walk_stmt_info wi; 13285 13286 memset (&wi, 0, sizeof (wi)); 13287 info.modified = false; 13288 wi.info = &info; 13289 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi); 13290 13291 if (greturn *return_stmt = dyn_cast <greturn *> (stmt)) 13292 { 13293 tree retval = gimple_return_retval (return_stmt); 13294 if (!retval) 13295 { 13296 gsi_remove (&gsi, true); 13297 continue; 13298 } 13299 13300 /* Replace `return foo' with `retval_array[iter] = foo'. */ 13301 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval), 13302 retval_array, iter, NULL, NULL); 13303 stmt = gimple_build_assign (ref, retval); 13304 gsi_replace (&gsi, stmt, true); 13305 info.modified = true; 13306 } 13307 13308 if (info.modified) 13309 { 13310 update_stmt (stmt); 13311 if (maybe_clean_eh_stmt (stmt)) 13312 gimple_purge_dead_eh_edges (gimple_bb (stmt)); 13313 } 13314 gsi_next (&gsi); 13315 } 13316 } 13317 } 13318 13319 /* Adjust the argument types in NODE to their appropriate vector 13320 counterparts. */ 13321 13322 static void 13323 simd_clone_adjust (struct cgraph_node *node) 13324 { 13325 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 13326 13327 targetm.simd_clone.adjust (node); 13328 13329 tree retval = simd_clone_adjust_return_type (node); 13330 ipa_parm_adjustment_vec adjustments 13331 = simd_clone_adjust_argument_types (node); 13332 13333 push_gimplify_context (); 13334 13335 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments); 13336 13337 /* Adjust all uses of vector arguments accordingly. Adjust all 13338 return values accordingly. */ 13339 tree iter = create_tmp_var (unsigned_type_node, "iter"); 13340 tree iter1 = make_ssa_name (iter); 13341 tree iter2 = make_ssa_name (iter); 13342 ipa_simd_modify_function_body (node, adjustments, retval, iter1); 13343 13344 /* Initialize the iteration variable. */ 13345 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 13346 basic_block body_bb = split_block_after_labels (entry_bb)->dest; 13347 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); 13348 /* Insert the SIMD array and iv initialization at function 13349 entry. */ 13350 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT); 13351 13352 pop_gimplify_context (NULL); 13353 13354 /* Create a new BB right before the original exit BB, to hold the 13355 iteration increment and the condition/branch. */ 13356 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src; 13357 basic_block incr_bb = create_empty_bb (orig_exit); 13358 add_bb_to_loop (incr_bb, body_bb->loop_father); 13359 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty 13360 flag. Set it now to be a FALLTHRU_EDGE. */ 13361 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1); 13362 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU; 13363 for (unsigned i = 0; 13364 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i) 13365 { 13366 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i); 13367 redirect_edge_succ (e, incr_bb); 13368 } 13369 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); 13370 e->probability = REG_BR_PROB_BASE; 13371 gsi = gsi_last_bb (incr_bb); 13372 gimple g = gimple_build_assign (iter2, PLUS_EXPR, iter1, 13373 build_int_cst (unsigned_type_node, 1)); 13374 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13375 13376 /* Mostly annotate the loop for the vectorizer (the rest is done below). */ 13377 struct loop *loop = alloc_loop (); 13378 cfun->has_force_vectorize_loops = true; 13379 loop->safelen = node->simdclone->simdlen; 13380 loop->force_vectorize = true; 13381 loop->header = body_bb; 13382 13383 /* Branch around the body if the mask applies. */ 13384 if (node->simdclone->inbranch) 13385 { 13386 gimple_stmt_iterator gsi = gsi_last_bb (loop->header); 13387 tree mask_array 13388 = node->simdclone->args[node->simdclone->nargs - 1].simd_array; 13389 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); 13390 tree aref = build4 (ARRAY_REF, 13391 TREE_TYPE (TREE_TYPE (mask_array)), 13392 mask_array, iter1, 13393 NULL, NULL); 13394 g = gimple_build_assign (mask, aref); 13395 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13396 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref))); 13397 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref))) 13398 { 13399 aref = build1 (VIEW_CONVERT_EXPR, 13400 build_nonstandard_integer_type (bitsize, 0), mask); 13401 mask = make_ssa_name (TREE_TYPE (aref)); 13402 g = gimple_build_assign (mask, aref); 13403 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13404 } 13405 13406 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)), 13407 NULL, NULL); 13408 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13409 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE); 13410 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE; 13411 } 13412 13413 /* Generate the condition. */ 13414 g = gimple_build_cond (LT_EXPR, 13415 iter2, 13416 build_int_cst (unsigned_type_node, 13417 node->simdclone->simdlen), 13418 NULL, NULL); 13419 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13420 e = split_block (incr_bb, gsi_stmt (gsi)); 13421 basic_block latch_bb = e->dest; 13422 basic_block new_exit_bb; 13423 new_exit_bb = split_block (latch_bb, NULL)->dest; 13424 loop->latch = latch_bb; 13425 13426 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb); 13427 13428 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); 13429 /* The successor of incr_bb is already pointing to latch_bb; just 13430 change the flags. 13431 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */ 13432 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE; 13433 13434 gphi *phi = create_phi_node (iter1, body_bb); 13435 edge preheader_edge = find_edge (entry_bb, body_bb); 13436 edge latch_edge = single_succ_edge (latch_bb); 13437 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge, 13438 UNKNOWN_LOCATION); 13439 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 13440 13441 /* Generate the new return. */ 13442 gsi = gsi_last_bb (new_exit_bb); 13443 if (retval 13444 && TREE_CODE (retval) == VIEW_CONVERT_EXPR 13445 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL) 13446 retval = TREE_OPERAND (retval, 0); 13447 else if (retval) 13448 { 13449 retval = build1 (VIEW_CONVERT_EXPR, 13450 TREE_TYPE (TREE_TYPE (node->decl)), 13451 retval); 13452 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL, 13453 false, GSI_CONTINUE_LINKING); 13454 } 13455 g = gimple_build_return (retval); 13456 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13457 13458 /* Handle aligned clauses by replacing default defs of the aligned 13459 uniform args with __builtin_assume_aligned (arg_N(D), alignment) 13460 lhs. Handle linear by adding PHIs. */ 13461 for (unsigned i = 0; i < node->simdclone->nargs; i++) 13462 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 13463 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg) 13464 || !is_gimple_reg_type 13465 (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 13466 { 13467 tree orig_arg = node->simdclone->args[i].orig_arg; 13468 if (is_gimple_reg_type (TREE_TYPE (orig_arg))) 13469 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 13470 else 13471 { 13472 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg)); 13473 gimple_add_tmp_var (iter1); 13474 } 13475 gsi = gsi_after_labels (entry_bb); 13476 g = gimple_build_assign (iter1, orig_arg); 13477 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13478 gsi = gsi_after_labels (body_bb); 13479 g = gimple_build_assign (orig_arg, iter1); 13480 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13481 } 13482 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 13483 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg) 13484 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 13485 == REFERENCE_TYPE 13486 && TREE_ADDRESSABLE 13487 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 13488 { 13489 tree orig_arg = node->simdclone->args[i].orig_arg; 13490 tree def = ssa_default_def (cfun, orig_arg); 13491 if (def && !has_zero_uses (def)) 13492 { 13493 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg))); 13494 gimple_add_tmp_var (iter1); 13495 gsi = gsi_after_labels (entry_bb); 13496 g = gimple_build_assign (iter1, build_simple_mem_ref (def)); 13497 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13498 gsi = gsi_after_labels (body_bb); 13499 g = gimple_build_assign (build_simple_mem_ref (def), iter1); 13500 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13501 } 13502 } 13503 else if (node->simdclone->args[i].alignment 13504 && node->simdclone->args[i].arg_type 13505 == SIMD_CLONE_ARG_TYPE_UNIFORM 13506 && (node->simdclone->args[i].alignment 13507 & (node->simdclone->args[i].alignment - 1)) == 0 13508 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 13509 == POINTER_TYPE) 13510 { 13511 unsigned int alignment = node->simdclone->args[i].alignment; 13512 tree orig_arg = node->simdclone->args[i].orig_arg; 13513 tree def = ssa_default_def (cfun, orig_arg); 13514 if (def && !has_zero_uses (def)) 13515 { 13516 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 13517 gimple_seq seq = NULL; 13518 bool need_cvt = false; 13519 gcall *call 13520 = gimple_build_call (fn, 2, def, size_int (alignment)); 13521 g = call; 13522 if (!useless_type_conversion_p (TREE_TYPE (orig_arg), 13523 ptr_type_node)) 13524 need_cvt = true; 13525 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg); 13526 gimple_call_set_lhs (g, t); 13527 gimple_seq_add_stmt_without_update (&seq, g); 13528 if (need_cvt) 13529 { 13530 t = make_ssa_name (orig_arg); 13531 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g)); 13532 gimple_seq_add_stmt_without_update (&seq, g); 13533 } 13534 gsi_insert_seq_on_edge_immediate 13535 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq); 13536 13537 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 13538 int freq = compute_call_stmt_bb_frequency (current_function_decl, 13539 entry_bb); 13540 node->create_edge (cgraph_node::get_create (fn), 13541 call, entry_bb->count, freq); 13542 13543 imm_use_iterator iter; 13544 use_operand_p use_p; 13545 gimple use_stmt; 13546 tree repl = gimple_get_lhs (g); 13547 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 13548 if (is_gimple_debug (use_stmt) || use_stmt == call) 13549 continue; 13550 else 13551 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 13552 SET_USE (use_p, repl); 13553 } 13554 } 13555 else if (node->simdclone->args[i].arg_type 13556 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) 13557 { 13558 tree orig_arg = node->simdclone->args[i].orig_arg; 13559 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 13560 || POINTER_TYPE_P (TREE_TYPE (orig_arg))); 13561 tree def = NULL_TREE; 13562 if (TREE_ADDRESSABLE (orig_arg)) 13563 { 13564 def = make_ssa_name (TREE_TYPE (orig_arg)); 13565 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 13566 iter2 = make_ssa_name (TREE_TYPE (orig_arg)); 13567 gsi = gsi_after_labels (entry_bb); 13568 g = gimple_build_assign (def, orig_arg); 13569 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13570 } 13571 else 13572 { 13573 def = ssa_default_def (cfun, orig_arg); 13574 if (!def || has_zero_uses (def)) 13575 def = NULL_TREE; 13576 else 13577 { 13578 iter1 = make_ssa_name (orig_arg); 13579 iter2 = make_ssa_name (orig_arg); 13580 } 13581 } 13582 if (def) 13583 { 13584 phi = create_phi_node (iter1, body_bb); 13585 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION); 13586 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 13587 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 13588 ? PLUS_EXPR : POINTER_PLUS_EXPR; 13589 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 13590 ? TREE_TYPE (orig_arg) : sizetype; 13591 tree addcst 13592 = build_int_cst (addtype, node->simdclone->args[i].linear_step); 13593 g = gimple_build_assign (iter2, code, iter1, addcst); 13594 gsi = gsi_last_bb (incr_bb); 13595 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 13596 13597 imm_use_iterator iter; 13598 use_operand_p use_p; 13599 gimple use_stmt; 13600 if (TREE_ADDRESSABLE (orig_arg)) 13601 { 13602 gsi = gsi_after_labels (body_bb); 13603 g = gimple_build_assign (orig_arg, iter1); 13604 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13605 } 13606 else 13607 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 13608 if (use_stmt == phi) 13609 continue; 13610 else 13611 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 13612 SET_USE (use_p, iter1); 13613 } 13614 } 13615 13616 calculate_dominance_info (CDI_DOMINATORS); 13617 add_loop (loop, loop->header->loop_father); 13618 update_ssa (TODO_update_ssa); 13619 13620 pop_cfun (); 13621 } 13622 13623 /* If the function in NODE is tagged as an elemental SIMD function, 13624 create the appropriate SIMD clones. */ 13625 13626 static void 13627 expand_simd_clones (struct cgraph_node *node) 13628 { 13629 tree attr = lookup_attribute ("omp declare simd", 13630 DECL_ATTRIBUTES (node->decl)); 13631 if (attr == NULL_TREE 13632 || node->global.inlined_to 13633 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl))) 13634 return; 13635 13636 /* Ignore 13637 #pragma omp declare simd 13638 extern int foo (); 13639 in C, there we don't know the argument types at all. */ 13640 if (!node->definition 13641 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE) 13642 return; 13643 13644 /* Call this before creating clone_info, as it might ggc_collect. */ 13645 if (node->definition && node->has_gimple_body_p ()) 13646 node->get_body (); 13647 13648 do 13649 { 13650 /* Start with parsing the "omp declare simd" attribute(s). */ 13651 bool inbranch_clause_specified; 13652 struct cgraph_simd_clone *clone_info 13653 = simd_clone_clauses_extract (node, TREE_VALUE (attr), 13654 &inbranch_clause_specified); 13655 if (clone_info == NULL) 13656 continue; 13657 13658 int orig_simdlen = clone_info->simdlen; 13659 tree base_type = simd_clone_compute_base_data_type (node, clone_info); 13660 /* The target can return 0 (no simd clones should be created), 13661 1 (just one ISA of simd clones should be created) or higher 13662 count of ISA variants. In that case, clone_info is initialized 13663 for the first ISA variant. */ 13664 int count 13665 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info, 13666 base_type, 0); 13667 if (count == 0) 13668 continue; 13669 13670 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED, 13671 also create one inbranch and one !inbranch clone of it. */ 13672 for (int i = 0; i < count * 2; i++) 13673 { 13674 struct cgraph_simd_clone *clone = clone_info; 13675 if (inbranch_clause_specified && (i & 1) != 0) 13676 continue; 13677 13678 if (i != 0) 13679 { 13680 clone = simd_clone_struct_alloc (clone_info->nargs 13681 + ((i & 1) != 0)); 13682 simd_clone_struct_copy (clone, clone_info); 13683 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen 13684 and simd_clone_adjust_argument_types did to the first 13685 clone's info. */ 13686 clone->nargs -= clone_info->inbranch; 13687 clone->simdlen = orig_simdlen; 13688 /* And call the target hook again to get the right ISA. */ 13689 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone, 13690 base_type, 13691 i / 2); 13692 if ((i & 1) != 0) 13693 clone->inbranch = 1; 13694 } 13695 13696 /* simd_clone_mangle might fail if such a clone has been created 13697 already. */ 13698 tree id = simd_clone_mangle (node, clone); 13699 if (id == NULL_TREE) 13700 continue; 13701 13702 /* Only when we are sure we want to create the clone actually 13703 clone the function (or definitions) or create another 13704 extern FUNCTION_DECL (for prototypes without definitions). */ 13705 struct cgraph_node *n = simd_clone_create (node); 13706 if (n == NULL) 13707 continue; 13708 13709 n->simdclone = clone; 13710 clone->origin = node; 13711 clone->next_clone = NULL; 13712 if (node->simd_clones == NULL) 13713 { 13714 clone->prev_clone = n; 13715 node->simd_clones = n; 13716 } 13717 else 13718 { 13719 clone->prev_clone = node->simd_clones->simdclone->prev_clone; 13720 clone->prev_clone->simdclone->next_clone = n; 13721 node->simd_clones->simdclone->prev_clone = n; 13722 } 13723 symtab->change_decl_assembler_name (n->decl, id); 13724 /* And finally adjust the return type, parameters and for 13725 definitions also function body. */ 13726 if (node->definition) 13727 simd_clone_adjust (n); 13728 else 13729 { 13730 simd_clone_adjust_return_type (n); 13731 simd_clone_adjust_argument_types (n); 13732 } 13733 } 13734 } 13735 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr)))); 13736 } 13737 13738 /* Entry point for IPA simd clone creation pass. */ 13739 13740 static unsigned int 13741 ipa_omp_simd_clone (void) 13742 { 13743 struct cgraph_node *node; 13744 FOR_EACH_FUNCTION (node) 13745 expand_simd_clones (node); 13746 return 0; 13747 } 13748 13749 namespace { 13750 13751 const pass_data pass_data_omp_simd_clone = 13752 { 13753 SIMPLE_IPA_PASS, /* type */ 13754 "simdclone", /* name */ 13755 OPTGROUP_NONE, /* optinfo_flags */ 13756 TV_NONE, /* tv_id */ 13757 ( PROP_ssa | PROP_cfg ), /* properties_required */ 13758 0, /* properties_provided */ 13759 0, /* properties_destroyed */ 13760 0, /* todo_flags_start */ 13761 0, /* todo_flags_finish */ 13762 }; 13763 13764 class pass_omp_simd_clone : public simple_ipa_opt_pass 13765 { 13766 public: 13767 pass_omp_simd_clone(gcc::context *ctxt) 13768 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt) 13769 {} 13770 13771 /* opt_pass methods: */ 13772 virtual bool gate (function *); 13773 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); } 13774 }; 13775 13776 bool 13777 pass_omp_simd_clone::gate (function *) 13778 { 13779 return ((flag_openmp || flag_openmp_simd 13780 || flag_cilkplus 13781 || (in_lto_p && !flag_wpa)) 13782 && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL)); 13783 } 13784 13785 } // anon namespace 13786 13787 simple_ipa_opt_pass * 13788 make_pass_omp_simd_clone (gcc::context *ctxt) 13789 { 13790 return new pass_omp_simd_clone (ctxt); 13791 } 13792 13793 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and 13794 adds their addresses and sizes to constructor-vector V_CTOR. */ 13795 static void 13796 add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls, 13797 vec<constructor_elt, va_gc> *v_ctor) 13798 { 13799 unsigned len = vec_safe_length (v_decls); 13800 for (unsigned i = 0; i < len; i++) 13801 { 13802 tree it = (*v_decls)[i]; 13803 bool is_function = TREE_CODE (it) != VAR_DECL; 13804 13805 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, build_fold_addr_expr (it)); 13806 if (!is_function) 13807 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, 13808 fold_convert (const_ptr_type_node, 13809 DECL_SIZE_UNIT (it))); 13810 } 13811 } 13812 13813 /* Create new symbols containing (address, size) pairs for global variables, 13814 marked with "omp declare target" attribute, as well as addresses for the 13815 functions, which are outlined offloading regions. */ 13816 void 13817 omp_finish_file (void) 13818 { 13819 unsigned num_funcs = vec_safe_length (offload_funcs); 13820 unsigned num_vars = vec_safe_length (offload_vars); 13821 13822 if (num_funcs == 0 && num_vars == 0) 13823 return; 13824 13825 if (targetm_common.have_named_sections) 13826 { 13827 vec<constructor_elt, va_gc> *v_f, *v_v; 13828 vec_alloc (v_f, num_funcs); 13829 vec_alloc (v_v, num_vars * 2); 13830 13831 add_decls_addresses_to_decl_constructor (offload_funcs, v_f); 13832 add_decls_addresses_to_decl_constructor (offload_vars, v_v); 13833 13834 tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node, 13835 num_vars * 2); 13836 tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node, 13837 num_funcs); 13838 TYPE_ALIGN (vars_decl_type) = TYPE_ALIGN (pointer_sized_int_node); 13839 TYPE_ALIGN (funcs_decl_type) = TYPE_ALIGN (pointer_sized_int_node); 13840 tree ctor_v = build_constructor (vars_decl_type, v_v); 13841 tree ctor_f = build_constructor (funcs_decl_type, v_f); 13842 TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1; 13843 TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1; 13844 tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL, 13845 get_identifier (".offload_func_table"), 13846 funcs_decl_type); 13847 tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL, 13848 get_identifier (".offload_var_table"), 13849 vars_decl_type); 13850 TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1; 13851 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node), 13852 otherwise a joint table in a binary will contain padding between 13853 tables from multiple object files. */ 13854 DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (vars_decl) = 1; 13855 DECL_ALIGN (funcs_decl) = TYPE_ALIGN (funcs_decl_type); 13856 DECL_ALIGN (vars_decl) = TYPE_ALIGN (vars_decl_type); 13857 DECL_INITIAL (funcs_decl) = ctor_f; 13858 DECL_INITIAL (vars_decl) = ctor_v; 13859 set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME); 13860 set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME); 13861 13862 varpool_node::finalize_decl (vars_decl); 13863 varpool_node::finalize_decl (funcs_decl); 13864 } 13865 else 13866 { 13867 for (unsigned i = 0; i < num_funcs; i++) 13868 { 13869 tree it = (*offload_funcs)[i]; 13870 targetm.record_offload_symbol (it); 13871 } 13872 for (unsigned i = 0; i < num_vars; i++) 13873 { 13874 tree it = (*offload_vars)[i]; 13875 targetm.record_offload_symbol (it); 13876 } 13877 } 13878 } 13879 13880 #include "gt-omp-low.h" 13881