1 /* A pass for lowering trees to RTL.
2 Copyright (C) 2004-2021 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
9 any later version.
10
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "tree-pass.h"
30 #include "memmodel.h"
31 #include "tm_p.h"
32 #include "ssa.h"
33 #include "optabs.h"
34 #include "regs.h" /* For reg_renumber. */
35 #include "emit-rtl.h"
36 #include "recog.h"
37 #include "cgraph.h"
38 #include "diagnostic.h"
39 #include "fold-const.h"
40 #include "varasm.h"
41 #include "stor-layout.h"
42 #include "stmt.h"
43 #include "print-tree.h"
44 #include "cfgrtl.h"
45 #include "cfganal.h"
46 #include "cfgbuild.h"
47 #include "cfgcleanup.h"
48 #include "dojump.h"
49 #include "explow.h"
50 #include "calls.h"
51 #include "expr.h"
52 #include "internal-fn.h"
53 #include "tree-eh.h"
54 #include "gimple-iterator.h"
55 #include "gimple-expr.h"
56 #include "gimple-walk.h"
57 #include "tree-cfg.h"
58 #include "tree-dfa.h"
59 #include "tree-ssa.h"
60 #include "except.h"
61 #include "gimple-pretty-print.h"
62 #include "toplev.h"
63 #include "debug.h"
64 #include "tree-inline.h"
65 #include "value-prof.h"
66 #include "tree-ssa-live.h"
67 #include "tree-outof-ssa.h"
68 #include "cfgloop.h"
69 #include "insn-attr.h" /* For INSN_SCHEDULING. */
70 #include "stringpool.h"
71 #include "attribs.h"
72 #include "asan.h"
73 #include "tree-ssa-address.h"
74 #include "output.h"
75 #include "builtins.h"
76 #include "opts.h"
77
78 /* Some systems use __main in a way incompatible with its use in gcc, in these
79 cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
80 give the same symbol without quotes for an alternative entry point. You
81 must define both, or neither. */
82 #ifndef NAME__MAIN
83 #define NAME__MAIN "__main"
84 #endif
85
86 /* This variable holds information helping the rewriting of SSA trees
87 into RTL. */
88 struct ssaexpand SA;
89
90 /* This variable holds the currently expanded gimple statement for purposes
91 of comminucating the profile info to the builtin expanders. */
92 gimple *currently_expanding_gimple_stmt;
93
94 static rtx expand_debug_expr (tree);
95
96 static bool defer_stack_allocation (tree, bool);
97
98 static void record_alignment_for_reg_var (unsigned int);
99
100 /* Return an expression tree corresponding to the RHS of GIMPLE
101 statement STMT. */
102
103 tree
gimple_assign_rhs_to_tree(gimple * stmt)104 gimple_assign_rhs_to_tree (gimple *stmt)
105 {
106 tree t;
107 switch (gimple_assign_rhs_class (stmt))
108 {
109 case GIMPLE_TERNARY_RHS:
110 t = build3 (gimple_assign_rhs_code (stmt),
111 TREE_TYPE (gimple_assign_lhs (stmt)),
112 gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt),
113 gimple_assign_rhs3 (stmt));
114 break;
115 case GIMPLE_BINARY_RHS:
116 t = build2 (gimple_assign_rhs_code (stmt),
117 TREE_TYPE (gimple_assign_lhs (stmt)),
118 gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt));
119 break;
120 case GIMPLE_UNARY_RHS:
121 t = build1 (gimple_assign_rhs_code (stmt),
122 TREE_TYPE (gimple_assign_lhs (stmt)),
123 gimple_assign_rhs1 (stmt));
124 break;
125 case GIMPLE_SINGLE_RHS:
126 {
127 t = gimple_assign_rhs1 (stmt);
128 /* Avoid modifying this tree in place below. */
129 if ((gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t)
130 && gimple_location (stmt) != EXPR_LOCATION (t))
131 || (gimple_block (stmt) && currently_expanding_to_rtl
132 && EXPR_P (t)))
133 t = copy_node (t);
134 break;
135 }
136 default:
137 gcc_unreachable ();
138 }
139
140 if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t))
141 SET_EXPR_LOCATION (t, gimple_location (stmt));
142
143 return t;
144 }
145
146
147 #ifndef STACK_ALIGNMENT_NEEDED
148 #define STACK_ALIGNMENT_NEEDED 1
149 #endif
150
151 #define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
152
153 /* Choose either CUR or NEXT as the leader DECL for a partition.
154 Prefer ignored decls, to simplify debug dumps and reduce ambiguity
155 out of the same user variable being in multiple partitions (this is
156 less likely for compiler-introduced temps). */
157
158 static tree
leader_merge(tree cur,tree next)159 leader_merge (tree cur, tree next)
160 {
161 if (cur == NULL || cur == next)
162 return next;
163
164 if (DECL_P (cur) && DECL_IGNORED_P (cur))
165 return cur;
166
167 if (DECL_P (next) && DECL_IGNORED_P (next))
168 return next;
169
170 return cur;
171 }
172
173 /* Associate declaration T with storage space X. If T is no
174 SSA name this is exactly SET_DECL_RTL, otherwise make the
175 partition of T associated with X. */
176 static inline void
set_rtl(tree t,rtx x)177 set_rtl (tree t, rtx x)
178 {
179 gcc_checking_assert (!x
180 || !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
181 || (use_register_for_decl (t)
182 ? (REG_P (x)
183 || (GET_CODE (x) == CONCAT
184 && (REG_P (XEXP (x, 0))
185 || SUBREG_P (XEXP (x, 0)))
186 && (REG_P (XEXP (x, 1))
187 || SUBREG_P (XEXP (x, 1))))
188 /* We need to accept PARALLELs for RESUT_DECLs
189 because of vector types with BLKmode returned
190 in multiple registers, but they are supposed
191 to be uncoalesced. */
192 || (GET_CODE (x) == PARALLEL
193 && SSAVAR (t)
194 && TREE_CODE (SSAVAR (t)) == RESULT_DECL
195 && (GET_MODE (x) == BLKmode
196 || !flag_tree_coalesce_vars)))
197 : (MEM_P (x) || x == pc_rtx
198 || (GET_CODE (x) == CONCAT
199 && MEM_P (XEXP (x, 0))
200 && MEM_P (XEXP (x, 1))))));
201 /* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
202 RESULT_DECLs has the expected mode. For memory, we accept
203 unpromoted modes, since that's what we're likely to get. For
204 PARM_DECLs and RESULT_DECLs, we'll have been called by
205 set_parm_rtl, which will give us the default def, so we don't
206 have to compute it ourselves. For RESULT_DECLs, we accept mode
207 mismatches too, as long as we have BLKmode or are not coalescing
208 across variables, so that we don't reject BLKmode PARALLELs or
209 unpromoted REGs. */
210 gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
211 || (SSAVAR (t)
212 && TREE_CODE (SSAVAR (t)) == RESULT_DECL
213 && (promote_ssa_mode (t, NULL) == BLKmode
214 || !flag_tree_coalesce_vars))
215 || !use_register_for_decl (t)
216 || GET_MODE (x) == promote_ssa_mode (t, NULL));
217
218 if (x)
219 {
220 bool skip = false;
221 tree cur = NULL_TREE;
222 rtx xm = x;
223
224 retry:
225 if (MEM_P (xm))
226 cur = MEM_EXPR (xm);
227 else if (REG_P (xm))
228 cur = REG_EXPR (xm);
229 else if (SUBREG_P (xm))
230 {
231 gcc_assert (subreg_lowpart_p (xm));
232 xm = SUBREG_REG (xm);
233 goto retry;
234 }
235 else if (GET_CODE (xm) == CONCAT)
236 {
237 xm = XEXP (xm, 0);
238 goto retry;
239 }
240 else if (GET_CODE (xm) == PARALLEL)
241 {
242 xm = XVECEXP (xm, 0, 0);
243 gcc_assert (GET_CODE (xm) == EXPR_LIST);
244 xm = XEXP (xm, 0);
245 goto retry;
246 }
247 else if (xm == pc_rtx)
248 skip = true;
249 else
250 gcc_unreachable ();
251
252 tree next = skip ? cur : leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
253
254 if (cur != next)
255 {
256 if (MEM_P (x))
257 set_mem_attributes (x,
258 next && TREE_CODE (next) == SSA_NAME
259 ? TREE_TYPE (next)
260 : next, true);
261 else
262 set_reg_attrs_for_decl_rtl (next, x);
263 }
264 }
265
266 if (TREE_CODE (t) == SSA_NAME)
267 {
268 int part = var_to_partition (SA.map, t);
269 if (part != NO_PARTITION)
270 {
271 if (SA.partition_to_pseudo[part])
272 gcc_assert (SA.partition_to_pseudo[part] == x);
273 else if (x != pc_rtx)
274 SA.partition_to_pseudo[part] = x;
275 }
276 /* For the benefit of debug information at -O0 (where
277 vartracking doesn't run) record the place also in the base
278 DECL. For PARMs and RESULTs, do so only when setting the
279 default def. */
280 if (x && x != pc_rtx && SSA_NAME_VAR (t)
281 && (VAR_P (SSA_NAME_VAR (t))
282 || SSA_NAME_IS_DEFAULT_DEF (t)))
283 {
284 tree var = SSA_NAME_VAR (t);
285 /* If we don't yet have something recorded, just record it now. */
286 if (!DECL_RTL_SET_P (var))
287 SET_DECL_RTL (var, x);
288 /* If we have it set already to "multiple places" don't
289 change this. */
290 else if (DECL_RTL (var) == pc_rtx)
291 ;
292 /* If we have something recorded and it's not the same place
293 as we want to record now, we have multiple partitions for the
294 same base variable, with different places. We can't just
295 randomly chose one, hence we have to say that we don't know.
296 This only happens with optimization, and there var-tracking
297 will figure out the right thing. */
298 else if (DECL_RTL (var) != x)
299 SET_DECL_RTL (var, pc_rtx);
300 }
301 }
302 else
303 SET_DECL_RTL (t, x);
304 }
305
306 /* This structure holds data relevant to one variable that will be
307 placed in a stack slot. */
308 class stack_var
309 {
310 public:
311 /* The Variable. */
312 tree decl;
313
314 /* Initially, the size of the variable. Later, the size of the partition,
315 if this variable becomes it's partition's representative. */
316 poly_uint64 size;
317
318 /* The *byte* alignment required for this variable. Or as, with the
319 size, the alignment for this partition. */
320 unsigned int alignb;
321
322 /* The partition representative. */
323 size_t representative;
324
325 /* The next stack variable in the partition, or EOC. */
326 size_t next;
327
328 /* The numbers of conflicting stack variables. */
329 bitmap conflicts;
330 };
331
332 #define EOC ((size_t)-1)
333
334 /* We have an array of such objects while deciding allocation. */
335 static class stack_var *stack_vars;
336 static size_t stack_vars_alloc;
337 static size_t stack_vars_num;
338 static hash_map<tree, size_t> *decl_to_stack_part;
339
340 /* Conflict bitmaps go on this obstack. This allows us to destroy
341 all of them in one big sweep. */
342 static bitmap_obstack stack_var_bitmap_obstack;
343
344 /* An array of indices such that stack_vars[stack_vars_sorted[i]].size
345 is non-decreasing. */
346 static size_t *stack_vars_sorted;
347
348 /* The phase of the stack frame. This is the known misalignment of
349 virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
350 (frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
351 static int frame_phase;
352
353 /* Used during expand_used_vars to remember if we saw any decls for
354 which we'd like to enable stack smashing protection. */
355 static bool has_protected_decls;
356
357 /* Used during expand_used_vars. Remember if we say a character buffer
358 smaller than our cutoff threshold. Used for -Wstack-protector. */
359 static bool has_short_buffer;
360
361 /* Compute the byte alignment to use for DECL. Ignore alignment
362 we can't do with expected alignment of the stack boundary. */
363
364 static unsigned int
align_local_variable(tree decl,bool really_expand)365 align_local_variable (tree decl, bool really_expand)
366 {
367 unsigned int align;
368
369 if (TREE_CODE (decl) == SSA_NAME)
370 {
371 tree type = TREE_TYPE (decl);
372 machine_mode mode = TYPE_MODE (type);
373
374 align = TYPE_ALIGN (type);
375 if (mode != BLKmode
376 && align < GET_MODE_ALIGNMENT (mode))
377 align = GET_MODE_ALIGNMENT (mode);
378 }
379 else
380 align = LOCAL_DECL_ALIGNMENT (decl);
381
382 if (hwasan_sanitize_stack_p ())
383 align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
384
385 if (TREE_CODE (decl) != SSA_NAME && really_expand)
386 /* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
387 That is done before IPA and could bump alignment based on host
388 backend even for offloaded code which wants different
389 LOCAL_DECL_ALIGNMENT. */
390 SET_DECL_ALIGN (decl, align);
391
392 return align / BITS_PER_UNIT;
393 }
394
395 /* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
396 down otherwise. Return truncated BASE value. */
397
398 static inline unsigned HOST_WIDE_INT
align_base(HOST_WIDE_INT base,unsigned HOST_WIDE_INT align,bool align_up)399 align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
400 {
401 return align_up ? (base + align - 1) & -align : base & -align;
402 }
403
404 /* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
405 Return the frame offset. */
406
407 static poly_int64
alloc_stack_frame_space(poly_int64 size,unsigned HOST_WIDE_INT align)408 alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
409 {
410 poly_int64 offset, new_frame_offset;
411
412 if (FRAME_GROWS_DOWNWARD)
413 {
414 new_frame_offset
415 = aligned_lower_bound (frame_offset - frame_phase - size,
416 align) + frame_phase;
417 offset = new_frame_offset;
418 }
419 else
420 {
421 new_frame_offset
422 = aligned_upper_bound (frame_offset - frame_phase,
423 align) + frame_phase;
424 offset = new_frame_offset;
425 new_frame_offset += size;
426 }
427 frame_offset = new_frame_offset;
428
429 if (frame_offset_overflow (frame_offset, cfun->decl))
430 frame_offset = offset = 0;
431
432 return offset;
433 }
434
435 /* Ensure that the stack is aligned to ALIGN bytes.
436 Return the new frame offset. */
437 static poly_int64
align_frame_offset(unsigned HOST_WIDE_INT align)438 align_frame_offset (unsigned HOST_WIDE_INT align)
439 {
440 return alloc_stack_frame_space (0, align);
441 }
442
443 /* Accumulate DECL into STACK_VARS. */
444
445 static void
add_stack_var(tree decl,bool really_expand)446 add_stack_var (tree decl, bool really_expand)
447 {
448 class stack_var *v;
449
450 if (stack_vars_num >= stack_vars_alloc)
451 {
452 if (stack_vars_alloc)
453 stack_vars_alloc = stack_vars_alloc * 3 / 2;
454 else
455 stack_vars_alloc = 32;
456 stack_vars
457 = XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
458 }
459 if (!decl_to_stack_part)
460 decl_to_stack_part = new hash_map<tree, size_t>;
461
462 v = &stack_vars[stack_vars_num];
463 decl_to_stack_part->put (decl, stack_vars_num);
464
465 v->decl = decl;
466 tree size = TREE_CODE (decl) == SSA_NAME
467 ? TYPE_SIZE_UNIT (TREE_TYPE (decl))
468 : DECL_SIZE_UNIT (decl);
469 v->size = tree_to_poly_uint64 (size);
470 /* Ensure that all variables have size, so that &a != &b for any two
471 variables that are simultaneously live. */
472 if (known_eq (v->size, 0U))
473 v->size = 1;
474 v->alignb = align_local_variable (decl, really_expand);
475 /* An alignment of zero can mightily confuse us later. */
476 gcc_assert (v->alignb != 0);
477
478 /* All variables are initially in their own partition. */
479 v->representative = stack_vars_num;
480 v->next = EOC;
481
482 /* All variables initially conflict with no other. */
483 v->conflicts = NULL;
484
485 /* Ensure that this decl doesn't get put onto the list twice. */
486 set_rtl (decl, pc_rtx);
487
488 stack_vars_num++;
489 }
490
491 /* Make the decls associated with luid's X and Y conflict. */
492
493 static void
add_stack_var_conflict(size_t x,size_t y)494 add_stack_var_conflict (size_t x, size_t y)
495 {
496 class stack_var *a = &stack_vars[x];
497 class stack_var *b = &stack_vars[y];
498 if (x == y)
499 return;
500 if (!a->conflicts)
501 a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
502 if (!b->conflicts)
503 b->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
504 bitmap_set_bit (a->conflicts, y);
505 bitmap_set_bit (b->conflicts, x);
506 }
507
508 /* Check whether the decls associated with luid's X and Y conflict. */
509
510 static bool
stack_var_conflict_p(size_t x,size_t y)511 stack_var_conflict_p (size_t x, size_t y)
512 {
513 class stack_var *a = &stack_vars[x];
514 class stack_var *b = &stack_vars[y];
515 if (x == y)
516 return false;
517 /* Partitions containing an SSA name result from gimple registers
518 with things like unsupported modes. They are top-level and
519 hence conflict with everything else. */
520 if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
521 return true;
522
523 if (!a->conflicts || !b->conflicts)
524 return false;
525 return bitmap_bit_p (a->conflicts, y);
526 }
527
528 /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
529 enter its partition number into bitmap DATA. */
530
531 static bool
visit_op(gimple *,tree op,tree,void * data)532 visit_op (gimple *, tree op, tree, void *data)
533 {
534 bitmap active = (bitmap)data;
535 op = get_base_address (op);
536 if (op
537 && DECL_P (op)
538 && DECL_RTL_IF_SET (op) == pc_rtx)
539 {
540 size_t *v = decl_to_stack_part->get (op);
541 if (v)
542 bitmap_set_bit (active, *v);
543 }
544 return false;
545 }
546
547 /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
548 record conflicts between it and all currently active other partitions
549 from bitmap DATA. */
550
551 static bool
visit_conflict(gimple *,tree op,tree,void * data)552 visit_conflict (gimple *, tree op, tree, void *data)
553 {
554 bitmap active = (bitmap)data;
555 op = get_base_address (op);
556 if (op
557 && DECL_P (op)
558 && DECL_RTL_IF_SET (op) == pc_rtx)
559 {
560 size_t *v = decl_to_stack_part->get (op);
561 if (v && bitmap_set_bit (active, *v))
562 {
563 size_t num = *v;
564 bitmap_iterator bi;
565 unsigned i;
566 gcc_assert (num < stack_vars_num);
567 EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
568 add_stack_var_conflict (num, i);
569 }
570 }
571 return false;
572 }
573
574 /* Helper routine for add_scope_conflicts, calculating the active partitions
575 at the end of BB, leaving the result in WORK. We're called to generate
576 conflicts when FOR_CONFLICT is true, otherwise we're just tracking
577 liveness. */
578
579 static void
add_scope_conflicts_1(basic_block bb,bitmap work,bool for_conflict)580 add_scope_conflicts_1 (basic_block bb, bitmap work, bool for_conflict)
581 {
582 edge e;
583 edge_iterator ei;
584 gimple_stmt_iterator gsi;
585 walk_stmt_load_store_addr_fn visit;
586
587 bitmap_clear (work);
588 FOR_EACH_EDGE (e, ei, bb->preds)
589 bitmap_ior_into (work, (bitmap)e->src->aux);
590
591 visit = visit_op;
592
593 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
594 {
595 gimple *stmt = gsi_stmt (gsi);
596 walk_stmt_load_store_addr_ops (stmt, work, NULL, NULL, visit);
597 }
598 for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
599 {
600 gimple *stmt = gsi_stmt (gsi);
601
602 if (gimple_clobber_p (stmt))
603 {
604 tree lhs = gimple_assign_lhs (stmt);
605 size_t *v;
606 /* Nested function lowering might introduce LHSs
607 that are COMPONENT_REFs. */
608 if (!VAR_P (lhs))
609 continue;
610 if (DECL_RTL_IF_SET (lhs) == pc_rtx
611 && (v = decl_to_stack_part->get (lhs)))
612 bitmap_clear_bit (work, *v);
613 }
614 else if (!is_gimple_debug (stmt))
615 {
616 if (for_conflict
617 && visit == visit_op)
618 {
619 /* If this is the first real instruction in this BB we need
620 to add conflicts for everything live at this point now.
621 Unlike classical liveness for named objects we can't
622 rely on seeing a def/use of the names we're interested in.
623 There might merely be indirect loads/stores. We'd not add any
624 conflicts for such partitions. */
625 bitmap_iterator bi;
626 unsigned i;
627 EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
628 {
629 class stack_var *a = &stack_vars[i];
630 if (!a->conflicts)
631 a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
632 bitmap_ior_into (a->conflicts, work);
633 }
634 visit = visit_conflict;
635 }
636 walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
637 }
638 }
639 }
640
641 /* Generate stack partition conflicts between all partitions that are
642 simultaneously live. */
643
644 static void
add_scope_conflicts(void)645 add_scope_conflicts (void)
646 {
647 basic_block bb;
648 bool changed;
649 bitmap work = BITMAP_ALLOC (NULL);
650 int *rpo;
651 int n_bbs;
652
653 /* We approximate the live range of a stack variable by taking the first
654 mention of its name as starting point(s), and by the end-of-scope
655 death clobber added by gimplify as ending point(s) of the range.
656 This overapproximates in the case we for instance moved an address-taken
657 operation upward, without also moving a dereference to it upwards.
658 But it's conservatively correct as a variable never can hold values
659 before its name is mentioned at least once.
660
661 We then do a mostly classical bitmap liveness algorithm. */
662
663 FOR_ALL_BB_FN (bb, cfun)
664 bb->aux = BITMAP_ALLOC (&stack_var_bitmap_obstack);
665
666 rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
667 n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
668
669 changed = true;
670 while (changed)
671 {
672 int i;
673 changed = false;
674 for (i = 0; i < n_bbs; i++)
675 {
676 bitmap active;
677 bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
678 active = (bitmap)bb->aux;
679 add_scope_conflicts_1 (bb, work, false);
680 if (bitmap_ior_into (active, work))
681 changed = true;
682 }
683 }
684
685 FOR_EACH_BB_FN (bb, cfun)
686 add_scope_conflicts_1 (bb, work, true);
687
688 free (rpo);
689 BITMAP_FREE (work);
690 FOR_ALL_BB_FN (bb, cfun)
691 BITMAP_FREE (bb->aux);
692 }
693
694 /* A subroutine of partition_stack_vars. A comparison function for qsort,
695 sorting an array of indices by the properties of the object. */
696
697 static int
stack_var_cmp(const void * a,const void * b)698 stack_var_cmp (const void *a, const void *b)
699 {
700 size_t ia = *(const size_t *)a;
701 size_t ib = *(const size_t *)b;
702 unsigned int aligna = stack_vars[ia].alignb;
703 unsigned int alignb = stack_vars[ib].alignb;
704 poly_int64 sizea = stack_vars[ia].size;
705 poly_int64 sizeb = stack_vars[ib].size;
706 tree decla = stack_vars[ia].decl;
707 tree declb = stack_vars[ib].decl;
708 bool largea, largeb;
709 unsigned int uida, uidb;
710
711 /* Primary compare on "large" alignment. Large comes first. */
712 largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
713 largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
714 if (largea != largeb)
715 return (int)largeb - (int)largea;
716
717 /* Secondary compare on size, decreasing */
718 int diff = compare_sizes_for_sort (sizeb, sizea);
719 if (diff != 0)
720 return diff;
721
722 /* Tertiary compare on true alignment, decreasing. */
723 if (aligna < alignb)
724 return -1;
725 if (aligna > alignb)
726 return 1;
727
728 /* Final compare on ID for sort stability, increasing.
729 Two SSA names are compared by their version, SSA names come before
730 non-SSA names, and two normal decls are compared by their DECL_UID. */
731 if (TREE_CODE (decla) == SSA_NAME)
732 {
733 if (TREE_CODE (declb) == SSA_NAME)
734 uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
735 else
736 return -1;
737 }
738 else if (TREE_CODE (declb) == SSA_NAME)
739 return 1;
740 else
741 uida = DECL_UID (decla), uidb = DECL_UID (declb);
742 if (uida < uidb)
743 return 1;
744 if (uida > uidb)
745 return -1;
746 return 0;
747 }
748
749 struct part_traits : unbounded_int_hashmap_traits <size_t, bitmap> {};
750 typedef hash_map<size_t, bitmap, part_traits> part_hashmap;
751
752 /* If the points-to solution *PI points to variables that are in a partition
753 together with other variables add all partition members to the pointed-to
754 variables bitmap. */
755
756 static void
add_partitioned_vars_to_ptset(struct pt_solution * pt,part_hashmap * decls_to_partitions,hash_set<bitmap> * visited,bitmap temp)757 add_partitioned_vars_to_ptset (struct pt_solution *pt,
758 part_hashmap *decls_to_partitions,
759 hash_set<bitmap> *visited, bitmap temp)
760 {
761 bitmap_iterator bi;
762 unsigned i;
763 bitmap *part;
764
765 if (pt->anything
766 || pt->vars == NULL
767 /* The pointed-to vars bitmap is shared, it is enough to
768 visit it once. */
769 || visited->add (pt->vars))
770 return;
771
772 bitmap_clear (temp);
773
774 /* By using a temporary bitmap to store all members of the partitions
775 we have to add we make sure to visit each of the partitions only
776 once. */
777 EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
778 if ((!temp
779 || !bitmap_bit_p (temp, i))
780 && (part = decls_to_partitions->get (i)))
781 bitmap_ior_into (temp, *part);
782 if (!bitmap_empty_p (temp))
783 bitmap_ior_into (pt->vars, temp);
784 }
785
786 /* Update points-to sets based on partition info, so we can use them on RTL.
787 The bitmaps representing stack partitions will be saved until expand,
788 where partitioned decls used as bases in memory expressions will be
789 rewritten. */
790
791 static void
update_alias_info_with_stack_vars(void)792 update_alias_info_with_stack_vars (void)
793 {
794 part_hashmap *decls_to_partitions = NULL;
795 size_t i, j;
796 tree var = NULL_TREE;
797
798 for (i = 0; i < stack_vars_num; i++)
799 {
800 bitmap part = NULL;
801 tree name;
802 struct ptr_info_def *pi;
803
804 /* Not interested in partitions with single variable. */
805 if (stack_vars[i].representative != i
806 || stack_vars[i].next == EOC)
807 continue;
808
809 if (!decls_to_partitions)
810 {
811 decls_to_partitions = new part_hashmap;
812 cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
813 }
814
815 /* Create an SSA_NAME that points to the partition for use
816 as base during alias-oracle queries on RTL for bases that
817 have been partitioned. */
818 if (var == NULL_TREE)
819 var = create_tmp_var (ptr_type_node);
820 name = make_ssa_name (var);
821
822 /* Create bitmaps representing partitions. They will be used for
823 points-to sets later, so use GGC alloc. */
824 part = BITMAP_GGC_ALLOC ();
825 for (j = i; j != EOC; j = stack_vars[j].next)
826 {
827 tree decl = stack_vars[j].decl;
828 unsigned int uid = DECL_PT_UID (decl);
829 bitmap_set_bit (part, uid);
830 decls_to_partitions->put (uid, part);
831 cfun->gimple_df->decls_to_pointers->put (decl, name);
832 if (TREE_ADDRESSABLE (decl))
833 TREE_ADDRESSABLE (name) = 1;
834 }
835
836 /* Make the SSA name point to all partition members. */
837 pi = get_ptr_info (name);
838 pt_solution_set (&pi->pt, part, false);
839 }
840
841 /* Make all points-to sets that contain one member of a partition
842 contain all members of the partition. */
843 if (decls_to_partitions)
844 {
845 unsigned i;
846 tree name;
847 hash_set<bitmap> visited;
848 bitmap temp = BITMAP_ALLOC (&stack_var_bitmap_obstack);
849
850 FOR_EACH_SSA_NAME (i, name, cfun)
851 {
852 struct ptr_info_def *pi;
853
854 if (POINTER_TYPE_P (TREE_TYPE (name))
855 && ((pi = SSA_NAME_PTR_INFO (name)) != NULL))
856 add_partitioned_vars_to_ptset (&pi->pt, decls_to_partitions,
857 &visited, temp);
858 }
859
860 add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped,
861 decls_to_partitions, &visited, temp);
862
863 delete decls_to_partitions;
864 BITMAP_FREE (temp);
865 }
866 }
867
868 /* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
869 partitioning algorithm. Partitions A and B are known to be non-conflicting.
870 Merge them into a single partition A. */
871
872 static void
union_stack_vars(size_t a,size_t b)873 union_stack_vars (size_t a, size_t b)
874 {
875 class stack_var *vb = &stack_vars[b];
876 bitmap_iterator bi;
877 unsigned u;
878
879 gcc_assert (stack_vars[b].next == EOC);
880 /* Add B to A's partition. */
881 stack_vars[b].next = stack_vars[a].next;
882 stack_vars[b].representative = a;
883 stack_vars[a].next = b;
884
885 /* Make sure A is big enough to hold B. */
886 stack_vars[a].size = upper_bound (stack_vars[a].size, stack_vars[b].size);
887
888 /* Update the required alignment of partition A to account for B. */
889 if (stack_vars[a].alignb < stack_vars[b].alignb)
890 stack_vars[a].alignb = stack_vars[b].alignb;
891
892 /* Update the interference graph and merge the conflicts. */
893 if (vb->conflicts)
894 {
895 EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
896 add_stack_var_conflict (a, stack_vars[u].representative);
897 BITMAP_FREE (vb->conflicts);
898 }
899 }
900
901 /* A subroutine of expand_used_vars. Binpack the variables into
902 partitions constrained by the interference graph. The overall
903 algorithm used is as follows:
904
905 Sort the objects by size in descending order.
906 For each object A {
907 S = size(A)
908 O = 0
909 loop {
910 Look for the largest non-conflicting object B with size <= S.
911 UNION (A, B)
912 }
913 }
914 */
915
916 static void
partition_stack_vars(void)917 partition_stack_vars (void)
918 {
919 size_t si, sj, n = stack_vars_num;
920
921 stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
922 for (si = 0; si < n; ++si)
923 stack_vars_sorted[si] = si;
924
925 if (n == 1)
926 return;
927
928 qsort (stack_vars_sorted, n, sizeof (size_t), stack_var_cmp);
929
930 for (si = 0; si < n; ++si)
931 {
932 size_t i = stack_vars_sorted[si];
933 unsigned int ialign = stack_vars[i].alignb;
934 poly_int64 isize = stack_vars[i].size;
935
936 /* Ignore objects that aren't partition representatives. If we
937 see a var that is not a partition representative, it must
938 have been merged earlier. */
939 if (stack_vars[i].representative != i)
940 continue;
941
942 for (sj = si + 1; sj < n; ++sj)
943 {
944 size_t j = stack_vars_sorted[sj];
945 unsigned int jalign = stack_vars[j].alignb;
946 poly_int64 jsize = stack_vars[j].size;
947
948 /* Ignore objects that aren't partition representatives. */
949 if (stack_vars[j].representative != j)
950 continue;
951
952 /* Do not mix objects of "small" (supported) alignment
953 and "large" (unsupported) alignment. */
954 if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
955 != (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
956 break;
957
958 /* For Address Sanitizer do not mix objects with different
959 sizes, as the shorter vars wouldn't be adequately protected.
960 Don't do that for "large" (unsupported) alignment objects,
961 those aren't protected anyway. */
962 if (asan_sanitize_stack_p ()
963 && maybe_ne (isize, jsize)
964 && ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
965 break;
966
967 /* Ignore conflicting objects. */
968 if (stack_var_conflict_p (i, j))
969 continue;
970
971 /* UNION the objects, placing J at OFFSET. */
972 union_stack_vars (i, j);
973 }
974 }
975
976 update_alias_info_with_stack_vars ();
977 }
978
979 /* A debugging aid for expand_used_vars. Dump the generated partitions. */
980
981 static void
dump_stack_var_partition(void)982 dump_stack_var_partition (void)
983 {
984 size_t si, i, j, n = stack_vars_num;
985
986 for (si = 0; si < n; ++si)
987 {
988 i = stack_vars_sorted[si];
989
990 /* Skip variables that aren't partition representatives, for now. */
991 if (stack_vars[i].representative != i)
992 continue;
993
994 fprintf (dump_file, "Partition %lu: size ", (unsigned long) i);
995 print_dec (stack_vars[i].size, dump_file);
996 fprintf (dump_file, " align %u\n", stack_vars[i].alignb);
997
998 for (j = i; j != EOC; j = stack_vars[j].next)
999 {
1000 fputc ('\t', dump_file);
1001 print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1002 }
1003 fputc ('\n', dump_file);
1004 }
1005 }
1006
1007 /* Assign rtl to DECL at BASE + OFFSET. */
1008
1009 static void
expand_one_stack_var_at(tree decl,rtx base,unsigned base_align,poly_int64 offset)1010 expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1011 poly_int64 offset)
1012 {
1013 unsigned align;
1014 rtx x;
1015
1016 /* If this fails, we've overflowed the stack frame. Error nicely? */
1017 gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1018
1019 if (hwasan_sanitize_stack_p ())
1020 x = targetm.memtag.add_tag (base, offset,
1021 hwasan_current_frame_tag ());
1022 else
1023 x = plus_constant (Pmode, base, offset);
1024
1025 x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1026 ? TYPE_MODE (TREE_TYPE (decl))
1027 : DECL_MODE (decl), x);
1028
1029 /* Set alignment we actually gave this decl if it isn't an SSA name.
1030 If it is we generate stack slots only accidentally so it isn't as
1031 important, we'll simply set the alignment directly on the MEM. */
1032
1033 if (stack_vars_base_reg_p (base))
1034 offset -= frame_phase;
1035 align = known_alignment (offset);
1036 align *= BITS_PER_UNIT;
1037 if (align == 0 || align > base_align)
1038 align = base_align;
1039
1040 if (TREE_CODE (decl) != SSA_NAME)
1041 {
1042 /* One would think that we could assert that we're not decreasing
1043 alignment here, but (at least) the i386 port does exactly this
1044 via the MINIMUM_ALIGNMENT hook. */
1045
1046 SET_DECL_ALIGN (decl, align);
1047 DECL_USER_ALIGN (decl) = 0;
1048 }
1049
1050 set_rtl (decl, x);
1051
1052 set_mem_align (x, align);
1053 }
1054
1055 class stack_vars_data
1056 {
1057 public:
1058 /* Vector of offset pairs, always end of some padding followed
1059 by start of the padding that needs Address Sanitizer protection.
1060 The vector is in reversed, highest offset pairs come first. */
1061 auto_vec<HOST_WIDE_INT> asan_vec;
1062
1063 /* Vector of partition representative decls in between the paddings. */
1064 auto_vec<tree> asan_decl_vec;
1065
1066 /* Base pseudo register for Address Sanitizer protected automatic vars. */
1067 rtx asan_base;
1068
1069 /* Alignment needed for the Address Sanitizer protected automatic vars. */
1070 unsigned int asan_alignb;
1071 };
1072
1073 /* A subroutine of expand_used_vars. Give each partition representative
1074 a unique location within the stack frame. Update each partition member
1075 with that location. */
1076 static void
expand_stack_vars(bool (* pred)(size_t),class stack_vars_data * data)1077 expand_stack_vars (bool (*pred) (size_t), class stack_vars_data *data)
1078 {
1079 size_t si, i, j, n = stack_vars_num;
1080 poly_uint64 large_size = 0, large_alloc = 0;
1081 rtx large_base = NULL;
1082 rtx large_untagged_base = NULL;
1083 unsigned large_align = 0;
1084 bool large_allocation_done = false;
1085 tree decl;
1086
1087 /* Determine if there are any variables requiring "large" alignment.
1088 Since these are dynamically allocated, we only process these if
1089 no predicate involved. */
1090 large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1091 if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1092 {
1093 /* Find the total size of these variables. */
1094 for (si = 0; si < n; ++si)
1095 {
1096 unsigned alignb;
1097
1098 i = stack_vars_sorted[si];
1099 alignb = stack_vars[i].alignb;
1100
1101 /* All "large" alignment decls come before all "small" alignment
1102 decls, but "large" alignment decls are not sorted based on
1103 their alignment. Increase large_align to track the largest
1104 required alignment. */
1105 if ((alignb * BITS_PER_UNIT) > large_align)
1106 large_align = alignb * BITS_PER_UNIT;
1107
1108 /* Stop when we get to the first decl with "small" alignment. */
1109 if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1110 break;
1111
1112 /* Skip variables that aren't partition representatives. */
1113 if (stack_vars[i].representative != i)
1114 continue;
1115
1116 /* Skip variables that have already had rtl assigned. See also
1117 add_stack_var where we perpetrate this pc_rtx hack. */
1118 decl = stack_vars[i].decl;
1119 if (TREE_CODE (decl) == SSA_NAME
1120 ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1121 : DECL_RTL (decl) != pc_rtx)
1122 continue;
1123
1124 large_size = aligned_upper_bound (large_size, alignb);
1125 large_size += stack_vars[i].size;
1126 }
1127 }
1128
1129 for (si = 0; si < n; ++si)
1130 {
1131 rtx base;
1132 unsigned base_align, alignb;
1133 poly_int64 offset = 0;
1134
1135 i = stack_vars_sorted[si];
1136
1137 /* Skip variables that aren't partition representatives, for now. */
1138 if (stack_vars[i].representative != i)
1139 continue;
1140
1141 /* Skip variables that have already had rtl assigned. See also
1142 add_stack_var where we perpetrate this pc_rtx hack. */
1143 decl = stack_vars[i].decl;
1144 if (TREE_CODE (decl) == SSA_NAME
1145 ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1146 : DECL_RTL (decl) != pc_rtx)
1147 continue;
1148
1149 /* Check the predicate to see whether this variable should be
1150 allocated in this pass. */
1151 if (pred && !pred (i))
1152 continue;
1153
1154 base = (hwasan_sanitize_stack_p ()
1155 ? hwasan_frame_base ()
1156 : virtual_stack_vars_rtx);
1157 alignb = stack_vars[i].alignb;
1158 if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1159 {
1160 poly_int64 hwasan_orig_offset;
1161 if (hwasan_sanitize_stack_p ())
1162 {
1163 /* There must be no tag granule "shared" between different
1164 objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1165 chunk can have more than one object in it.
1166
1167 We ensure this by forcing the end of the last bit of data to
1168 be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1169 the start of each variable to be aligned to
1170 HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1171
1172 We can't align just one of the start or end, since there are
1173 untagged things stored on the stack which we do not align to
1174 HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1175 or the end of tagged objects then untagged objects could end
1176 up sharing the first granule of a tagged object or sharing the
1177 last granule of a tagged object respectively. */
1178 hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1179 gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1180 }
1181 /* ASAN description strings don't yet have a syntax for expressing
1182 polynomial offsets. */
1183 HOST_WIDE_INT prev_offset;
1184 if (asan_sanitize_stack_p ()
1185 && pred
1186 && frame_offset.is_constant (&prev_offset)
1187 && stack_vars[i].size.is_constant ())
1188 {
1189 if (data->asan_vec.is_empty ())
1190 {
1191 align_frame_offset (ASAN_RED_ZONE_SIZE);
1192 prev_offset = frame_offset.to_constant ();
1193 }
1194 prev_offset = align_base (prev_offset,
1195 ASAN_MIN_RED_ZONE_SIZE,
1196 !FRAME_GROWS_DOWNWARD);
1197 tree repr_decl = NULL_TREE;
1198 unsigned HOST_WIDE_INT size
1199 = asan_var_and_redzone_size (stack_vars[i].size.to_constant ());
1200 if (data->asan_vec.is_empty ())
1201 size = MAX (size, ASAN_RED_ZONE_SIZE);
1202
1203 unsigned HOST_WIDE_INT alignment = MAX (alignb,
1204 ASAN_MIN_RED_ZONE_SIZE);
1205 offset = alloc_stack_frame_space (size, alignment);
1206
1207 data->asan_vec.safe_push (prev_offset);
1208 /* Allocating a constant amount of space from a constant
1209 starting offset must give a constant result. */
1210 data->asan_vec.safe_push ((offset + stack_vars[i].size)
1211 .to_constant ());
1212 /* Find best representative of the partition.
1213 Prefer those with DECL_NAME, even better
1214 satisfying asan_protect_stack_decl predicate. */
1215 for (j = i; j != EOC; j = stack_vars[j].next)
1216 if (asan_protect_stack_decl (stack_vars[j].decl)
1217 && DECL_NAME (stack_vars[j].decl))
1218 {
1219 repr_decl = stack_vars[j].decl;
1220 break;
1221 }
1222 else if (repr_decl == NULL_TREE
1223 && DECL_P (stack_vars[j].decl)
1224 && DECL_NAME (stack_vars[j].decl))
1225 repr_decl = stack_vars[j].decl;
1226 if (repr_decl == NULL_TREE)
1227 repr_decl = stack_vars[i].decl;
1228 data->asan_decl_vec.safe_push (repr_decl);
1229
1230 /* Make sure a representative is unpoison if another
1231 variable in the partition is handled by
1232 use-after-scope sanitization. */
1233 if (asan_handled_variables != NULL
1234 && !asan_handled_variables->contains (repr_decl))
1235 {
1236 for (j = i; j != EOC; j = stack_vars[j].next)
1237 if (asan_handled_variables->contains (stack_vars[j].decl))
1238 break;
1239 if (j != EOC)
1240 asan_handled_variables->add (repr_decl);
1241 }
1242
1243 data->asan_alignb = MAX (data->asan_alignb, alignb);
1244 if (data->asan_base == NULL)
1245 data->asan_base = gen_reg_rtx (Pmode);
1246 base = data->asan_base;
1247
1248 if (!STRICT_ALIGNMENT)
1249 base_align = crtl->max_used_stack_slot_alignment;
1250 else
1251 base_align = MAX (crtl->max_used_stack_slot_alignment,
1252 GET_MODE_ALIGNMENT (SImode)
1253 << ASAN_SHADOW_SHIFT);
1254 }
1255 else
1256 {
1257 offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
1258 base_align = crtl->max_used_stack_slot_alignment;
1259
1260 if (hwasan_sanitize_stack_p ())
1261 {
1262 /* Align again since the point of this alignment is to handle
1263 the "end" of the object (i.e. smallest address after the
1264 stack object). For FRAME_GROWS_DOWNWARD that requires
1265 aligning the stack before allocating, but for a frame that
1266 grows upwards that requires aligning the stack after
1267 allocation.
1268
1269 Use `frame_offset` to record the offset value rather than
1270 `offset` since the `frame_offset` describes the extent
1271 allocated for this particular variable while `offset`
1272 describes the address that this variable starts at. */
1273 align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1274 hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1275 hwasan_orig_offset, frame_offset);
1276 }
1277 }
1278 }
1279 else
1280 {
1281 /* Large alignment is only processed in the last pass. */
1282 if (pred)
1283 continue;
1284
1285 /* If there were any variables requiring "large" alignment, allocate
1286 space. */
1287 if (maybe_ne (large_size, 0U) && ! large_allocation_done)
1288 {
1289 poly_int64 loffset;
1290 rtx large_allocsize;
1291
1292 large_allocsize = gen_int_mode (large_size, Pmode);
1293 get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1294 loffset = alloc_stack_frame_space
1295 (rtx_to_poly_int64 (large_allocsize),
1296 PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1297 large_base = get_dynamic_stack_base (loffset, large_align, base);
1298 large_allocation_done = true;
1299 }
1300
1301 gcc_assert (large_base != NULL);
1302 large_alloc = aligned_upper_bound (large_alloc, alignb);
1303 offset = large_alloc;
1304 large_alloc += stack_vars[i].size;
1305 if (hwasan_sanitize_stack_p ())
1306 {
1307 /* An object with a large alignment requirement means that the
1308 alignment requirement is greater than the required alignment
1309 for tags. */
1310 if (!large_untagged_base)
1311 large_untagged_base
1312 = targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1313 /* Ensure the end of the variable is also aligned correctly. */
1314 poly_int64 align_again
1315 = aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
1316 /* For large allocations we always allocate a chunk of space
1317 (which is addressed by large_untagged_base/large_base) and
1318 then use positive offsets from that. Hence the farthest
1319 offset is `align_again` and the nearest offset from the base
1320 is `offset`. */
1321 hwasan_record_stack_var (large_untagged_base, large_base,
1322 offset, align_again);
1323 }
1324
1325 base = large_base;
1326 base_align = large_align;
1327 }
1328
1329 /* Create rtl for each variable based on their location within the
1330 partition. */
1331 for (j = i; j != EOC; j = stack_vars[j].next)
1332 {
1333 expand_one_stack_var_at (stack_vars[j].decl,
1334 base, base_align, offset);
1335 }
1336 if (hwasan_sanitize_stack_p ())
1337 hwasan_increment_frame_tag ();
1338 }
1339
1340 gcc_assert (known_eq (large_alloc, large_size));
1341 }
1342
1343 /* Take into account all sizes of partitions and reset DECL_RTLs. */
1344 static poly_uint64
account_stack_vars(void)1345 account_stack_vars (void)
1346 {
1347 size_t si, j, i, n = stack_vars_num;
1348 poly_uint64 size = 0;
1349
1350 for (si = 0; si < n; ++si)
1351 {
1352 i = stack_vars_sorted[si];
1353
1354 /* Skip variables that aren't partition representatives, for now. */
1355 if (stack_vars[i].representative != i)
1356 continue;
1357
1358 size += stack_vars[i].size;
1359 for (j = i; j != EOC; j = stack_vars[j].next)
1360 set_rtl (stack_vars[j].decl, NULL);
1361 }
1362 return size;
1363 }
1364
1365 /* Record the RTL assignment X for the default def of PARM. */
1366
1367 extern void
set_parm_rtl(tree parm,rtx x)1368 set_parm_rtl (tree parm, rtx x)
1369 {
1370 gcc_assert (TREE_CODE (parm) == PARM_DECL
1371 || TREE_CODE (parm) == RESULT_DECL);
1372
1373 if (x && !MEM_P (x))
1374 {
1375 unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1376 TYPE_MODE (TREE_TYPE (parm)),
1377 TYPE_ALIGN (TREE_TYPE (parm)));
1378
1379 /* If the variable alignment is very large we'll dynamicaly
1380 allocate it, which means that in-frame portion is just a
1381 pointer. ??? We've got a pseudo for sure here, do we
1382 actually dynamically allocate its spilling area if needed?
1383 ??? Isn't it a problem when Pmode alignment also exceeds
1384 MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1385 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1386 align = GET_MODE_ALIGNMENT (Pmode);
1387
1388 record_alignment_for_reg_var (align);
1389 }
1390
1391 tree ssa = ssa_default_def (cfun, parm);
1392 if (!ssa)
1393 return set_rtl (parm, x);
1394
1395 int part = var_to_partition (SA.map, ssa);
1396 gcc_assert (part != NO_PARTITION);
1397
1398 bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1399 gcc_assert (changed);
1400
1401 set_rtl (ssa, x);
1402 gcc_assert (DECL_RTL (parm) == x);
1403 }
1404
1405 /* A subroutine of expand_one_var. Called to immediately assign rtl
1406 to a variable to be allocated in the stack frame. */
1407
1408 static void
expand_one_stack_var_1(tree var)1409 expand_one_stack_var_1 (tree var)
1410 {
1411 poly_uint64 size;
1412 poly_int64 offset;
1413 unsigned byte_align;
1414
1415 if (TREE_CODE (var) == SSA_NAME)
1416 {
1417 tree type = TREE_TYPE (var);
1418 size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1419 }
1420 else
1421 size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1422
1423 byte_align = align_local_variable (var, true);
1424
1425 /* We handle highly aligned variables in expand_stack_vars. */
1426 gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1427
1428 rtx base;
1429 if (hwasan_sanitize_stack_p ())
1430 {
1431 /* Allocate zero bytes to align the stack. */
1432 poly_int64 hwasan_orig_offset
1433 = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1434 offset = alloc_stack_frame_space (size, byte_align);
1435 align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1436 base = hwasan_frame_base ();
1437 /* Use `frame_offset` to automatically account for machines where the
1438 frame grows upwards.
1439
1440 `offset` will always point to the "start" of the stack object, which
1441 will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1442 the "furthest" offset from the base delimiting the current stack
1443 object. `frame_offset` will always delimit the extent that the frame.
1444 */
1445 hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1446 hwasan_orig_offset, frame_offset);
1447 }
1448 else
1449 {
1450 offset = alloc_stack_frame_space (size, byte_align);
1451 base = virtual_stack_vars_rtx;
1452 }
1453
1454 expand_one_stack_var_at (var, base,
1455 crtl->max_used_stack_slot_alignment, offset);
1456
1457 if (hwasan_sanitize_stack_p ())
1458 hwasan_increment_frame_tag ();
1459 }
1460
1461 /* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1462 already assigned some MEM. */
1463
1464 static void
expand_one_stack_var(tree var)1465 expand_one_stack_var (tree var)
1466 {
1467 if (TREE_CODE (var) == SSA_NAME)
1468 {
1469 int part = var_to_partition (SA.map, var);
1470 if (part != NO_PARTITION)
1471 {
1472 rtx x = SA.partition_to_pseudo[part];
1473 gcc_assert (x);
1474 gcc_assert (MEM_P (x));
1475 return;
1476 }
1477 }
1478
1479 return expand_one_stack_var_1 (var);
1480 }
1481
1482 /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1483 that will reside in a hard register. */
1484
1485 static void
expand_one_hard_reg_var(tree var)1486 expand_one_hard_reg_var (tree var)
1487 {
1488 rest_of_decl_compilation (var, 0, 0);
1489 }
1490
1491 /* Record the alignment requirements of some variable assigned to a
1492 pseudo. */
1493
1494 static void
record_alignment_for_reg_var(unsigned int align)1495 record_alignment_for_reg_var (unsigned int align)
1496 {
1497 if (SUPPORTS_STACK_ALIGNMENT
1498 && crtl->stack_alignment_estimated < align)
1499 {
1500 /* stack_alignment_estimated shouldn't change after stack
1501 realign decision made */
1502 gcc_assert (!crtl->stack_realign_processed);
1503 crtl->stack_alignment_estimated = align;
1504 }
1505
1506 /* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1507 So here we only make sure stack_alignment_needed >= align. */
1508 if (crtl->stack_alignment_needed < align)
1509 crtl->stack_alignment_needed = align;
1510 if (crtl->max_used_stack_slot_alignment < align)
1511 crtl->max_used_stack_slot_alignment = align;
1512 }
1513
1514 /* Create RTL for an SSA partition. */
1515
1516 static void
expand_one_ssa_partition(tree var)1517 expand_one_ssa_partition (tree var)
1518 {
1519 int part = var_to_partition (SA.map, var);
1520 gcc_assert (part != NO_PARTITION);
1521
1522 if (SA.partition_to_pseudo[part])
1523 return;
1524
1525 unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1526 TYPE_MODE (TREE_TYPE (var)),
1527 TYPE_ALIGN (TREE_TYPE (var)));
1528
1529 /* If the variable alignment is very large we'll dynamicaly allocate
1530 it, which means that in-frame portion is just a pointer. */
1531 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1532 align = GET_MODE_ALIGNMENT (Pmode);
1533
1534 record_alignment_for_reg_var (align);
1535
1536 if (!use_register_for_decl (var))
1537 {
1538 if (defer_stack_allocation (var, true))
1539 add_stack_var (var, true);
1540 else
1541 expand_one_stack_var_1 (var);
1542 return;
1543 }
1544
1545 machine_mode reg_mode = promote_ssa_mode (var, NULL);
1546 rtx x = gen_reg_rtx (reg_mode);
1547
1548 set_rtl (var, x);
1549
1550 /* For a promoted variable, X will not be used directly but wrapped in a
1551 SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1552 will assume that its upper bits can be inferred from its lower bits.
1553 Therefore, if X isn't initialized on every path from the entry, then
1554 we must do it manually in order to fulfill the above assumption. */
1555 if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1556 && bitmap_bit_p (SA.partitions_for_undefined_values, part))
1557 emit_move_insn (x, CONST0_RTX (reg_mode));
1558 }
1559
1560 /* Record the association between the RTL generated for partition PART
1561 and the underlying variable of the SSA_NAME VAR. */
1562
1563 static void
adjust_one_expanded_partition_var(tree var)1564 adjust_one_expanded_partition_var (tree var)
1565 {
1566 if (!var)
1567 return;
1568
1569 tree decl = SSA_NAME_VAR (var);
1570
1571 int part = var_to_partition (SA.map, var);
1572 if (part == NO_PARTITION)
1573 return;
1574
1575 rtx x = SA.partition_to_pseudo[part];
1576
1577 gcc_assert (x);
1578
1579 set_rtl (var, x);
1580
1581 if (!REG_P (x))
1582 return;
1583
1584 /* Note if the object is a user variable. */
1585 if (decl && !DECL_ARTIFICIAL (decl))
1586 mark_user_reg (x);
1587
1588 if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1589 mark_reg_pointer (x, get_pointer_alignment (var));
1590 }
1591
1592 /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1593 that will reside in a pseudo register. */
1594
1595 static void
expand_one_register_var(tree var)1596 expand_one_register_var (tree var)
1597 {
1598 if (TREE_CODE (var) == SSA_NAME)
1599 {
1600 int part = var_to_partition (SA.map, var);
1601 if (part != NO_PARTITION)
1602 {
1603 rtx x = SA.partition_to_pseudo[part];
1604 gcc_assert (x);
1605 gcc_assert (REG_P (x));
1606 return;
1607 }
1608 gcc_unreachable ();
1609 }
1610
1611 tree decl = var;
1612 tree type = TREE_TYPE (decl);
1613 machine_mode reg_mode = promote_decl_mode (decl, NULL);
1614 rtx x = gen_reg_rtx (reg_mode);
1615
1616 set_rtl (var, x);
1617
1618 /* Note if the object is a user variable. */
1619 if (!DECL_ARTIFICIAL (decl))
1620 mark_user_reg (x);
1621
1622 if (POINTER_TYPE_P (type))
1623 mark_reg_pointer (x, get_pointer_alignment (var));
1624 }
1625
1626 /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1627 has some associated error, e.g. its type is error-mark. We just need
1628 to pick something that won't crash the rest of the compiler. */
1629
1630 static void
expand_one_error_var(tree var)1631 expand_one_error_var (tree var)
1632 {
1633 machine_mode mode = DECL_MODE (var);
1634 rtx x;
1635
1636 if (mode == BLKmode)
1637 x = gen_rtx_MEM (BLKmode, const0_rtx);
1638 else if (mode == VOIDmode)
1639 x = const0_rtx;
1640 else
1641 x = gen_reg_rtx (mode);
1642
1643 SET_DECL_RTL (var, x);
1644 }
1645
1646 /* A subroutine of expand_one_var. VAR is a variable that will be
1647 allocated to the local stack frame. Return true if we wish to
1648 add VAR to STACK_VARS so that it will be coalesced with other
1649 variables. Return false to allocate VAR immediately.
1650
1651 This function is used to reduce the number of variables considered
1652 for coalescing, which reduces the size of the quadratic problem. */
1653
1654 static bool
defer_stack_allocation(tree var,bool toplevel)1655 defer_stack_allocation (tree var, bool toplevel)
1656 {
1657 tree size_unit = TREE_CODE (var) == SSA_NAME
1658 ? TYPE_SIZE_UNIT (TREE_TYPE (var))
1659 : DECL_SIZE_UNIT (var);
1660 poly_uint64 size;
1661
1662 /* Whether the variable is small enough for immediate allocation not to be
1663 a problem with regard to the frame size. */
1664 bool smallish
1665 = (poly_int_tree_p (size_unit, &size)
1666 && (estimated_poly_value (size)
1667 < param_min_size_for_stack_sharing));
1668
1669 /* If stack protection is enabled, *all* stack variables must be deferred,
1670 so that we can re-order the strings to the top of the frame.
1671 Similarly for Address Sanitizer. */
1672 if (flag_stack_protect || asan_sanitize_stack_p ())
1673 return true;
1674
1675 unsigned int align = TREE_CODE (var) == SSA_NAME
1676 ? TYPE_ALIGN (TREE_TYPE (var))
1677 : DECL_ALIGN (var);
1678
1679 /* We handle "large" alignment via dynamic allocation. We want to handle
1680 this extra complication in only one place, so defer them. */
1681 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1682 return true;
1683
1684 bool ignored = TREE_CODE (var) == SSA_NAME
1685 ? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
1686 : DECL_IGNORED_P (var);
1687
1688 /* When optimization is enabled, DECL_IGNORED_P variables originally scoped
1689 might be detached from their block and appear at toplevel when we reach
1690 here. We want to coalesce them with variables from other blocks when
1691 the immediate contribution to the frame size would be noticeable. */
1692 if (toplevel && optimize > 0 && ignored && !smallish)
1693 return true;
1694
1695 /* Variables declared in the outermost scope automatically conflict
1696 with every other variable. The only reason to want to defer them
1697 at all is that, after sorting, we can more efficiently pack
1698 small variables in the stack frame. Continue to defer at -O2. */
1699 if (toplevel && optimize < 2)
1700 return false;
1701
1702 /* Without optimization, *most* variables are allocated from the
1703 stack, which makes the quadratic problem large exactly when we
1704 want compilation to proceed as quickly as possible. On the
1705 other hand, we don't want the function's stack frame size to
1706 get completely out of hand. So we avoid adding scalars and
1707 "small" aggregates to the list at all. */
1708 if (optimize == 0 && smallish)
1709 return false;
1710
1711 return true;
1712 }
1713
1714 /* A subroutine of expand_used_vars. Expand one variable according to
1715 its flavor. Variables to be placed on the stack are not actually
1716 expanded yet, merely recorded.
1717 When REALLY_EXPAND is false, only add stack values to be allocated.
1718 Return stack usage this variable is supposed to take.
1719 */
1720
1721 static poly_uint64
expand_one_var(tree var,bool toplevel,bool really_expand)1722 expand_one_var (tree var, bool toplevel, bool really_expand)
1723 {
1724 unsigned int align = BITS_PER_UNIT;
1725 tree origvar = var;
1726
1727 var = SSAVAR (var);
1728
1729 if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
1730 {
1731 if (is_global_var (var))
1732 return 0;
1733
1734 /* Because we don't know if VAR will be in register or on stack,
1735 we conservatively assume it will be on stack even if VAR is
1736 eventually put into register after RA pass. For non-automatic
1737 variables, which won't be on stack, we collect alignment of
1738 type and ignore user specified alignment. Similarly for
1739 SSA_NAMEs for which use_register_for_decl returns true. */
1740 if (TREE_STATIC (var)
1741 || DECL_EXTERNAL (var)
1742 || (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
1743 align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1744 TYPE_MODE (TREE_TYPE (var)),
1745 TYPE_ALIGN (TREE_TYPE (var)));
1746 else if (DECL_HAS_VALUE_EXPR_P (var)
1747 || (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
1748 /* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
1749 or variables which were assigned a stack slot already by
1750 expand_one_stack_var_at - in the latter case DECL_ALIGN has been
1751 changed from the offset chosen to it. */
1752 align = crtl->stack_alignment_estimated;
1753 else
1754 align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
1755
1756 /* If the variable alignment is very large we'll dynamicaly allocate
1757 it, which means that in-frame portion is just a pointer. */
1758 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1759 align = GET_MODE_ALIGNMENT (Pmode);
1760 }
1761
1762 record_alignment_for_reg_var (align);
1763
1764 poly_uint64 size;
1765 if (TREE_CODE (origvar) == SSA_NAME)
1766 {
1767 gcc_assert (!VAR_P (var)
1768 || (!DECL_EXTERNAL (var)
1769 && !DECL_HAS_VALUE_EXPR_P (var)
1770 && !TREE_STATIC (var)
1771 && TREE_TYPE (var) != error_mark_node
1772 && !DECL_HARD_REGISTER (var)
1773 && really_expand));
1774 }
1775 if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
1776 ;
1777 else if (DECL_EXTERNAL (var))
1778 ;
1779 else if (DECL_HAS_VALUE_EXPR_P (var))
1780 ;
1781 else if (TREE_STATIC (var))
1782 ;
1783 else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
1784 ;
1785 else if (TREE_TYPE (var) == error_mark_node)
1786 {
1787 if (really_expand)
1788 expand_one_error_var (var);
1789 }
1790 else if (VAR_P (var) && DECL_HARD_REGISTER (var))
1791 {
1792 if (really_expand)
1793 {
1794 expand_one_hard_reg_var (var);
1795 if (!DECL_HARD_REGISTER (var))
1796 /* Invalid register specification. */
1797 expand_one_error_var (var);
1798 }
1799 }
1800 else if (use_register_for_decl (var))
1801 {
1802 if (really_expand)
1803 expand_one_register_var (origvar);
1804 }
1805 else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), &size)
1806 || !valid_constant_size_p (DECL_SIZE_UNIT (var)))
1807 {
1808 /* Reject variables which cover more than half of the address-space. */
1809 if (really_expand)
1810 {
1811 if (DECL_NONLOCAL_FRAME (var))
1812 error_at (DECL_SOURCE_LOCATION (current_function_decl),
1813 "total size of local objects is too large");
1814 else
1815 error_at (DECL_SOURCE_LOCATION (var),
1816 "size of variable %q+D is too large", var);
1817 expand_one_error_var (var);
1818 }
1819 }
1820 else if (defer_stack_allocation (var, toplevel))
1821 add_stack_var (origvar, really_expand);
1822 else
1823 {
1824 if (really_expand)
1825 {
1826 if (lookup_attribute ("naked",
1827 DECL_ATTRIBUTES (current_function_decl)))
1828 error ("cannot allocate stack for variable %q+D, naked function",
1829 var);
1830
1831 expand_one_stack_var (origvar);
1832 }
1833 return size;
1834 }
1835 return 0;
1836 }
1837
1838 /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1839 expanding variables. Those variables that can be put into registers
1840 are allocated pseudos; those that can't are put on the stack.
1841
1842 TOPLEVEL is true if this is the outermost BLOCK. */
1843
1844 static void
expand_used_vars_for_block(tree block,bool toplevel)1845 expand_used_vars_for_block (tree block, bool toplevel)
1846 {
1847 tree t;
1848
1849 /* Expand all variables at this level. */
1850 for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1851 if (TREE_USED (t)
1852 && ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1853 || !DECL_NONSHAREABLE (t)))
1854 expand_one_var (t, toplevel, true);
1855
1856 /* Expand all variables at containing levels. */
1857 for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1858 expand_used_vars_for_block (t, false);
1859 }
1860
1861 /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1862 and clear TREE_USED on all local variables. */
1863
1864 static void
clear_tree_used(tree block)1865 clear_tree_used (tree block)
1866 {
1867 tree t;
1868
1869 for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1870 /* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
1871 if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1872 || !DECL_NONSHAREABLE (t))
1873 TREE_USED (t) = 0;
1874
1875 for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1876 clear_tree_used (t);
1877 }
1878
1879 /* Examine TYPE and determine a bit mask of the following features. */
1880
1881 #define SPCT_HAS_LARGE_CHAR_ARRAY 1
1882 #define SPCT_HAS_SMALL_CHAR_ARRAY 2
1883 #define SPCT_HAS_ARRAY 4
1884 #define SPCT_HAS_AGGREGATE 8
1885
1886 static unsigned int
stack_protect_classify_type(tree type)1887 stack_protect_classify_type (tree type)
1888 {
1889 unsigned int ret = 0;
1890 tree t;
1891
1892 switch (TREE_CODE (type))
1893 {
1894 case ARRAY_TYPE:
1895 t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
1896 if (t == char_type_node
1897 || t == signed_char_type_node
1898 || t == unsigned_char_type_node)
1899 {
1900 unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
1901 unsigned HOST_WIDE_INT len;
1902
1903 if (!TYPE_SIZE_UNIT (type)
1904 || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
1905 len = max;
1906 else
1907 len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
1908
1909 if (len < max)
1910 ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
1911 else
1912 ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
1913 }
1914 else
1915 ret = SPCT_HAS_ARRAY;
1916 break;
1917
1918 case UNION_TYPE:
1919 case QUAL_UNION_TYPE:
1920 case RECORD_TYPE:
1921 ret = SPCT_HAS_AGGREGATE;
1922 for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
1923 if (TREE_CODE (t) == FIELD_DECL)
1924 ret |= stack_protect_classify_type (TREE_TYPE (t));
1925 break;
1926
1927 default:
1928 break;
1929 }
1930
1931 return ret;
1932 }
1933
1934 /* Return nonzero if DECL should be segregated into the "vulnerable" upper
1935 part of the local stack frame. Remember if we ever return nonzero for
1936 any variable in this function. The return value is the phase number in
1937 which the variable should be allocated. */
1938
1939 static int
stack_protect_decl_phase(tree decl)1940 stack_protect_decl_phase (tree decl)
1941 {
1942 unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
1943 int ret = 0;
1944
1945 if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
1946 has_short_buffer = true;
1947
1948 tree attribs = DECL_ATTRIBUTES (current_function_decl);
1949 if (!lookup_attribute ("no_stack_protector", attribs)
1950 && (flag_stack_protect == SPCT_FLAG_ALL
1951 || flag_stack_protect == SPCT_FLAG_STRONG
1952 || (flag_stack_protect == SPCT_FLAG_EXPLICIT
1953 && lookup_attribute ("stack_protect", attribs))))
1954 {
1955 if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
1956 && !(bits & SPCT_HAS_AGGREGATE))
1957 ret = 1;
1958 else if (bits & SPCT_HAS_ARRAY)
1959 ret = 2;
1960 }
1961 else
1962 ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
1963
1964 if (ret)
1965 has_protected_decls = true;
1966
1967 return ret;
1968 }
1969
1970 /* Two helper routines that check for phase 1 and phase 2. These are used
1971 as callbacks for expand_stack_vars. */
1972
1973 static bool
stack_protect_decl_phase_1(size_t i)1974 stack_protect_decl_phase_1 (size_t i)
1975 {
1976 return stack_protect_decl_phase (stack_vars[i].decl) == 1;
1977 }
1978
1979 static bool
stack_protect_decl_phase_2(size_t i)1980 stack_protect_decl_phase_2 (size_t i)
1981 {
1982 return stack_protect_decl_phase (stack_vars[i].decl) == 2;
1983 }
1984
1985 /* And helper function that checks for asan phase (with stack protector
1986 it is phase 3). This is used as callback for expand_stack_vars.
1987 Returns true if any of the vars in the partition need to be protected. */
1988
1989 static bool
asan_decl_phase_3(size_t i)1990 asan_decl_phase_3 (size_t i)
1991 {
1992 while (i != EOC)
1993 {
1994 if (asan_protect_stack_decl (stack_vars[i].decl))
1995 return true;
1996 i = stack_vars[i].next;
1997 }
1998 return false;
1999 }
2000
2001 /* Ensure that variables in different stack protection phases conflict
2002 so that they are not merged and share the same stack slot.
2003 Return true if there are any address taken variables. */
2004
2005 static bool
add_stack_protection_conflicts(void)2006 add_stack_protection_conflicts (void)
2007 {
2008 size_t i, j, n = stack_vars_num;
2009 unsigned char *phase;
2010 bool ret = false;
2011
2012 phase = XNEWVEC (unsigned char, n);
2013 for (i = 0; i < n; ++i)
2014 {
2015 phase[i] = stack_protect_decl_phase (stack_vars[i].decl);
2016 if (TREE_ADDRESSABLE (stack_vars[i].decl))
2017 ret = true;
2018 }
2019
2020 for (i = 0; i < n; ++i)
2021 {
2022 unsigned char ph_i = phase[i];
2023 for (j = i + 1; j < n; ++j)
2024 if (ph_i != phase[j])
2025 add_stack_var_conflict (i, j);
2026 }
2027
2028 XDELETEVEC (phase);
2029 return ret;
2030 }
2031
2032 /* Create a decl for the guard at the top of the stack frame. */
2033
2034 static void
create_stack_guard(void)2035 create_stack_guard (void)
2036 {
2037 tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2038 VAR_DECL, NULL, ptr_type_node);
2039 TREE_THIS_VOLATILE (guard) = 1;
2040 TREE_USED (guard) = 1;
2041 expand_one_stack_var (guard);
2042 crtl->stack_protect_guard = guard;
2043 }
2044
2045 /* Prepare for expanding variables. */
2046 static void
init_vars_expansion(void)2047 init_vars_expansion (void)
2048 {
2049 /* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2050 bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2051
2052 /* A map from decl to stack partition. */
2053 decl_to_stack_part = new hash_map<tree, size_t>;
2054
2055 /* Initialize local stack smashing state. */
2056 has_protected_decls = false;
2057 has_short_buffer = false;
2058 if (hwasan_sanitize_stack_p ())
2059 hwasan_record_frame_init ();
2060 }
2061
2062 /* Free up stack variable graph data. */
2063 static void
fini_vars_expansion(void)2064 fini_vars_expansion (void)
2065 {
2066 bitmap_obstack_release (&stack_var_bitmap_obstack);
2067 if (stack_vars)
2068 XDELETEVEC (stack_vars);
2069 if (stack_vars_sorted)
2070 XDELETEVEC (stack_vars_sorted);
2071 stack_vars = NULL;
2072 stack_vars_sorted = NULL;
2073 stack_vars_alloc = stack_vars_num = 0;
2074 delete decl_to_stack_part;
2075 decl_to_stack_part = NULL;
2076 }
2077
2078 /* Make a fair guess for the size of the stack frame of the function
2079 in NODE. This doesn't have to be exact, the result is only used in
2080 the inline heuristics. So we don't want to run the full stack var
2081 packing algorithm (which is quadratic in the number of stack vars).
2082 Instead, we calculate the total size of all stack vars. This turns
2083 out to be a pretty fair estimate -- packing of stack vars doesn't
2084 happen very often. */
2085
2086 HOST_WIDE_INT
estimated_stack_frame_size(struct cgraph_node * node)2087 estimated_stack_frame_size (struct cgraph_node *node)
2088 {
2089 poly_int64 size = 0;
2090 size_t i;
2091 tree var;
2092 struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2093
2094 push_cfun (fn);
2095
2096 init_vars_expansion ();
2097
2098 FOR_EACH_LOCAL_DECL (fn, i, var)
2099 if (auto_var_in_fn_p (var, fn->decl))
2100 size += expand_one_var (var, true, false);
2101
2102 if (stack_vars_num > 0)
2103 {
2104 /* Fake sorting the stack vars for account_stack_vars (). */
2105 stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
2106 for (i = 0; i < stack_vars_num; ++i)
2107 stack_vars_sorted[i] = i;
2108 size += account_stack_vars ();
2109 }
2110
2111 fini_vars_expansion ();
2112 pop_cfun ();
2113 return estimated_poly_value (size);
2114 }
2115
2116 /* Check if the current function has calls that use a return slot. */
2117
2118 static bool
stack_protect_return_slot_p()2119 stack_protect_return_slot_p ()
2120 {
2121 basic_block bb;
2122
2123 FOR_ALL_BB_FN (bb, cfun)
2124 for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2125 !gsi_end_p (gsi); gsi_next (&gsi))
2126 {
2127 gimple *stmt = gsi_stmt (gsi);
2128 /* This assumes that calls to internal-only functions never
2129 use a return slot. */
2130 if (is_gimple_call (stmt)
2131 && !gimple_call_internal_p (stmt)
2132 && aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2133 gimple_call_fndecl (stmt)))
2134 return true;
2135 }
2136 return false;
2137 }
2138
2139 /* Expand all variables used in the function. */
2140
2141 static rtx_insn *
expand_used_vars(void)2142 expand_used_vars (void)
2143 {
2144 tree var, outer_block = DECL_INITIAL (current_function_decl);
2145 auto_vec<tree> maybe_local_decls;
2146 rtx_insn *var_end_seq = NULL;
2147 unsigned i;
2148 unsigned len;
2149 bool gen_stack_protect_signal = false;
2150
2151 /* Compute the phase of the stack frame for this function. */
2152 {
2153 int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2154 int off = targetm.starting_frame_offset () % align;
2155 frame_phase = off ? align - off : 0;
2156 }
2157
2158 /* Set TREE_USED on all variables in the local_decls. */
2159 FOR_EACH_LOCAL_DECL (cfun, i, var)
2160 TREE_USED (var) = 1;
2161 /* Clear TREE_USED on all variables associated with a block scope. */
2162 clear_tree_used (DECL_INITIAL (current_function_decl));
2163
2164 init_vars_expansion ();
2165
2166 if (targetm.use_pseudo_pic_reg ())
2167 pic_offset_table_rtx = gen_reg_rtx (Pmode);
2168
2169 for (i = 0; i < SA.map->num_partitions; i++)
2170 {
2171 if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2172 continue;
2173
2174 tree var = partition_to_var (SA.map, i);
2175
2176 gcc_assert (!virtual_operand_p (var));
2177
2178 expand_one_ssa_partition (var);
2179 }
2180
2181 if (flag_stack_protect == SPCT_FLAG_STRONG)
2182 gen_stack_protect_signal = stack_protect_return_slot_p ();
2183
2184 /* At this point all variables on the local_decls with TREE_USED
2185 set are not associated with any block scope. Lay them out. */
2186
2187 len = vec_safe_length (cfun->local_decls);
2188 FOR_EACH_LOCAL_DECL (cfun, i, var)
2189 {
2190 bool expand_now = false;
2191
2192 /* Expanded above already. */
2193 if (is_gimple_reg (var))
2194 {
2195 TREE_USED (var) = 0;
2196 goto next;
2197 }
2198 /* We didn't set a block for static or extern because it's hard
2199 to tell the difference between a global variable (re)declared
2200 in a local scope, and one that's really declared there to
2201 begin with. And it doesn't really matter much, since we're
2202 not giving them stack space. Expand them now. */
2203 else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2204 expand_now = true;
2205
2206 /* Expand variables not associated with any block now. Those created by
2207 the optimizers could be live anywhere in the function. Those that
2208 could possibly have been scoped originally and detached from their
2209 block will have their allocation deferred so we coalesce them with
2210 others when optimization is enabled. */
2211 else if (TREE_USED (var))
2212 expand_now = true;
2213
2214 /* Finally, mark all variables on the list as used. We'll use
2215 this in a moment when we expand those associated with scopes. */
2216 TREE_USED (var) = 1;
2217
2218 if (expand_now)
2219 expand_one_var (var, true, true);
2220
2221 next:
2222 if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2223 {
2224 rtx rtl = DECL_RTL_IF_SET (var);
2225
2226 /* Keep artificial non-ignored vars in cfun->local_decls
2227 chain until instantiate_decls. */
2228 if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2229 add_local_decl (cfun, var);
2230 else if (rtl == NULL_RTX)
2231 /* If rtl isn't set yet, which can happen e.g. with
2232 -fstack-protector, retry before returning from this
2233 function. */
2234 maybe_local_decls.safe_push (var);
2235 }
2236 }
2237
2238 /* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2239
2240 +-----------------+-----------------+
2241 | ...processed... | ...duplicates...|
2242 +-----------------+-----------------+
2243 ^
2244 +-- LEN points here.
2245
2246 We just want the duplicates, as those are the artificial
2247 non-ignored vars that we want to keep until instantiate_decls.
2248 Move them down and truncate the array. */
2249 if (!vec_safe_is_empty (cfun->local_decls))
2250 cfun->local_decls->block_remove (0, len);
2251
2252 /* At this point, all variables within the block tree with TREE_USED
2253 set are actually used by the optimized function. Lay them out. */
2254 expand_used_vars_for_block (outer_block, true);
2255
2256 tree attribs = DECL_ATTRIBUTES (current_function_decl);
2257 if (stack_vars_num > 0)
2258 {
2259 bool has_addressable_vars = false;
2260
2261 add_scope_conflicts ();
2262
2263 /* If stack protection is enabled, we don't share space between
2264 vulnerable data and non-vulnerable data. */
2265 if (flag_stack_protect != 0
2266 && !lookup_attribute ("no_stack_protector", attribs)
2267 && (flag_stack_protect != SPCT_FLAG_EXPLICIT
2268 || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2269 && lookup_attribute ("stack_protect", attribs))))
2270 has_addressable_vars = add_stack_protection_conflicts ();
2271
2272 if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2273 gen_stack_protect_signal = true;
2274
2275 /* Now that we have collected all stack variables, and have computed a
2276 minimal interference graph, attempt to save some stack space. */
2277 partition_stack_vars ();
2278 if (dump_file)
2279 dump_stack_var_partition ();
2280 }
2281
2282
2283 if (!lookup_attribute ("no_stack_protector", attribs))
2284 switch (flag_stack_protect)
2285 {
2286 case SPCT_FLAG_ALL:
2287 create_stack_guard ();
2288 break;
2289
2290 case SPCT_FLAG_STRONG:
2291 if (gen_stack_protect_signal
2292 || cfun->calls_alloca
2293 || has_protected_decls
2294 || lookup_attribute ("stack_protect",
2295 DECL_ATTRIBUTES (current_function_decl)))
2296 create_stack_guard ();
2297 break;
2298
2299 case SPCT_FLAG_DEFAULT:
2300 if (cfun->calls_alloca
2301 || has_protected_decls
2302 || lookup_attribute ("stack_protect",
2303 DECL_ATTRIBUTES (current_function_decl)))
2304 create_stack_guard ();
2305 break;
2306
2307 case SPCT_FLAG_EXPLICIT:
2308 if (lookup_attribute ("stack_protect",
2309 DECL_ATTRIBUTES (current_function_decl)))
2310 create_stack_guard ();
2311 break;
2312
2313 default:
2314 break;
2315 }
2316
2317 /* Assign rtl to each variable based on these partitions. */
2318 if (stack_vars_num > 0)
2319 {
2320 class stack_vars_data data;
2321
2322 data.asan_base = NULL_RTX;
2323 data.asan_alignb = 0;
2324
2325 /* Reorder decls to be protected by iterating over the variables
2326 array multiple times, and allocating out of each phase in turn. */
2327 /* ??? We could probably integrate this into the qsort we did
2328 earlier, such that we naturally see these variables first,
2329 and thus naturally allocate things in the right order. */
2330 if (has_protected_decls)
2331 {
2332 /* Phase 1 contains only character arrays. */
2333 expand_stack_vars (stack_protect_decl_phase_1, &data);
2334
2335 /* Phase 2 contains other kinds of arrays. */
2336 if (!lookup_attribute ("no_stack_protector", attribs)
2337 && (flag_stack_protect == SPCT_FLAG_ALL
2338 || flag_stack_protect == SPCT_FLAG_STRONG
2339 || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2340 && lookup_attribute ("stack_protect", attribs))))
2341 expand_stack_vars (stack_protect_decl_phase_2, &data);
2342 }
2343
2344 if (asan_sanitize_stack_p ())
2345 /* Phase 3, any partitions that need asan protection
2346 in addition to phase 1 and 2. */
2347 expand_stack_vars (asan_decl_phase_3, &data);
2348
2349 /* ASAN description strings don't yet have a syntax for expressing
2350 polynomial offsets. */
2351 HOST_WIDE_INT prev_offset;
2352 if (!data.asan_vec.is_empty ()
2353 && frame_offset.is_constant (&prev_offset))
2354 {
2355 HOST_WIDE_INT offset, sz, redzonesz;
2356 redzonesz = ASAN_RED_ZONE_SIZE;
2357 sz = data.asan_vec[0] - prev_offset;
2358 if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2359 && data.asan_alignb <= 4096
2360 && sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2361 redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2362 & ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2363 /* Allocating a constant amount of space from a constant
2364 starting offset must give a constant result. */
2365 offset = (alloc_stack_frame_space (redzonesz, ASAN_RED_ZONE_SIZE)
2366 .to_constant ());
2367 data.asan_vec.safe_push (prev_offset);
2368 data.asan_vec.safe_push (offset);
2369 /* Leave space for alignment if STRICT_ALIGNMENT. */
2370 if (STRICT_ALIGNMENT)
2371 alloc_stack_frame_space ((GET_MODE_ALIGNMENT (SImode)
2372 << ASAN_SHADOW_SHIFT)
2373 / BITS_PER_UNIT, 1);
2374
2375 var_end_seq
2376 = asan_emit_stack_protection (virtual_stack_vars_rtx,
2377 data.asan_base,
2378 data.asan_alignb,
2379 data.asan_vec.address (),
2380 data.asan_decl_vec.address (),
2381 data.asan_vec.length ());
2382 }
2383
2384 expand_stack_vars (NULL, &data);
2385 }
2386
2387 if (hwasan_sanitize_stack_p ())
2388 hwasan_emit_prologue ();
2389 if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2390 var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2391 virtual_stack_vars_rtx,
2392 var_end_seq);
2393 else if (hwasan_sanitize_allocas_p () && cfun->calls_alloca)
2394 /* When using out-of-line instrumentation we only want to emit one function
2395 call for clearing the tags in a region of shadow stack. When there are
2396 alloca calls in this frame we want to emit a call using the
2397 virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2398 rtx we created in expand_stack_vars. */
2399 var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2400 virtual_stack_vars_rtx);
2401 else if (hwasan_sanitize_stack_p ())
2402 /* If no variables were stored on the stack, `hwasan_get_frame_extent`
2403 will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2404 NULL (i.e. an empty sequence). */
2405 var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2406 virtual_stack_vars_rtx);
2407
2408 fini_vars_expansion ();
2409
2410 /* If there were any artificial non-ignored vars without rtl
2411 found earlier, see if deferred stack allocation hasn't assigned
2412 rtl to them. */
2413 FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2414 {
2415 rtx rtl = DECL_RTL_IF_SET (var);
2416
2417 /* Keep artificial non-ignored vars in cfun->local_decls
2418 chain until instantiate_decls. */
2419 if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2420 add_local_decl (cfun, var);
2421 }
2422
2423 /* If the target requires that FRAME_OFFSET be aligned, do it. */
2424 if (STACK_ALIGNMENT_NEEDED)
2425 {
2426 HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2427 if (FRAME_GROWS_DOWNWARD)
2428 frame_offset = aligned_lower_bound (frame_offset, align);
2429 else
2430 frame_offset = aligned_upper_bound (frame_offset, align);
2431 }
2432
2433 return var_end_seq;
2434 }
2435
2436
2437 /* If we need to produce a detailed dump, print the tree representation
2438 for STMT to the dump file. SINCE is the last RTX after which the RTL
2439 generated for STMT should have been appended. */
2440
2441 static void
maybe_dump_rtl_for_gimple_stmt(gimple * stmt,rtx_insn * since)2442 maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2443 {
2444 if (dump_file && (dump_flags & TDF_DETAILS))
2445 {
2446 fprintf (dump_file, "\n;; ");
2447 print_gimple_stmt (dump_file, stmt, 0,
2448 TDF_SLIM | (dump_flags & TDF_LINENO));
2449 fprintf (dump_file, "\n");
2450
2451 print_rtl (dump_file, since ? NEXT_INSN (since) : since);
2452 }
2453 }
2454
2455 /* Maps the blocks that do not contain tree labels to rtx labels. */
2456
2457 static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2458
2459 /* Returns the label_rtx expression for a label starting basic block BB. */
2460
2461 static rtx_code_label *
label_rtx_for_bb(basic_block bb ATTRIBUTE_UNUSED)2462 label_rtx_for_bb (basic_block bb ATTRIBUTE_UNUSED)
2463 {
2464 gimple_stmt_iterator gsi;
2465 tree lab;
2466
2467 if (bb->flags & BB_RTL)
2468 return block_label (bb);
2469
2470 rtx_code_label **elt = lab_rtx_for_bb->get (bb);
2471 if (elt)
2472 return *elt;
2473
2474 /* Find the tree label if it is present. */
2475
2476 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2477 {
2478 glabel *lab_stmt;
2479
2480 lab_stmt = dyn_cast <glabel *> (gsi_stmt (gsi));
2481 if (!lab_stmt)
2482 break;
2483
2484 lab = gimple_label_label (lab_stmt);
2485 if (DECL_NONLOCAL (lab))
2486 break;
2487
2488 return jump_target_rtx (lab);
2489 }
2490
2491 rtx_code_label *l = gen_label_rtx ();
2492 lab_rtx_for_bb->put (bb, l);
2493 return l;
2494 }
2495
2496
2497 /* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2498 of a basic block where we just expanded the conditional at the end,
2499 possibly clean up the CFG and instruction sequence. LAST is the
2500 last instruction before the just emitted jump sequence. */
2501
2502 static void
maybe_cleanup_end_of_block(edge e,rtx_insn * last)2503 maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2504 {
2505 /* Special case: when jumpif decides that the condition is
2506 trivial it emits an unconditional jump (and the necessary
2507 barrier). But we still have two edges, the fallthru one is
2508 wrong. purge_dead_edges would clean this up later. Unfortunately
2509 we have to insert insns (and split edges) before
2510 find_many_sub_basic_blocks and hence before purge_dead_edges.
2511 But splitting edges might create new blocks which depend on the
2512 fact that if there are two edges there's no barrier. So the
2513 barrier would get lost and verify_flow_info would ICE. Instead
2514 of auditing all edge splitters to care for the barrier (which
2515 normally isn't there in a cleaned CFG), fix it here. */
2516 if (BARRIER_P (get_last_insn ()))
2517 {
2518 rtx_insn *insn;
2519 remove_edge (e);
2520 /* Now, we have a single successor block, if we have insns to
2521 insert on the remaining edge we potentially will insert
2522 it at the end of this block (if the dest block isn't feasible)
2523 in order to avoid splitting the edge. This insertion will take
2524 place in front of the last jump. But we might have emitted
2525 multiple jumps (conditional and one unconditional) to the
2526 same destination. Inserting in front of the last one then
2527 is a problem. See PR 40021. We fix this by deleting all
2528 jumps except the last unconditional one. */
2529 insn = PREV_INSN (get_last_insn ());
2530 /* Make sure we have an unconditional jump. Otherwise we're
2531 confused. */
2532 gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2533 for (insn = PREV_INSN (insn); insn != last;)
2534 {
2535 insn = PREV_INSN (insn);
2536 if (JUMP_P (NEXT_INSN (insn)))
2537 {
2538 if (!any_condjump_p (NEXT_INSN (insn)))
2539 {
2540 gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2541 delete_insn (NEXT_INSN (NEXT_INSN (insn)));
2542 }
2543 delete_insn (NEXT_INSN (insn));
2544 }
2545 }
2546 }
2547 }
2548
2549 /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2550 Returns a new basic block if we've terminated the current basic
2551 block and created a new one. */
2552
2553 static basic_block
expand_gimple_cond(basic_block bb,gcond * stmt)2554 expand_gimple_cond (basic_block bb, gcond *stmt)
2555 {
2556 basic_block new_bb, dest;
2557 edge true_edge;
2558 edge false_edge;
2559 rtx_insn *last2, *last;
2560 enum tree_code code;
2561 tree op0, op1;
2562
2563 code = gimple_cond_code (stmt);
2564 op0 = gimple_cond_lhs (stmt);
2565 op1 = gimple_cond_rhs (stmt);
2566 /* We're sometimes presented with such code:
2567 D.123_1 = x < y;
2568 if (D.123_1 != 0)
2569 ...
2570 This would expand to two comparisons which then later might
2571 be cleaned up by combine. But some pattern matchers like if-conversion
2572 work better when there's only one compare, so make up for this
2573 here as special exception if TER would have made the same change. */
2574 if (SA.values
2575 && TREE_CODE (op0) == SSA_NAME
2576 && TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2577 && TREE_CODE (op1) == INTEGER_CST
2578 && ((gimple_cond_code (stmt) == NE_EXPR
2579 && integer_zerop (op1))
2580 || (gimple_cond_code (stmt) == EQ_EXPR
2581 && integer_onep (op1)))
2582 && bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2583 {
2584 gimple *second = SSA_NAME_DEF_STMT (op0);
2585 if (gimple_code (second) == GIMPLE_ASSIGN)
2586 {
2587 enum tree_code code2 = gimple_assign_rhs_code (second);
2588 if (TREE_CODE_CLASS (code2) == tcc_comparison)
2589 {
2590 code = code2;
2591 op0 = gimple_assign_rhs1 (second);
2592 op1 = gimple_assign_rhs2 (second);
2593 }
2594 /* If jumps are cheap and the target does not support conditional
2595 compare, turn some more codes into jumpy sequences. */
2596 else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2597 && targetm.gen_ccmp_first == NULL)
2598 {
2599 if ((code2 == BIT_AND_EXPR
2600 && TYPE_PRECISION (TREE_TYPE (op0)) == 1
2601 && TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
2602 || code2 == TRUTH_AND_EXPR)
2603 {
2604 code = TRUTH_ANDIF_EXPR;
2605 op0 = gimple_assign_rhs1 (second);
2606 op1 = gimple_assign_rhs2 (second);
2607 }
2608 else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
2609 {
2610 code = TRUTH_ORIF_EXPR;
2611 op0 = gimple_assign_rhs1 (second);
2612 op1 = gimple_assign_rhs2 (second);
2613 }
2614 }
2615 }
2616 }
2617
2618 /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
2619 into (x - C2) * C3 < C4. */
2620 if ((code == EQ_EXPR || code == NE_EXPR)
2621 && TREE_CODE (op0) == SSA_NAME
2622 && TREE_CODE (op1) == INTEGER_CST)
2623 code = maybe_optimize_mod_cmp (code, &op0, &op1);
2624
2625 /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
2626 if (!TYPE_UNSIGNED (TREE_TYPE (op0))
2627 && (code == LT_EXPR || code == LE_EXPR
2628 || code == GT_EXPR || code == GE_EXPR)
2629 && integer_zerop (op1)
2630 && TREE_CODE (op0) == SSA_NAME)
2631 maybe_optimize_sub_cmp_0 (code, &op0, &op1);
2632
2633 last2 = last = get_last_insn ();
2634
2635 extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
2636 set_curr_insn_location (gimple_location (stmt));
2637
2638 /* These flags have no purpose in RTL land. */
2639 true_edge->flags &= ~EDGE_TRUE_VALUE;
2640 false_edge->flags &= ~EDGE_FALSE_VALUE;
2641
2642 /* We can either have a pure conditional jump with one fallthru edge or
2643 two-way jump that needs to be decomposed into two basic blocks. */
2644 if (false_edge->dest == bb->next_bb)
2645 {
2646 jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
2647 true_edge->probability);
2648 maybe_dump_rtl_for_gimple_stmt (stmt, last);
2649 if (true_edge->goto_locus != UNKNOWN_LOCATION)
2650 set_curr_insn_location (true_edge->goto_locus);
2651 false_edge->flags |= EDGE_FALLTHRU;
2652 maybe_cleanup_end_of_block (false_edge, last);
2653 return NULL;
2654 }
2655 if (true_edge->dest == bb->next_bb)
2656 {
2657 jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
2658 false_edge->probability);
2659 maybe_dump_rtl_for_gimple_stmt (stmt, last);
2660 if (false_edge->goto_locus != UNKNOWN_LOCATION)
2661 set_curr_insn_location (false_edge->goto_locus);
2662 true_edge->flags |= EDGE_FALLTHRU;
2663 maybe_cleanup_end_of_block (true_edge, last);
2664 return NULL;
2665 }
2666
2667 jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
2668 true_edge->probability);
2669 last = get_last_insn ();
2670 if (false_edge->goto_locus != UNKNOWN_LOCATION)
2671 set_curr_insn_location (false_edge->goto_locus);
2672 emit_jump (label_rtx_for_bb (false_edge->dest));
2673
2674 BB_END (bb) = last;
2675 if (BARRIER_P (BB_END (bb)))
2676 BB_END (bb) = PREV_INSN (BB_END (bb));
2677 update_bb_for_insn (bb);
2678
2679 new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
2680 dest = false_edge->dest;
2681 redirect_edge_succ (false_edge, new_bb);
2682 false_edge->flags |= EDGE_FALLTHRU;
2683 new_bb->count = false_edge->count ();
2684 loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
2685 add_bb_to_loop (new_bb, loop);
2686 if (loop->latch == bb
2687 && loop->header == dest)
2688 loop->latch = new_bb;
2689 make_single_succ_edge (new_bb, dest, 0);
2690 if (BARRIER_P (BB_END (new_bb)))
2691 BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
2692 update_bb_for_insn (new_bb);
2693
2694 maybe_dump_rtl_for_gimple_stmt (stmt, last2);
2695
2696 if (true_edge->goto_locus != UNKNOWN_LOCATION)
2697 {
2698 set_curr_insn_location (true_edge->goto_locus);
2699 true_edge->goto_locus = curr_insn_location ();
2700 }
2701
2702 return new_bb;
2703 }
2704
2705 /* Mark all calls that can have a transaction restart. */
2706
2707 static void
mark_transaction_restart_calls(gimple * stmt)2708 mark_transaction_restart_calls (gimple *stmt)
2709 {
2710 struct tm_restart_node dummy;
2711 tm_restart_node **slot;
2712
2713 if (!cfun->gimple_df->tm_restart)
2714 return;
2715
2716 dummy.stmt = stmt;
2717 slot = cfun->gimple_df->tm_restart->find_slot (&dummy, NO_INSERT);
2718 if (slot)
2719 {
2720 struct tm_restart_node *n = *slot;
2721 tree list = n->label_or_list;
2722 rtx_insn *insn;
2723
2724 for (insn = next_real_insn (get_last_insn ());
2725 !CALL_P (insn);
2726 insn = next_real_insn (insn))
2727 continue;
2728
2729 if (TREE_CODE (list) == LABEL_DECL)
2730 add_reg_note (insn, REG_TM, label_rtx (list));
2731 else
2732 for (; list ; list = TREE_CHAIN (list))
2733 add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
2734 }
2735 }
2736
2737 /* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
2738 statement STMT. */
2739
2740 static void
expand_call_stmt(gcall * stmt)2741 expand_call_stmt (gcall *stmt)
2742 {
2743 tree exp, decl, lhs;
2744 bool builtin_p;
2745 size_t i;
2746
2747 if (gimple_call_internal_p (stmt))
2748 {
2749 expand_internal_call (stmt);
2750 return;
2751 }
2752
2753 /* If this is a call to a built-in function and it has no effect other
2754 than setting the lhs, try to implement it using an internal function
2755 instead. */
2756 decl = gimple_call_fndecl (stmt);
2757 if (gimple_call_lhs (stmt)
2758 && !gimple_has_side_effects (stmt)
2759 && (optimize || (decl && called_as_built_in (decl))))
2760 {
2761 internal_fn ifn = replacement_internal_fn (stmt);
2762 if (ifn != IFN_LAST)
2763 {
2764 expand_internal_call (ifn, stmt);
2765 return;
2766 }
2767 }
2768
2769 exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (stmt) + 3);
2770
2771 CALL_EXPR_FN (exp) = gimple_call_fn (stmt);
2772 builtin_p = decl && fndecl_built_in_p (decl);
2773
2774 /* If this is not a builtin function, the function type through which the
2775 call is made may be different from the type of the function. */
2776 if (!builtin_p)
2777 CALL_EXPR_FN (exp)
2778 = fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
2779 CALL_EXPR_FN (exp));
2780
2781 TREE_TYPE (exp) = gimple_call_return_type (stmt);
2782 CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (stmt);
2783
2784 for (i = 0; i < gimple_call_num_args (stmt); i++)
2785 {
2786 tree arg = gimple_call_arg (stmt, i);
2787 gimple *def;
2788 /* TER addresses into arguments of builtin functions so we have a
2789 chance to infer more correct alignment information. See PR39954. */
2790 if (builtin_p
2791 && TREE_CODE (arg) == SSA_NAME
2792 && (def = get_gimple_for_ssa_name (arg))
2793 && gimple_assign_rhs_code (def) == ADDR_EXPR)
2794 arg = gimple_assign_rhs1 (def);
2795 CALL_EXPR_ARG (exp, i) = arg;
2796 }
2797
2798 if (gimple_has_side_effects (stmt))
2799 TREE_SIDE_EFFECTS (exp) = 1;
2800
2801 if (gimple_call_nothrow_p (stmt))
2802 TREE_NOTHROW (exp) = 1;
2803
2804 if (gimple_no_warning_p (stmt))
2805 TREE_NO_WARNING (exp) = 1;
2806
2807 CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (stmt);
2808 CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (stmt);
2809 CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
2810 if (decl
2811 && fndecl_built_in_p (decl, BUILT_IN_NORMAL)
2812 && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
2813 CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
2814 else
2815 CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
2816 CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (stmt);
2817 CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (stmt);
2818 SET_EXPR_LOCATION (exp, gimple_location (stmt));
2819
2820 /* Ensure RTL is created for debug args. */
2821 if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
2822 {
2823 vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
2824 unsigned int ix;
2825 tree dtemp;
2826
2827 if (debug_args)
2828 for (ix = 1; (*debug_args)->iterate (ix, &dtemp); ix += 2)
2829 {
2830 gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
2831 expand_debug_expr (dtemp);
2832 }
2833 }
2834
2835 rtx_insn *before_call = get_last_insn ();
2836 lhs = gimple_call_lhs (stmt);
2837 if (lhs)
2838 expand_assignment (lhs, exp, false);
2839 else
2840 expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
2841
2842 /* If the gimple call is an indirect call and has 'nocf_check'
2843 attribute find a generated CALL insn to mark it as no
2844 control-flow verification is needed. */
2845 if (gimple_call_nocf_check_p (stmt)
2846 && !gimple_call_fndecl (stmt))
2847 {
2848 rtx_insn *last = get_last_insn ();
2849 while (!CALL_P (last)
2850 && last != before_call)
2851 last = PREV_INSN (last);
2852
2853 if (last != before_call)
2854 add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
2855 }
2856
2857 mark_transaction_restart_calls (stmt);
2858 }
2859
2860
2861 /* Generate RTL for an asm statement (explicit assembler code).
2862 STRING is a STRING_CST node containing the assembler code text,
2863 or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
2864 insn is volatile; don't optimize it. */
2865
2866 static void
expand_asm_loc(tree string,int vol,location_t locus)2867 expand_asm_loc (tree string, int vol, location_t locus)
2868 {
2869 rtx body;
2870
2871 body = gen_rtx_ASM_INPUT_loc (VOIDmode,
2872 ggc_strdup (TREE_STRING_POINTER (string)),
2873 locus);
2874
2875 MEM_VOLATILE_P (body) = vol;
2876
2877 /* Non-empty basic ASM implicitly clobbers memory. */
2878 if (TREE_STRING_LENGTH (string) != 0)
2879 {
2880 rtx asm_op, clob;
2881 unsigned i, nclobbers;
2882 auto_vec<rtx> input_rvec, output_rvec;
2883 auto_vec<machine_mode> input_mode;
2884 auto_vec<const char *> constraints;
2885 auto_vec<rtx> clobber_rvec;
2886 HARD_REG_SET clobbered_regs;
2887 CLEAR_HARD_REG_SET (clobbered_regs);
2888
2889 clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
2890 clobber_rvec.safe_push (clob);
2891
2892 if (targetm.md_asm_adjust)
2893 targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
2894 constraints, clobber_rvec, clobbered_regs);
2895
2896 asm_op = body;
2897 nclobbers = clobber_rvec.length ();
2898 body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nclobbers));
2899
2900 XVECEXP (body, 0, 0) = asm_op;
2901 for (i = 0; i < nclobbers; i++)
2902 XVECEXP (body, 0, i + 1) = gen_rtx_CLOBBER (VOIDmode, clobber_rvec[i]);
2903 }
2904
2905 emit_insn (body);
2906 }
2907
2908 /* Return the number of times character C occurs in string S. */
2909 static int
n_occurrences(int c,const char * s)2910 n_occurrences (int c, const char *s)
2911 {
2912 int n = 0;
2913 while (*s)
2914 n += (*s++ == c);
2915 return n;
2916 }
2917
2918 /* A subroutine of expand_asm_operands. Check that all operands have
2919 the same number of alternatives. Return true if so. */
2920
2921 static bool
check_operand_nalternatives(const vec<const char * > & constraints)2922 check_operand_nalternatives (const vec<const char *> &constraints)
2923 {
2924 unsigned len = constraints.length();
2925 if (len > 0)
2926 {
2927 int nalternatives = n_occurrences (',', constraints[0]);
2928
2929 if (nalternatives + 1 > MAX_RECOG_ALTERNATIVES)
2930 {
2931 error ("too many alternatives in %<asm%>");
2932 return false;
2933 }
2934
2935 for (unsigned i = 1; i < len; ++i)
2936 if (n_occurrences (',', constraints[i]) != nalternatives)
2937 {
2938 error ("operand constraints for %<asm%> differ "
2939 "in number of alternatives");
2940 return false;
2941 }
2942 }
2943 return true;
2944 }
2945
2946 /* Check for overlap between registers marked in CLOBBERED_REGS and
2947 anything inappropriate in T. Emit error and return the register
2948 variable definition for error, NULL_TREE for ok. */
2949
2950 static bool
tree_conflicts_with_clobbers_p(tree t,HARD_REG_SET * clobbered_regs)2951 tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs)
2952 {
2953 /* Conflicts between asm-declared register variables and the clobber
2954 list are not allowed. */
2955 tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
2956
2957 if (overlap)
2958 {
2959 error ("%<asm%> specifier for variable %qE conflicts with "
2960 "%<asm%> clobber list",
2961 DECL_NAME (overlap));
2962
2963 /* Reset registerness to stop multiple errors emitted for a single
2964 variable. */
2965 DECL_REGISTER (overlap) = 0;
2966 return true;
2967 }
2968
2969 return false;
2970 }
2971
2972 /* Check that the given REGNO spanning NREGS is a valid
2973 asm clobber operand. Some HW registers cannot be
2974 saved/restored, hence they should not be clobbered by
2975 asm statements. */
2976 static bool
asm_clobber_reg_is_valid(int regno,int nregs,const char * regname)2977 asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
2978 {
2979 bool is_valid = true;
2980 HARD_REG_SET regset;
2981
2982 CLEAR_HARD_REG_SET (regset);
2983
2984 add_range_to_hard_reg_set (®set, regno, nregs);
2985
2986 /* Clobbering the PIC register is an error. */
2987 if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
2988 && overlaps_hard_reg_set_p (regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
2989 {
2990 /* ??? Diagnose during gimplification? */
2991 error ("PIC register clobbered by %qs in %<asm%>", regname);
2992 is_valid = false;
2993 }
2994 else if (!in_hard_reg_set_p
2995 (accessible_reg_set, reg_raw_mode[regno], regno))
2996 {
2997 /* ??? Diagnose during gimplification? */
2998 error ("the register %qs cannot be clobbered in %<asm%>"
2999 " for the current target", regname);
3000 is_valid = false;
3001 }
3002
3003 /* Clobbering the stack pointer register is deprecated. GCC expects
3004 the value of the stack pointer after an asm statement to be the same
3005 as it was before, so no asm can validly clobber the stack pointer in
3006 the usual sense. Adding the stack pointer to the clobber list has
3007 traditionally had some undocumented and somewhat obscure side-effects. */
3008 if (overlaps_hard_reg_set_p (regset, Pmode, STACK_POINTER_REGNUM))
3009 {
3010 crtl->sp_is_clobbered_by_asm = true;
3011 if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3012 " %qs in a clobber list is deprecated", regname))
3013 inform (input_location, "the value of the stack pointer after"
3014 " an %<asm%> statement must be the same as it was before"
3015 " the statement");
3016 }
3017
3018 return is_valid;
3019 }
3020
3021 /* Generate RTL for an asm statement with arguments.
3022 STRING is the instruction template.
3023 OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3024 Each output or input has an expression in the TREE_VALUE and
3025 a tree list in TREE_PURPOSE which in turn contains a constraint
3026 name in TREE_VALUE (or NULL_TREE) and a constraint string
3027 in TREE_PURPOSE.
3028 CLOBBERS is a list of STRING_CST nodes each naming a hard register
3029 that is clobbered by this insn.
3030
3031 LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3032 should be the fallthru basic block of the asm goto.
3033
3034 Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3035 Some elements of OUTPUTS may be replaced with trees representing temporary
3036 values. The caller should copy those temporary values to the originally
3037 specified lvalues.
3038
3039 VOL nonzero means the insn is volatile; don't optimize it. */
3040
3041 static void
expand_asm_stmt(gasm * stmt)3042 expand_asm_stmt (gasm *stmt)
3043 {
3044 class save_input_location
3045 {
3046 location_t old;
3047
3048 public:
3049 explicit save_input_location(location_t where)
3050 {
3051 old = input_location;
3052 input_location = where;
3053 }
3054
3055 ~save_input_location()
3056 {
3057 input_location = old;
3058 }
3059 };
3060
3061 location_t locus = gimple_location (stmt);
3062
3063 if (gimple_asm_input_p (stmt))
3064 {
3065 const char *s = gimple_asm_string (stmt);
3066 tree string = build_string (strlen (s), s);
3067 expand_asm_loc (string, gimple_asm_volatile_p (stmt), locus);
3068 return;
3069 }
3070
3071 /* There are some legacy diagnostics in here, and also avoids an extra
3072 parameter to targetm.md_asm_adjust. */
3073 save_input_location s_i_l(locus);
3074
3075 unsigned noutputs = gimple_asm_noutputs (stmt);
3076 unsigned ninputs = gimple_asm_ninputs (stmt);
3077 unsigned nlabels = gimple_asm_nlabels (stmt);
3078 unsigned i;
3079 bool error_seen = false;
3080
3081 /* ??? Diagnose during gimplification? */
3082 if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3083 {
3084 error ("more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3085 return;
3086 }
3087
3088 auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3089 auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3090 auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3091
3092 /* Copy the gimple vectors into new vectors that we can manipulate. */
3093
3094 output_tvec.safe_grow (noutputs, true);
3095 input_tvec.safe_grow (ninputs, true);
3096 constraints.safe_grow (noutputs + ninputs, true);
3097
3098 for (i = 0; i < noutputs; ++i)
3099 {
3100 tree t = gimple_asm_output_op (stmt, i);
3101 output_tvec[i] = TREE_VALUE (t);
3102 constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3103 }
3104 for (i = 0; i < ninputs; i++)
3105 {
3106 tree t = gimple_asm_input_op (stmt, i);
3107 input_tvec[i] = TREE_VALUE (t);
3108 constraints[i + noutputs]
3109 = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3110 }
3111
3112 /* ??? Diagnose during gimplification? */
3113 if (! check_operand_nalternatives (constraints))
3114 return;
3115
3116 /* Count the number of meaningful clobbered registers, ignoring what
3117 we would ignore later. */
3118 auto_vec<rtx> clobber_rvec;
3119 HARD_REG_SET clobbered_regs;
3120 CLEAR_HARD_REG_SET (clobbered_regs);
3121
3122 if (unsigned n = gimple_asm_nclobbers (stmt))
3123 {
3124 clobber_rvec.reserve (n);
3125 for (i = 0; i < n; i++)
3126 {
3127 tree t = gimple_asm_clobber_op (stmt, i);
3128 const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3129 int nregs, j;
3130
3131 j = decode_reg_name_and_count (regname, &nregs);
3132 if (j < 0)
3133 {
3134 if (j == -2)
3135 {
3136 /* ??? Diagnose during gimplification? */
3137 error ("unknown register name %qs in %<asm%>", regname);
3138 error_seen = true;
3139 }
3140 else if (j == -4)
3141 {
3142 rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3143 clobber_rvec.safe_push (x);
3144 }
3145 else
3146 {
3147 /* Otherwise we should have -1 == empty string
3148 or -3 == cc, which is not a register. */
3149 gcc_assert (j == -1 || j == -3);
3150 }
3151 }
3152 else
3153 for (int reg = j; reg < j + nregs; reg++)
3154 {
3155 if (!asm_clobber_reg_is_valid (reg, nregs, regname))
3156 return;
3157
3158 SET_HARD_REG_BIT (clobbered_regs, reg);
3159 rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3160 clobber_rvec.safe_push (x);
3161 }
3162 }
3163 }
3164
3165 /* First pass over inputs and outputs checks validity and sets
3166 mark_addressable if needed. */
3167 /* ??? Diagnose during gimplification? */
3168
3169 for (i = 0; i < noutputs; ++i)
3170 {
3171 tree val = output_tvec[i];
3172 tree type = TREE_TYPE (val);
3173 const char *constraint;
3174 bool is_inout;
3175 bool allows_reg;
3176 bool allows_mem;
3177
3178 /* Try to parse the output constraint. If that fails, there's
3179 no point in going further. */
3180 constraint = constraints[i];
3181 if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3182 &allows_mem, &allows_reg, &is_inout))
3183 return;
3184
3185 /* If the output is a hard register, verify it doesn't conflict with
3186 any other operand's possible hard register use. */
3187 if (DECL_P (val)
3188 && REG_P (DECL_RTL (val))
3189 && HARD_REGISTER_P (DECL_RTL (val)))
3190 {
3191 unsigned j, output_hregno = REGNO (DECL_RTL (val));
3192 bool early_clobber_p = strchr (constraints[i], '&') != NULL;
3193 unsigned long match;
3194
3195 /* Verify the other outputs do not use the same hard register. */
3196 for (j = i + 1; j < noutputs; ++j)
3197 if (DECL_P (output_tvec[j])
3198 && REG_P (DECL_RTL (output_tvec[j]))
3199 && HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3200 && output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3201 {
3202 error ("invalid hard register usage between output operands");
3203 error_seen = true;
3204 }
3205
3206 /* Verify matching constraint operands use the same hard register
3207 and that the non-matching constraint operands do not use the same
3208 hard register if the output is an early clobber operand. */
3209 for (j = 0; j < ninputs; ++j)
3210 if (DECL_P (input_tvec[j])
3211 && REG_P (DECL_RTL (input_tvec[j]))
3212 && HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3213 {
3214 unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3215 switch (*constraints[j + noutputs])
3216 {
3217 case '0': case '1': case '2': case '3': case '4':
3218 case '5': case '6': case '7': case '8': case '9':
3219 match = strtoul (constraints[j + noutputs], NULL, 10);
3220 break;
3221 default:
3222 match = ULONG_MAX;
3223 break;
3224 }
3225 if (i == match
3226 && output_hregno != input_hregno)
3227 {
3228 error ("invalid hard register usage between output "
3229 "operand and matching constraint operand");
3230 error_seen = true;
3231 }
3232 else if (early_clobber_p
3233 && i != match
3234 && output_hregno == input_hregno)
3235 {
3236 error ("invalid hard register usage between "
3237 "earlyclobber operand and input operand");
3238 error_seen = true;
3239 }
3240 }
3241 }
3242
3243 if (! allows_reg
3244 && (allows_mem
3245 || is_inout
3246 || (DECL_P (val)
3247 && REG_P (DECL_RTL (val))
3248 && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3249 mark_addressable (val);
3250 }
3251
3252 for (i = 0; i < ninputs; ++i)
3253 {
3254 bool allows_reg, allows_mem;
3255 const char *constraint;
3256
3257 constraint = constraints[i + noutputs];
3258 if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3259 constraints.address (),
3260 &allows_mem, &allows_reg))
3261 return;
3262
3263 if (! allows_reg && allows_mem)
3264 mark_addressable (input_tvec[i]);
3265 }
3266
3267 /* Second pass evaluates arguments. */
3268
3269 /* Make sure stack is consistent for asm goto. */
3270 if (nlabels > 0)
3271 do_pending_stack_adjust ();
3272 int old_generating_concat_p = generating_concat_p;
3273
3274 /* Vector of RTX's of evaluated output operands. */
3275 auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3276 auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3277 rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3278
3279 output_rvec.safe_grow (noutputs, true);
3280
3281 for (i = 0; i < noutputs; ++i)
3282 {
3283 tree val = output_tvec[i];
3284 tree type = TREE_TYPE (val);
3285 bool is_inout, allows_reg, allows_mem, ok;
3286 rtx op;
3287
3288 ok = parse_output_constraint (&constraints[i], i, ninputs,
3289 noutputs, &allows_mem, &allows_reg,
3290 &is_inout);
3291 gcc_assert (ok);
3292
3293 /* If an output operand is not a decl or indirect ref and our constraint
3294 allows a register, make a temporary to act as an intermediate.
3295 Make the asm insn write into that, then we will copy it to
3296 the real output operand. Likewise for promoted variables. */
3297
3298 generating_concat_p = 0;
3299
3300 if ((TREE_CODE (val) == INDIRECT_REF && allows_mem)
3301 || (DECL_P (val)
3302 && (allows_mem || REG_P (DECL_RTL (val)))
3303 && ! (REG_P (DECL_RTL (val))
3304 && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3305 || ! allows_reg
3306 || is_inout
3307 || TREE_ADDRESSABLE (type))
3308 {
3309 op = expand_expr (val, NULL_RTX, VOIDmode,
3310 !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3311 if (MEM_P (op))
3312 op = validize_mem (op);
3313
3314 if (! allows_reg && !MEM_P (op))
3315 {
3316 error ("output number %d not directly addressable", i);
3317 error_seen = true;
3318 }
3319 if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3320 || GET_CODE (op) == CONCAT)
3321 {
3322 rtx old_op = op;
3323 op = gen_reg_rtx (GET_MODE (op));
3324
3325 generating_concat_p = old_generating_concat_p;
3326
3327 if (is_inout)
3328 emit_move_insn (op, old_op);
3329
3330 push_to_sequence2 (after_rtl_seq, after_rtl_end);
3331 emit_move_insn (old_op, op);
3332 after_rtl_seq = get_insns ();
3333 after_rtl_end = get_last_insn ();
3334 end_sequence ();
3335 }
3336 }
3337 else
3338 {
3339 op = assign_temp (type, 0, 1);
3340 op = validize_mem (op);
3341 if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3342 set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), op);
3343
3344 generating_concat_p = old_generating_concat_p;
3345
3346 push_to_sequence2 (after_rtl_seq, after_rtl_end);
3347 expand_assignment (val, make_tree (type, op), false);
3348 after_rtl_seq = get_insns ();
3349 after_rtl_end = get_last_insn ();
3350 end_sequence ();
3351 }
3352 output_rvec[i] = op;
3353
3354 if (is_inout)
3355 inout_opnum.safe_push (i);
3356 }
3357
3358 const char *str = gimple_asm_string (stmt);
3359 if (error_seen)
3360 {
3361 ninputs = 0;
3362 noutputs = 0;
3363 inout_opnum.truncate (0);
3364 output_rvec.truncate (0);
3365 clobber_rvec.truncate (0);
3366 constraints.truncate (0);
3367 CLEAR_HARD_REG_SET (clobbered_regs);
3368 str = "";
3369 }
3370
3371 auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3372 auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3373
3374 input_rvec.safe_grow (ninputs, true);
3375 input_mode.safe_grow (ninputs, true);
3376
3377 generating_concat_p = 0;
3378
3379 for (i = 0; i < ninputs; ++i)
3380 {
3381 tree val = input_tvec[i];
3382 tree type = TREE_TYPE (val);
3383 bool allows_reg, allows_mem, ok;
3384 const char *constraint;
3385 rtx op;
3386
3387 constraint = constraints[i + noutputs];
3388 ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3389 constraints.address (),
3390 &allows_mem, &allows_reg);
3391 gcc_assert (ok);
3392
3393 /* EXPAND_INITIALIZER will not generate code for valid initializer
3394 constants, but will still generate code for other types of operand.
3395 This is the behavior we want for constant constraints. */
3396 op = expand_expr (val, NULL_RTX, VOIDmode,
3397 allows_reg ? EXPAND_NORMAL
3398 : allows_mem ? EXPAND_MEMORY
3399 : EXPAND_INITIALIZER);
3400
3401 /* Never pass a CONCAT to an ASM. */
3402 if (GET_CODE (op) == CONCAT)
3403 op = force_reg (GET_MODE (op), op);
3404 else if (MEM_P (op))
3405 op = validize_mem (op);
3406
3407 if (asm_operand_ok (op, constraint, NULL) <= 0)
3408 {
3409 if (allows_reg && TYPE_MODE (type) != BLKmode)
3410 op = force_reg (TYPE_MODE (type), op);
3411 else if (!allows_mem)
3412 warning (0, "%<asm%> operand %d probably does not match "
3413 "constraints",
3414 i + noutputs);
3415 else if (MEM_P (op))
3416 {
3417 /* We won't recognize either volatile memory or memory
3418 with a queued address as available a memory_operand
3419 at this point. Ignore it: clearly this *is* a memory. */
3420 }
3421 else
3422 gcc_unreachable ();
3423 }
3424 input_rvec[i] = op;
3425 input_mode[i] = TYPE_MODE (type);
3426 }
3427
3428 /* For in-out operands, copy output rtx to input rtx. */
3429 unsigned ninout = inout_opnum.length ();
3430 for (i = 0; i < ninout; i++)
3431 {
3432 int j = inout_opnum[i];
3433 rtx o = output_rvec[j];
3434
3435 input_rvec.safe_push (o);
3436 input_mode.safe_push (GET_MODE (o));
3437
3438 char buffer[16];
3439 sprintf (buffer, "%d", j);
3440 constraints.safe_push (ggc_strdup (buffer));
3441 }
3442 ninputs += ninout;
3443
3444 /* Sometimes we wish to automatically clobber registers across an asm.
3445 Case in point is when the i386 backend moved from cc0 to a hard reg --
3446 maintaining source-level compatibility means automatically clobbering
3447 the flags register. */
3448 rtx_insn *after_md_seq = NULL;
3449 if (targetm.md_asm_adjust)
3450 after_md_seq
3451 = targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3452 constraints, clobber_rvec, clobbered_regs);
3453
3454 /* Do not allow the hook to change the output and input count,
3455 lest it mess up the operand numbering. */
3456 gcc_assert (output_rvec.length() == noutputs);
3457 gcc_assert (input_rvec.length() == ninputs);
3458 gcc_assert (constraints.length() == noutputs + ninputs);
3459
3460 /* But it certainly can adjust the clobbers. */
3461 unsigned nclobbers = clobber_rvec.length ();
3462
3463 /* Third pass checks for easy conflicts. */
3464 /* ??? Why are we doing this on trees instead of rtx. */
3465
3466 bool clobber_conflict_found = 0;
3467 for (i = 0; i < noutputs; ++i)
3468 if (tree_conflicts_with_clobbers_p (output_tvec[i], &clobbered_regs))
3469 clobber_conflict_found = 1;
3470 for (i = 0; i < ninputs - ninout; ++i)
3471 if (tree_conflicts_with_clobbers_p (input_tvec[i], &clobbered_regs))
3472 clobber_conflict_found = 1;
3473
3474 /* Make vectors for the expression-rtx, constraint strings,
3475 and named operands. */
3476
3477 rtvec argvec = rtvec_alloc (ninputs);
3478 rtvec constraintvec = rtvec_alloc (ninputs);
3479 rtvec labelvec = rtvec_alloc (nlabels);
3480
3481 rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3482 : GET_MODE (output_rvec[0])),
3483 ggc_strdup (str),
3484 "", 0, argvec, constraintvec,
3485 labelvec, locus);
3486 MEM_VOLATILE_P (body) = gimple_asm_volatile_p (stmt);
3487
3488 for (i = 0; i < ninputs; ++i)
3489 {
3490 ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3491 ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3492 = gen_rtx_ASM_INPUT_loc (input_mode[i],
3493 constraints[i + noutputs],
3494 locus);
3495 }
3496
3497 /* Copy labels to the vector. */
3498 rtx_code_label *fallthru_label = NULL;
3499 if (nlabels > 0)
3500 {
3501 basic_block fallthru_bb = NULL;
3502 edge fallthru = find_fallthru_edge (gimple_bb (stmt)->succs);
3503 if (fallthru)
3504 fallthru_bb = fallthru->dest;
3505
3506 for (i = 0; i < nlabels; ++i)
3507 {
3508 tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3509 rtx_insn *r;
3510 /* If asm goto has any labels in the fallthru basic block, use
3511 a label that we emit immediately after the asm goto. Expansion
3512 may insert further instructions into the same basic block after
3513 asm goto and if we don't do this, insertion of instructions on
3514 the fallthru edge might misbehave. See PR58670. */
3515 if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3516 {
3517 if (fallthru_label == NULL_RTX)
3518 fallthru_label = gen_label_rtx ();
3519 r = fallthru_label;
3520 }
3521 else
3522 r = label_rtx (label);
3523 ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3524 }
3525 }
3526
3527 /* Now, for each output, construct an rtx
3528 (set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3529 ARGVEC CONSTRAINTS OPNAMES))
3530 If there is more than one, put them inside a PARALLEL. */
3531
3532 if (noutputs == 0 && nclobbers == 0)
3533 {
3534 /* No output operands: put in a raw ASM_OPERANDS rtx. */
3535 if (nlabels > 0)
3536 emit_jump_insn (body);
3537 else
3538 emit_insn (body);
3539 }
3540 else if (noutputs == 1 && nclobbers == 0)
3541 {
3542 ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3543 if (nlabels > 0)
3544 emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3545 else
3546 emit_insn (gen_rtx_SET (output_rvec[0], body));
3547 }
3548 else
3549 {
3550 rtx obody = body;
3551 int num = noutputs;
3552
3553 if (num == 0)
3554 num = 1;
3555
3556 body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (num + nclobbers));
3557
3558 /* For each output operand, store a SET. */
3559 for (i = 0; i < noutputs; ++i)
3560 {
3561 rtx src, o = output_rvec[i];
3562 if (i == 0)
3563 {
3564 ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3565 src = obody;
3566 }
3567 else
3568 {
3569 src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3570 ASM_OPERANDS_TEMPLATE (obody),
3571 constraints[i], i, argvec,
3572 constraintvec, labelvec, locus);
3573 MEM_VOLATILE_P (src) = gimple_asm_volatile_p (stmt);
3574 }
3575 XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3576 }
3577
3578 /* If there are no outputs (but there are some clobbers)
3579 store the bare ASM_OPERANDS into the PARALLEL. */
3580 if (i == 0)
3581 XVECEXP (body, 0, i++) = obody;
3582
3583 /* Store (clobber REG) for each clobbered register specified. */
3584 for (unsigned j = 0; j < nclobbers; ++j)
3585 {
3586 rtx clobbered_reg = clobber_rvec[j];
3587
3588 /* Do sanity check for overlap between clobbers and respectively
3589 input and outputs that hasn't been handled. Such overlap
3590 should have been detected and reported above. */
3591 if (!clobber_conflict_found && REG_P (clobbered_reg))
3592 {
3593 /* We test the old body (obody) contents to avoid
3594 tripping over the under-construction body. */
3595 for (unsigned k = 0; k < noutputs; ++k)
3596 if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3597 internal_error ("%<asm%> clobber conflict with "
3598 "output operand");
3599
3600 for (unsigned k = 0; k < ninputs - ninout; ++k)
3601 if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
3602 internal_error ("%<asm%> clobber conflict with "
3603 "input operand");
3604 }
3605
3606 XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
3607 }
3608
3609 if (nlabels > 0)
3610 emit_jump_insn (body);
3611 else
3612 emit_insn (body);
3613 }
3614
3615 generating_concat_p = old_generating_concat_p;
3616
3617 if (fallthru_label)
3618 emit_label (fallthru_label);
3619
3620 if (after_md_seq)
3621 emit_insn (after_md_seq);
3622 if (after_rtl_seq)
3623 {
3624 if (nlabels == 0)
3625 emit_insn (after_rtl_seq);
3626 else
3627 {
3628 edge e;
3629 edge_iterator ei;
3630
3631 FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
3632 {
3633 start_sequence ();
3634 for (rtx_insn *curr = after_rtl_seq;
3635 curr != NULL_RTX;
3636 curr = NEXT_INSN (curr))
3637 emit_insn (copy_insn (PATTERN (curr)));
3638 rtx_insn *copy = get_insns ();
3639 end_sequence ();
3640 insert_insn_on_edge (copy, e);
3641 }
3642 }
3643 }
3644
3645 free_temp_slots ();
3646 crtl->has_asm_statement = 1;
3647 }
3648
3649 /* Emit code to jump to the address
3650 specified by the pointer expression EXP. */
3651
3652 static void
expand_computed_goto(tree exp)3653 expand_computed_goto (tree exp)
3654 {
3655 rtx x = expand_normal (exp);
3656
3657 do_pending_stack_adjust ();
3658 emit_indirect_jump (x);
3659 }
3660
3661 /* Generate RTL code for a `goto' statement with target label LABEL.
3662 LABEL should be a LABEL_DECL tree node that was or will later be
3663 defined with `expand_label'. */
3664
3665 static void
expand_goto(tree label)3666 expand_goto (tree label)
3667 {
3668 if (flag_checking)
3669 {
3670 /* Check for a nonlocal goto to a containing function. Should have
3671 gotten translated to __builtin_nonlocal_goto. */
3672 tree context = decl_function_context (label);
3673 gcc_assert (!context || context == current_function_decl);
3674 }
3675
3676 emit_jump (jump_target_rtx (label));
3677 }
3678
3679 /* Output a return with no value. */
3680
3681 static void
expand_null_return_1(void)3682 expand_null_return_1 (void)
3683 {
3684 clear_pending_stack_adjust ();
3685 do_pending_stack_adjust ();
3686 emit_jump (return_label);
3687 }
3688
3689 /* Generate RTL to return from the current function, with no value.
3690 (That is, we do not do anything about returning any value.) */
3691
3692 void
expand_null_return(void)3693 expand_null_return (void)
3694 {
3695 /* If this function was declared to return a value, but we
3696 didn't, clobber the return registers so that they are not
3697 propagated live to the rest of the function. */
3698 clobber_return_register ();
3699
3700 expand_null_return_1 ();
3701 }
3702
3703 /* Generate RTL to return from the current function, with value VAL. */
3704
3705 static void
expand_value_return(rtx val)3706 expand_value_return (rtx val)
3707 {
3708 /* Copy the value to the return location unless it's already there. */
3709
3710 tree decl = DECL_RESULT (current_function_decl);
3711 rtx return_reg = DECL_RTL (decl);
3712 if (return_reg != val)
3713 {
3714 tree funtype = TREE_TYPE (current_function_decl);
3715 tree type = TREE_TYPE (decl);
3716 int unsignedp = TYPE_UNSIGNED (type);
3717 machine_mode old_mode = DECL_MODE (decl);
3718 machine_mode mode;
3719 if (DECL_BY_REFERENCE (decl))
3720 mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
3721 else
3722 mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
3723
3724 if (mode != old_mode)
3725 val = convert_modes (mode, old_mode, val, unsignedp);
3726
3727 if (GET_CODE (return_reg) == PARALLEL)
3728 emit_group_load (return_reg, val, type, int_size_in_bytes (type));
3729 else
3730 emit_move_insn (return_reg, val);
3731 }
3732
3733 expand_null_return_1 ();
3734 }
3735
3736 /* Generate RTL to evaluate the expression RETVAL and return it
3737 from the current function. */
3738
3739 static void
expand_return(tree retval)3740 expand_return (tree retval)
3741 {
3742 rtx result_rtl;
3743 rtx val = 0;
3744 tree retval_rhs;
3745
3746 /* If function wants no value, give it none. */
3747 if (TREE_CODE (TREE_TYPE (TREE_TYPE (current_function_decl))) == VOID_TYPE)
3748 {
3749 expand_normal (retval);
3750 expand_null_return ();
3751 return;
3752 }
3753
3754 if (retval == error_mark_node)
3755 {
3756 /* Treat this like a return of no value from a function that
3757 returns a value. */
3758 expand_null_return ();
3759 return;
3760 }
3761 else if ((TREE_CODE (retval) == MODIFY_EXPR
3762 || TREE_CODE (retval) == INIT_EXPR)
3763 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
3764 retval_rhs = TREE_OPERAND (retval, 1);
3765 else
3766 retval_rhs = retval;
3767
3768 result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
3769
3770 /* If we are returning the RESULT_DECL, then the value has already
3771 been stored into it, so we don't have to do anything special. */
3772 if (TREE_CODE (retval_rhs) == RESULT_DECL)
3773 expand_value_return (result_rtl);
3774
3775 /* If the result is an aggregate that is being returned in one (or more)
3776 registers, load the registers here. */
3777
3778 else if (retval_rhs != 0
3779 && TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
3780 && REG_P (result_rtl))
3781 {
3782 val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
3783 if (val)
3784 {
3785 /* Use the mode of the result value on the return register. */
3786 PUT_MODE (result_rtl, GET_MODE (val));
3787 expand_value_return (val);
3788 }
3789 else
3790 expand_null_return ();
3791 }
3792 else if (retval_rhs != 0
3793 && !VOID_TYPE_P (TREE_TYPE (retval_rhs))
3794 && (REG_P (result_rtl)
3795 || (GET_CODE (result_rtl) == PARALLEL)))
3796 {
3797 /* Compute the return value into a temporary (usually a pseudo reg). */
3798 val
3799 = assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
3800 val = expand_expr (retval_rhs, val, GET_MODE (val), EXPAND_NORMAL);
3801 val = force_not_mem (val);
3802 expand_value_return (val);
3803 }
3804 else
3805 {
3806 /* No hard reg used; calculate value into hard return reg. */
3807 expand_expr (retval, const0_rtx, VOIDmode, EXPAND_NORMAL);
3808 expand_value_return (result_rtl);
3809 }
3810 }
3811
3812 /* Expand a clobber of LHS. If LHS is stored it in a multi-part
3813 register, tell the rtl optimizers that its value is no longer
3814 needed. */
3815
3816 static void
expand_clobber(tree lhs)3817 expand_clobber (tree lhs)
3818 {
3819 if (DECL_P (lhs))
3820 {
3821 rtx decl_rtl = DECL_RTL_IF_SET (lhs);
3822 if (decl_rtl && REG_P (decl_rtl))
3823 {
3824 machine_mode decl_mode = GET_MODE (decl_rtl);
3825 if (maybe_gt (GET_MODE_SIZE (decl_mode),
3826 REGMODE_NATURAL_SIZE (decl_mode)))
3827 emit_clobber (decl_rtl);
3828 }
3829 }
3830 }
3831
3832 /* A subroutine of expand_gimple_stmt, expanding one gimple statement
3833 STMT that doesn't require special handling for outgoing edges. That
3834 is no tailcalls and no GIMPLE_COND. */
3835
3836 static void
expand_gimple_stmt_1(gimple * stmt)3837 expand_gimple_stmt_1 (gimple *stmt)
3838 {
3839 tree op0;
3840
3841 set_curr_insn_location (gimple_location (stmt));
3842
3843 switch (gimple_code (stmt))
3844 {
3845 case GIMPLE_GOTO:
3846 op0 = gimple_goto_dest (stmt);
3847 if (TREE_CODE (op0) == LABEL_DECL)
3848 expand_goto (op0);
3849 else
3850 expand_computed_goto (op0);
3851 break;
3852 case GIMPLE_LABEL:
3853 expand_label (gimple_label_label (as_a <glabel *> (stmt)));
3854 break;
3855 case GIMPLE_NOP:
3856 case GIMPLE_PREDICT:
3857 break;
3858 case GIMPLE_SWITCH:
3859 {
3860 gswitch *swtch = as_a <gswitch *> (stmt);
3861 if (gimple_switch_num_labels (swtch) == 1)
3862 expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
3863 else
3864 expand_case (swtch);
3865 }
3866 break;
3867 case GIMPLE_ASM:
3868 expand_asm_stmt (as_a <gasm *> (stmt));
3869 break;
3870 case GIMPLE_CALL:
3871 expand_call_stmt (as_a <gcall *> (stmt));
3872 break;
3873
3874 case GIMPLE_RETURN:
3875 {
3876 op0 = gimple_return_retval (as_a <greturn *> (stmt));
3877
3878 /* If a return doesn't have a location, it very likely represents
3879 multiple user returns so we cannot let it inherit the location
3880 of the last statement of the previous basic block in RTL. */
3881 if (!gimple_has_location (stmt))
3882 set_curr_insn_location (cfun->function_end_locus);
3883
3884 if (op0 && op0 != error_mark_node)
3885 {
3886 tree result = DECL_RESULT (current_function_decl);
3887
3888 /* If we are not returning the current function's RESULT_DECL,
3889 build an assignment to it. */
3890 if (op0 != result)
3891 {
3892 /* I believe that a function's RESULT_DECL is unique. */
3893 gcc_assert (TREE_CODE (op0) != RESULT_DECL);
3894
3895 /* ??? We'd like to use simply expand_assignment here,
3896 but this fails if the value is of BLKmode but the return
3897 decl is a register. expand_return has special handling
3898 for this combination, which eventually should move
3899 to common code. See comments there. Until then, let's
3900 build a modify expression :-/ */
3901 op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
3902 result, op0);
3903 }
3904 }
3905
3906 if (!op0)
3907 expand_null_return ();
3908 else
3909 expand_return (op0);
3910 }
3911 break;
3912
3913 case GIMPLE_ASSIGN:
3914 {
3915 gassign *assign_stmt = as_a <gassign *> (stmt);
3916 tree lhs = gimple_assign_lhs (assign_stmt);
3917
3918 /* Tree expand used to fiddle with |= and &= of two bitfield
3919 COMPONENT_REFs here. This can't happen with gimple, the LHS
3920 of binary assigns must be a gimple reg. */
3921
3922 if (TREE_CODE (lhs) != SSA_NAME
3923 || gimple_assign_rhs_class (assign_stmt) == GIMPLE_SINGLE_RHS)
3924 {
3925 tree rhs = gimple_assign_rhs1 (assign_stmt);
3926 gcc_assert (gimple_assign_rhs_class (assign_stmt)
3927 == GIMPLE_SINGLE_RHS);
3928 if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (rhs)
3929 /* Do not put locations on possibly shared trees. */
3930 && !is_gimple_min_invariant (rhs))
3931 SET_EXPR_LOCATION (rhs, gimple_location (stmt));
3932 if (TREE_CLOBBER_P (rhs))
3933 /* This is a clobber to mark the going out of scope for
3934 this LHS. */
3935 expand_clobber (lhs);
3936 else
3937 expand_assignment (lhs, rhs,
3938 gimple_assign_nontemporal_move_p (
3939 assign_stmt));
3940 }
3941 else
3942 {
3943 rtx target, temp;
3944 bool nontemporal = gimple_assign_nontemporal_move_p (assign_stmt);
3945 struct separate_ops ops;
3946 bool promoted = false;
3947
3948 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3949 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
3950 promoted = true;
3951
3952 ops.code = gimple_assign_rhs_code (assign_stmt);
3953 ops.type = TREE_TYPE (lhs);
3954 switch (get_gimple_rhs_class (ops.code))
3955 {
3956 case GIMPLE_TERNARY_RHS:
3957 ops.op2 = gimple_assign_rhs3 (assign_stmt);
3958 /* Fallthru */
3959 case GIMPLE_BINARY_RHS:
3960 ops.op1 = gimple_assign_rhs2 (assign_stmt);
3961 /* Fallthru */
3962 case GIMPLE_UNARY_RHS:
3963 ops.op0 = gimple_assign_rhs1 (assign_stmt);
3964 break;
3965 default:
3966 gcc_unreachable ();
3967 }
3968 ops.location = gimple_location (stmt);
3969
3970 /* If we want to use a nontemporal store, force the value to
3971 register first. If we store into a promoted register,
3972 don't directly expand to target. */
3973 temp = nontemporal || promoted ? NULL_RTX : target;
3974 temp = expand_expr_real_2 (&ops, temp, GET_MODE (target),
3975 EXPAND_NORMAL);
3976
3977 if (temp == target)
3978 ;
3979 else if (promoted)
3980 {
3981 int unsignedp = SUBREG_PROMOTED_SIGN (target);
3982 /* If TEMP is a VOIDmode constant, use convert_modes to make
3983 sure that we properly convert it. */
3984 if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
3985 {
3986 temp = convert_modes (GET_MODE (target),
3987 TYPE_MODE (ops.type),
3988 temp, unsignedp);
3989 temp = convert_modes (GET_MODE (SUBREG_REG (target)),
3990 GET_MODE (target), temp, unsignedp);
3991 }
3992
3993 convert_move (SUBREG_REG (target), temp, unsignedp);
3994 }
3995 else if (nontemporal && emit_storent_insn (target, temp))
3996 ;
3997 else
3998 {
3999 temp = force_operand (temp, target);
4000 if (temp != target)
4001 emit_move_insn (target, temp);
4002 }
4003 }
4004 }
4005 break;
4006
4007 default:
4008 gcc_unreachable ();
4009 }
4010 }
4011
4012 /* Expand one gimple statement STMT and return the last RTL instruction
4013 before any of the newly generated ones.
4014
4015 In addition to generating the necessary RTL instructions this also
4016 sets REG_EH_REGION notes if necessary and sets the current source
4017 location for diagnostics. */
4018
4019 static rtx_insn *
expand_gimple_stmt(gimple * stmt)4020 expand_gimple_stmt (gimple *stmt)
4021 {
4022 location_t saved_location = input_location;
4023 rtx_insn *last = get_last_insn ();
4024 int lp_nr;
4025
4026 gcc_assert (cfun);
4027
4028 /* We need to save and restore the current source location so that errors
4029 discovered during expansion are emitted with the right location. But
4030 it would be better if the diagnostic routines used the source location
4031 embedded in the tree nodes rather than globals. */
4032 if (gimple_has_location (stmt))
4033 input_location = gimple_location (stmt);
4034
4035 expand_gimple_stmt_1 (stmt);
4036
4037 /* Free any temporaries used to evaluate this statement. */
4038 free_temp_slots ();
4039
4040 input_location = saved_location;
4041
4042 /* Mark all insns that may trap. */
4043 lp_nr = lookup_stmt_eh_lp (stmt);
4044 if (lp_nr)
4045 {
4046 rtx_insn *insn;
4047 for (insn = next_real_insn (last); insn;
4048 insn = next_real_insn (insn))
4049 {
4050 if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4051 /* If we want exceptions for non-call insns, any
4052 may_trap_p instruction may throw. */
4053 && GET_CODE (PATTERN (insn)) != CLOBBER
4054 && GET_CODE (PATTERN (insn)) != USE
4055 && insn_could_throw_p (insn))
4056 make_reg_eh_region_note (insn, 0, lp_nr);
4057 }
4058 }
4059
4060 return last;
4061 }
4062
4063 /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4064 that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4065 generated a tail call (something that might be denied by the ABI
4066 rules governing the call; see calls.c).
4067
4068 Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4069 can still reach the rest of BB. The case here is __builtin_sqrt,
4070 where the NaN result goes through the external function (with a
4071 tailcall) and the normal result happens via a sqrt instruction. */
4072
4073 static basic_block
expand_gimple_tailcall(basic_block bb,gcall * stmt,bool * can_fallthru)4074 expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru)
4075 {
4076 rtx_insn *last2, *last;
4077 edge e;
4078 edge_iterator ei;
4079 profile_probability probability;
4080
4081 last2 = last = expand_gimple_stmt (stmt);
4082
4083 for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
4084 if (CALL_P (last) && SIBLING_CALL_P (last))
4085 goto found;
4086
4087 maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4088
4089 *can_fallthru = true;
4090 return NULL;
4091
4092 found:
4093 /* ??? Wouldn't it be better to just reset any pending stack adjust?
4094 Any instructions emitted here are about to be deleted. */
4095 do_pending_stack_adjust ();
4096
4097 /* Remove any non-eh, non-abnormal edges that don't go to exit. */
4098 /* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4099 EH or abnormal edges, we shouldn't have created a tail call in
4100 the first place. So it seems to me we should just be removing
4101 all edges here, or redirecting the existing fallthru edge to
4102 the exit block. */
4103
4104 probability = profile_probability::never ();
4105
4106 for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
4107 {
4108 if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4109 {
4110 if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4111 e->dest->count -= e->count ();
4112 probability += e->probability;
4113 remove_edge (e);
4114 }
4115 else
4116 ei_next (&ei);
4117 }
4118
4119 /* This is somewhat ugly: the call_expr expander often emits instructions
4120 after the sibcall (to perform the function return). These confuse the
4121 find_many_sub_basic_blocks code, so we need to get rid of these. */
4122 last = NEXT_INSN (last);
4123 gcc_assert (BARRIER_P (last));
4124
4125 *can_fallthru = false;
4126 while (NEXT_INSN (last))
4127 {
4128 /* For instance an sqrt builtin expander expands if with
4129 sibcall in the then and label for `else`. */
4130 if (LABEL_P (NEXT_INSN (last)))
4131 {
4132 *can_fallthru = true;
4133 break;
4134 }
4135 delete_insn (NEXT_INSN (last));
4136 }
4137
4138 e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4139 | EDGE_SIBCALL);
4140 e->probability = probability;
4141 BB_END (bb) = last;
4142 update_bb_for_insn (bb);
4143
4144 if (NEXT_INSN (last))
4145 {
4146 bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
4147
4148 last = BB_END (bb);
4149 if (BARRIER_P (last))
4150 BB_END (bb) = PREV_INSN (last);
4151 }
4152
4153 maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4154
4155 return bb;
4156 }
4157
4158 /* Return the difference between the floor and the truncated result of
4159 a signed division by OP1 with remainder MOD. */
4160 static rtx
floor_sdiv_adjust(machine_mode mode,rtx mod,rtx op1)4161 floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4162 {
4163 /* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4164 return gen_rtx_IF_THEN_ELSE
4165 (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4166 gen_rtx_IF_THEN_ELSE
4167 (mode, gen_rtx_LT (BImode,
4168 gen_rtx_DIV (mode, op1, mod),
4169 const0_rtx),
4170 constm1_rtx, const0_rtx),
4171 const0_rtx);
4172 }
4173
4174 /* Return the difference between the ceil and the truncated result of
4175 a signed division by OP1 with remainder MOD. */
4176 static rtx
ceil_sdiv_adjust(machine_mode mode,rtx mod,rtx op1)4177 ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4178 {
4179 /* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4180 return gen_rtx_IF_THEN_ELSE
4181 (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4182 gen_rtx_IF_THEN_ELSE
4183 (mode, gen_rtx_GT (BImode,
4184 gen_rtx_DIV (mode, op1, mod),
4185 const0_rtx),
4186 const1_rtx, const0_rtx),
4187 const0_rtx);
4188 }
4189
4190 /* Return the difference between the ceil and the truncated result of
4191 an unsigned division by OP1 with remainder MOD. */
4192 static rtx
ceil_udiv_adjust(machine_mode mode,rtx mod,rtx op1 ATTRIBUTE_UNUSED)4193 ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4194 {
4195 /* (mod != 0 ? 1 : 0) */
4196 return gen_rtx_IF_THEN_ELSE
4197 (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4198 const1_rtx, const0_rtx);
4199 }
4200
4201 /* Return the difference between the rounded and the truncated result
4202 of a signed division by OP1 with remainder MOD. Halfway cases are
4203 rounded away from zero, rather than to the nearest even number. */
4204 static rtx
round_sdiv_adjust(machine_mode mode,rtx mod,rtx op1)4205 round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4206 {
4207 /* (abs (mod) >= abs (op1) - abs (mod)
4208 ? (op1 / mod > 0 ? 1 : -1)
4209 : 0) */
4210 return gen_rtx_IF_THEN_ELSE
4211 (mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4212 gen_rtx_MINUS (mode,
4213 gen_rtx_ABS (mode, op1),
4214 gen_rtx_ABS (mode, mod))),
4215 gen_rtx_IF_THEN_ELSE
4216 (mode, gen_rtx_GT (BImode,
4217 gen_rtx_DIV (mode, op1, mod),
4218 const0_rtx),
4219 const1_rtx, constm1_rtx),
4220 const0_rtx);
4221 }
4222
4223 /* Return the difference between the rounded and the truncated result
4224 of a unsigned division by OP1 with remainder MOD. Halfway cases
4225 are rounded away from zero, rather than to the nearest even
4226 number. */
4227 static rtx
round_udiv_adjust(machine_mode mode,rtx mod,rtx op1)4228 round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4229 {
4230 /* (mod >= op1 - mod ? 1 : 0) */
4231 return gen_rtx_IF_THEN_ELSE
4232 (mode, gen_rtx_GE (BImode, mod,
4233 gen_rtx_MINUS (mode, op1, mod)),
4234 const1_rtx, const0_rtx);
4235 }
4236
4237 /* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4238 any rtl. */
4239
4240 static rtx
convert_debug_memory_address(scalar_int_mode mode,rtx x,addr_space_t as)4241 convert_debug_memory_address (scalar_int_mode mode, rtx x,
4242 addr_space_t as)
4243 {
4244 #ifndef POINTERS_EXTEND_UNSIGNED
4245 gcc_assert (mode == Pmode
4246 || mode == targetm.addr_space.address_mode (as));
4247 gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4248 #else
4249 rtx temp;
4250
4251 gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4252
4253 if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4254 return x;
4255
4256 /* X must have some form of address mode already. */
4257 scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4258 if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (xmode))
4259 x = lowpart_subreg (mode, x, xmode);
4260 else if (POINTERS_EXTEND_UNSIGNED > 0)
4261 x = gen_rtx_ZERO_EXTEND (mode, x);
4262 else if (!POINTERS_EXTEND_UNSIGNED)
4263 x = gen_rtx_SIGN_EXTEND (mode, x);
4264 else
4265 {
4266 switch (GET_CODE (x))
4267 {
4268 case SUBREG:
4269 if ((SUBREG_PROMOTED_VAR_P (x)
4270 || (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4271 || (GET_CODE (SUBREG_REG (x)) == PLUS
4272 && REG_P (XEXP (SUBREG_REG (x), 0))
4273 && REG_POINTER (XEXP (SUBREG_REG (x), 0))
4274 && CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4275 && GET_MODE (SUBREG_REG (x)) == mode)
4276 return SUBREG_REG (x);
4277 break;
4278 case LABEL_REF:
4279 temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4280 LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4281 return temp;
4282 case SYMBOL_REF:
4283 temp = shallow_copy_rtx (x);
4284 PUT_MODE (temp, mode);
4285 return temp;
4286 case CONST:
4287 temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4288 if (temp)
4289 temp = gen_rtx_CONST (mode, temp);
4290 return temp;
4291 case PLUS:
4292 case MINUS:
4293 if (CONST_INT_P (XEXP (x, 1)))
4294 {
4295 temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4296 if (temp)
4297 return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4298 }
4299 break;
4300 default:
4301 break;
4302 }
4303 /* Don't know how to express ptr_extend as operation in debug info. */
4304 return NULL;
4305 }
4306 #endif /* POINTERS_EXTEND_UNSIGNED */
4307
4308 return x;
4309 }
4310
4311 /* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4312 by avoid_deep_ter_for_debug. */
4313
4314 static hash_map<tree, tree> *deep_ter_debug_map;
4315
4316 /* Split too deep TER chains for debug stmts using debug temporaries. */
4317
4318 static void
avoid_deep_ter_for_debug(gimple * stmt,int depth)4319 avoid_deep_ter_for_debug (gimple *stmt, int depth)
4320 {
4321 use_operand_p use_p;
4322 ssa_op_iter iter;
4323 FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4324 {
4325 tree use = USE_FROM_PTR (use_p);
4326 if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4327 continue;
4328 gimple *g = get_gimple_for_ssa_name (use);
4329 if (g == NULL)
4330 continue;
4331 if (depth > 6 && !stmt_ends_bb_p (g))
4332 {
4333 if (deep_ter_debug_map == NULL)
4334 deep_ter_debug_map = new hash_map<tree, tree>;
4335
4336 tree &vexpr = deep_ter_debug_map->get_or_insert (use);
4337 if (vexpr != NULL)
4338 continue;
4339 vexpr = make_node (DEBUG_EXPR_DECL);
4340 gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4341 DECL_ARTIFICIAL (vexpr) = 1;
4342 TREE_TYPE (vexpr) = TREE_TYPE (use);
4343 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (use)));
4344 gimple_stmt_iterator gsi = gsi_for_stmt (g);
4345 gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4346 avoid_deep_ter_for_debug (def_temp, 0);
4347 }
4348 else
4349 avoid_deep_ter_for_debug (g, depth + 1);
4350 }
4351 }
4352
4353 /* Return an RTX equivalent to the value of the parameter DECL. */
4354
4355 static rtx
expand_debug_parm_decl(tree decl)4356 expand_debug_parm_decl (tree decl)
4357 {
4358 rtx incoming = DECL_INCOMING_RTL (decl);
4359
4360 if (incoming
4361 && GET_MODE (incoming) != BLKmode
4362 && ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4363 || (MEM_P (incoming)
4364 && REG_P (XEXP (incoming, 0))
4365 && HARD_REGISTER_P (XEXP (incoming, 0)))))
4366 {
4367 rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4368
4369 #ifdef HAVE_window_save
4370 /* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4371 If the target machine has an explicit window save instruction, the
4372 actual entry value is the corresponding OUTGOING_REGNO instead. */
4373 if (REG_P (incoming)
4374 && OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4375 incoming
4376 = gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4377 OUTGOING_REGNO (REGNO (incoming)), 0);
4378 else if (MEM_P (incoming))
4379 {
4380 rtx reg = XEXP (incoming, 0);
4381 if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4382 {
4383 reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4384 incoming = replace_equiv_address_nv (incoming, reg);
4385 }
4386 else
4387 incoming = copy_rtx (incoming);
4388 }
4389 #endif
4390
4391 ENTRY_VALUE_EXP (rtl) = incoming;
4392 return rtl;
4393 }
4394
4395 if (incoming
4396 && GET_MODE (incoming) != BLKmode
4397 && !TREE_ADDRESSABLE (decl)
4398 && MEM_P (incoming)
4399 && (XEXP (incoming, 0) == virtual_incoming_args_rtx
4400 || (GET_CODE (XEXP (incoming, 0)) == PLUS
4401 && XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4402 && CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4403 return copy_rtx (incoming);
4404
4405 return NULL_RTX;
4406 }
4407
4408 /* Return an RTX equivalent to the value of the tree expression EXP. */
4409
4410 static rtx
expand_debug_expr(tree exp)4411 expand_debug_expr (tree exp)
4412 {
4413 rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4414 machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4415 machine_mode inner_mode = VOIDmode;
4416 int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4417 addr_space_t as;
4418 scalar_int_mode op0_mode, op1_mode, addr_mode;
4419
4420 switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4421 {
4422 case tcc_expression:
4423 switch (TREE_CODE (exp))
4424 {
4425 case COND_EXPR:
4426 case DOT_PROD_EXPR:
4427 case SAD_EXPR:
4428 case WIDEN_MULT_PLUS_EXPR:
4429 case WIDEN_MULT_MINUS_EXPR:
4430 goto ternary;
4431
4432 case TRUTH_ANDIF_EXPR:
4433 case TRUTH_ORIF_EXPR:
4434 case TRUTH_AND_EXPR:
4435 case TRUTH_OR_EXPR:
4436 case TRUTH_XOR_EXPR:
4437 goto binary;
4438
4439 case TRUTH_NOT_EXPR:
4440 goto unary;
4441
4442 default:
4443 break;
4444 }
4445 break;
4446
4447 ternary:
4448 op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4449 if (!op2)
4450 return NULL_RTX;
4451 /* Fall through. */
4452
4453 binary:
4454 case tcc_binary:
4455 if (mode == BLKmode)
4456 return NULL_RTX;
4457 op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4458 if (!op1)
4459 return NULL_RTX;
4460 switch (TREE_CODE (exp))
4461 {
4462 case LSHIFT_EXPR:
4463 case RSHIFT_EXPR:
4464 case LROTATE_EXPR:
4465 case RROTATE_EXPR:
4466 case WIDEN_LSHIFT_EXPR:
4467 /* Ensure second operand isn't wider than the first one. */
4468 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4469 if (is_a <scalar_int_mode> (inner_mode, &op1_mode)
4470 && (GET_MODE_UNIT_PRECISION (mode)
4471 < GET_MODE_PRECISION (op1_mode)))
4472 op1 = lowpart_subreg (GET_MODE_INNER (mode), op1, op1_mode);
4473 break;
4474 default:
4475 break;
4476 }
4477 /* Fall through. */
4478
4479 unary:
4480 case tcc_unary:
4481 if (mode == BLKmode)
4482 return NULL_RTX;
4483 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4484 op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4485 if (!op0)
4486 return NULL_RTX;
4487 break;
4488
4489 case tcc_comparison:
4490 unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4491 goto binary;
4492
4493 case tcc_type:
4494 case tcc_statement:
4495 gcc_unreachable ();
4496
4497 case tcc_constant:
4498 case tcc_exceptional:
4499 case tcc_declaration:
4500 case tcc_reference:
4501 case tcc_vl_exp:
4502 break;
4503 }
4504
4505 switch (TREE_CODE (exp))
4506 {
4507 case STRING_CST:
4508 if (!lookup_constant_def (exp))
4509 {
4510 if (strlen (TREE_STRING_POINTER (exp)) + 1
4511 != (size_t) TREE_STRING_LENGTH (exp))
4512 return NULL_RTX;
4513 op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4514 op0 = gen_rtx_MEM (BLKmode, op0);
4515 set_mem_attributes (op0, exp, 0);
4516 return op0;
4517 }
4518 /* Fall through. */
4519
4520 case INTEGER_CST:
4521 case REAL_CST:
4522 case FIXED_CST:
4523 op0 = expand_expr (exp, NULL_RTX, mode, EXPAND_INITIALIZER);
4524 return op0;
4525
4526 case POLY_INT_CST:
4527 return immed_wide_int_const (poly_int_cst_value (exp), mode);
4528
4529 case COMPLEX_CST:
4530 gcc_assert (COMPLEX_MODE_P (mode));
4531 op0 = expand_debug_expr (TREE_REALPART (exp));
4532 op1 = expand_debug_expr (TREE_IMAGPART (exp));
4533 return gen_rtx_CONCAT (mode, op0, op1);
4534
4535 case DEBUG_EXPR_DECL:
4536 op0 = DECL_RTL_IF_SET (exp);
4537
4538 if (op0)
4539 {
4540 if (GET_MODE (op0) != mode)
4541 gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4542 else
4543 return op0;
4544 }
4545
4546 op0 = gen_rtx_DEBUG_EXPR (mode);
4547 DEBUG_EXPR_TREE_DECL (op0) = exp;
4548 SET_DECL_RTL (exp, op0);
4549
4550 return op0;
4551
4552 case VAR_DECL:
4553 case PARM_DECL:
4554 case FUNCTION_DECL:
4555 case LABEL_DECL:
4556 case CONST_DECL:
4557 case RESULT_DECL:
4558 op0 = DECL_RTL_IF_SET (exp);
4559
4560 /* This decl was probably optimized away. */
4561 if (!op0
4562 /* At least label RTXen are sometimes replaced by
4563 NOTE_INSN_DELETED_LABEL. Any notes here are not
4564 handled by copy_rtx. */
4565 || NOTE_P (op0))
4566 {
4567 if (!VAR_P (exp)
4568 || DECL_EXTERNAL (exp)
4569 || !TREE_STATIC (exp)
4570 || !DECL_NAME (exp)
4571 || DECL_HARD_REGISTER (exp)
4572 || DECL_IN_CONSTANT_POOL (exp)
4573 || mode == VOIDmode)
4574 return NULL;
4575
4576 op0 = make_decl_rtl_for_debug (exp);
4577 if (!MEM_P (op0)
4578 || GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
4579 || SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
4580 return NULL;
4581 }
4582 else
4583 op0 = copy_rtx (op0);
4584
4585 if (GET_MODE (op0) == BLKmode
4586 /* If op0 is not BLKmode, but mode is, adjust_mode
4587 below would ICE. While it is likely a FE bug,
4588 try to be robust here. See PR43166. */
4589 || mode == BLKmode
4590 || (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
4591 {
4592 gcc_assert (MEM_P (op0));
4593 op0 = adjust_address_nv (op0, mode, 0);
4594 return op0;
4595 }
4596
4597 /* Fall through. */
4598
4599 adjust_mode:
4600 case PAREN_EXPR:
4601 CASE_CONVERT:
4602 {
4603 inner_mode = GET_MODE (op0);
4604
4605 if (mode == inner_mode)
4606 return op0;
4607
4608 if (inner_mode == VOIDmode)
4609 {
4610 if (TREE_CODE (exp) == SSA_NAME)
4611 inner_mode = TYPE_MODE (TREE_TYPE (exp));
4612 else
4613 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4614 if (mode == inner_mode)
4615 return op0;
4616 }
4617
4618 if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
4619 {
4620 if (GET_MODE_UNIT_BITSIZE (mode)
4621 == GET_MODE_UNIT_BITSIZE (inner_mode))
4622 op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
4623 else if (GET_MODE_UNIT_BITSIZE (mode)
4624 < GET_MODE_UNIT_BITSIZE (inner_mode))
4625 op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
4626 else
4627 op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
4628 }
4629 else if (FLOAT_MODE_P (mode))
4630 {
4631 gcc_assert (TREE_CODE (exp) != SSA_NAME);
4632 if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
4633 op0 = simplify_gen_unary (UNSIGNED_FLOAT, mode, op0, inner_mode);
4634 else
4635 op0 = simplify_gen_unary (FLOAT, mode, op0, inner_mode);
4636 }
4637 else if (FLOAT_MODE_P (inner_mode))
4638 {
4639 if (unsignedp)
4640 op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
4641 else
4642 op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
4643 }
4644 else if (GET_MODE_UNIT_PRECISION (mode)
4645 == GET_MODE_UNIT_PRECISION (inner_mode))
4646 op0 = lowpart_subreg (mode, op0, inner_mode);
4647 else if (GET_MODE_UNIT_PRECISION (mode)
4648 < GET_MODE_UNIT_PRECISION (inner_mode))
4649 op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
4650 else if (UNARY_CLASS_P (exp)
4651 ? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
4652 : unsignedp)
4653 op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
4654 else
4655 op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
4656
4657 return op0;
4658 }
4659
4660 case MEM_REF:
4661 if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4662 {
4663 tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
4664 TREE_OPERAND (exp, 0),
4665 TREE_OPERAND (exp, 1));
4666 if (newexp)
4667 return expand_debug_expr (newexp);
4668 }
4669 /* FALLTHROUGH */
4670 case INDIRECT_REF:
4671 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4672 op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4673 if (!op0)
4674 return NULL;
4675
4676 if (TREE_CODE (exp) == MEM_REF)
4677 {
4678 if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
4679 || (GET_CODE (op0) == PLUS
4680 && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
4681 /* (mem (debug_implicit_ptr)) might confuse aliasing.
4682 Instead just use get_inner_reference. */
4683 goto component_ref;
4684
4685 op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4686 poly_int64 offset;
4687 if (!op1 || !poly_int_rtx_p (op1, &offset))
4688 return NULL;
4689
4690 op0 = plus_constant (inner_mode, op0, offset);
4691 }
4692
4693 as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4694
4695 op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
4696 op0, as);
4697 if (op0 == NULL_RTX)
4698 return NULL;
4699
4700 op0 = gen_rtx_MEM (mode, op0);
4701 set_mem_attributes (op0, exp, 0);
4702 if (TREE_CODE (exp) == MEM_REF
4703 && !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4704 set_mem_expr (op0, NULL_TREE);
4705 set_mem_addr_space (op0, as);
4706
4707 return op0;
4708
4709 case TARGET_MEM_REF:
4710 if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
4711 && !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
4712 return NULL;
4713
4714 op0 = expand_debug_expr
4715 (tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
4716 if (!op0)
4717 return NULL;
4718
4719 as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4720 op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
4721 op0, as);
4722 if (op0 == NULL_RTX)
4723 return NULL;
4724
4725 op0 = gen_rtx_MEM (mode, op0);
4726
4727 set_mem_attributes (op0, exp, 0);
4728 set_mem_addr_space (op0, as);
4729
4730 return op0;
4731
4732 component_ref:
4733 case ARRAY_REF:
4734 case ARRAY_RANGE_REF:
4735 case COMPONENT_REF:
4736 case BIT_FIELD_REF:
4737 case REALPART_EXPR:
4738 case IMAGPART_EXPR:
4739 case VIEW_CONVERT_EXPR:
4740 {
4741 machine_mode mode1;
4742 poly_int64 bitsize, bitpos;
4743 tree offset;
4744 int reversep, volatilep = 0;
4745 tree tem
4746 = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
4747 &unsignedp, &reversep, &volatilep);
4748 rtx orig_op0;
4749
4750 if (known_eq (bitsize, 0))
4751 return NULL;
4752
4753 orig_op0 = op0 = expand_debug_expr (tem);
4754
4755 if (!op0)
4756 return NULL;
4757
4758 if (offset)
4759 {
4760 machine_mode addrmode, offmode;
4761
4762 if (!MEM_P (op0))
4763 return NULL;
4764
4765 op0 = XEXP (op0, 0);
4766 addrmode = GET_MODE (op0);
4767 if (addrmode == VOIDmode)
4768 addrmode = Pmode;
4769
4770 op1 = expand_debug_expr (offset);
4771 if (!op1)
4772 return NULL;
4773
4774 offmode = GET_MODE (op1);
4775 if (offmode == VOIDmode)
4776 offmode = TYPE_MODE (TREE_TYPE (offset));
4777
4778 if (addrmode != offmode)
4779 op1 = lowpart_subreg (addrmode, op1, offmode);
4780
4781 /* Don't use offset_address here, we don't need a
4782 recognizable address, and we don't want to generate
4783 code. */
4784 op0 = gen_rtx_MEM (mode, simplify_gen_binary (PLUS, addrmode,
4785 op0, op1));
4786 }
4787
4788 if (MEM_P (op0))
4789 {
4790 if (mode1 == VOIDmode)
4791 {
4792 if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
4793 return NULL;
4794 /* Bitfield. */
4795 mode1 = smallest_int_mode_for_size (bitsize);
4796 }
4797 poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
4798 if (maybe_ne (bytepos, 0))
4799 {
4800 op0 = adjust_address_nv (op0, mode1, bytepos);
4801 bitpos = num_trailing_bits (bitpos);
4802 }
4803 else if (known_eq (bitpos, 0)
4804 && known_eq (bitsize, GET_MODE_BITSIZE (mode)))
4805 op0 = adjust_address_nv (op0, mode, 0);
4806 else if (GET_MODE (op0) != mode1)
4807 op0 = adjust_address_nv (op0, mode1, 0);
4808 else
4809 op0 = copy_rtx (op0);
4810 if (op0 == orig_op0)
4811 op0 = shallow_copy_rtx (op0);
4812 if (TREE_CODE (tem) != SSA_NAME)
4813 set_mem_attributes (op0, exp, 0);
4814 }
4815
4816 if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
4817 return op0;
4818
4819 if (maybe_lt (bitpos, 0))
4820 return NULL;
4821
4822 if (GET_MODE (op0) == BLKmode || mode == BLKmode)
4823 return NULL;
4824
4825 poly_int64 bytepos;
4826 if (multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
4827 && known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
4828 {
4829 machine_mode opmode = GET_MODE (op0);
4830
4831 if (opmode == VOIDmode)
4832 opmode = TYPE_MODE (TREE_TYPE (tem));
4833
4834 /* This condition may hold if we're expanding the address
4835 right past the end of an array that turned out not to
4836 be addressable (i.e., the address was only computed in
4837 debug stmts). The gen_subreg below would rightfully
4838 crash, and the address doesn't really exist, so just
4839 drop it. */
4840 if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
4841 return NULL;
4842
4843 if (multiple_p (bitpos, GET_MODE_BITSIZE (mode)))
4844 return simplify_gen_subreg (mode, op0, opmode, bytepos);
4845 }
4846
4847 return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
4848 && TYPE_UNSIGNED (TREE_TYPE (exp))
4849 ? SIGN_EXTRACT
4850 : ZERO_EXTRACT, mode,
4851 GET_MODE (op0) != VOIDmode
4852 ? GET_MODE (op0)
4853 : TYPE_MODE (TREE_TYPE (tem)),
4854 op0, gen_int_mode (bitsize, word_mode),
4855 gen_int_mode (bitpos, word_mode));
4856 }
4857
4858 case ABS_EXPR:
4859 case ABSU_EXPR:
4860 return simplify_gen_unary (ABS, mode, op0, mode);
4861
4862 case NEGATE_EXPR:
4863 return simplify_gen_unary (NEG, mode, op0, mode);
4864
4865 case BIT_NOT_EXPR:
4866 return simplify_gen_unary (NOT, mode, op0, mode);
4867
4868 case FLOAT_EXPR:
4869 return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
4870 0)))
4871 ? UNSIGNED_FLOAT : FLOAT, mode, op0,
4872 inner_mode);
4873
4874 case FIX_TRUNC_EXPR:
4875 return simplify_gen_unary (unsignedp ? UNSIGNED_FIX : FIX, mode, op0,
4876 inner_mode);
4877
4878 case POINTER_PLUS_EXPR:
4879 /* For the rare target where pointers are not the same size as
4880 size_t, we need to check for mis-matched modes and correct
4881 the addend. */
4882 if (op0 && op1
4883 && is_a <scalar_int_mode> (GET_MODE (op0), &op0_mode)
4884 && is_a <scalar_int_mode> (GET_MODE (op1), &op1_mode)
4885 && op0_mode != op1_mode)
4886 {
4887 if (GET_MODE_BITSIZE (op0_mode) < GET_MODE_BITSIZE (op1_mode)
4888 /* If OP0 is a partial mode, then we must truncate, even
4889 if it has the same bitsize as OP1 as GCC's
4890 representation of partial modes is opaque. */
4891 || (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
4892 && (GET_MODE_BITSIZE (op0_mode)
4893 == GET_MODE_BITSIZE (op1_mode))))
4894 op1 = simplify_gen_unary (TRUNCATE, op0_mode, op1, op1_mode);
4895 else
4896 /* We always sign-extend, regardless of the signedness of
4897 the operand, because the operand is always unsigned
4898 here even if the original C expression is signed. */
4899 op1 = simplify_gen_unary (SIGN_EXTEND, op0_mode, op1, op1_mode);
4900 }
4901 /* Fall through. */
4902 case PLUS_EXPR:
4903 return simplify_gen_binary (PLUS, mode, op0, op1);
4904
4905 case MINUS_EXPR:
4906 case POINTER_DIFF_EXPR:
4907 return simplify_gen_binary (MINUS, mode, op0, op1);
4908
4909 case MULT_EXPR:
4910 return simplify_gen_binary (MULT, mode, op0, op1);
4911
4912 case RDIV_EXPR:
4913 case TRUNC_DIV_EXPR:
4914 case EXACT_DIV_EXPR:
4915 if (unsignedp)
4916 return simplify_gen_binary (UDIV, mode, op0, op1);
4917 else
4918 return simplify_gen_binary (DIV, mode, op0, op1);
4919
4920 case TRUNC_MOD_EXPR:
4921 return simplify_gen_binary (unsignedp ? UMOD : MOD, mode, op0, op1);
4922
4923 case FLOOR_DIV_EXPR:
4924 if (unsignedp)
4925 return simplify_gen_binary (UDIV, mode, op0, op1);
4926 else
4927 {
4928 rtx div = simplify_gen_binary (DIV, mode, op0, op1);
4929 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4930 rtx adj = floor_sdiv_adjust (mode, mod, op1);
4931 return simplify_gen_binary (PLUS, mode, div, adj);
4932 }
4933
4934 case FLOOR_MOD_EXPR:
4935 if (unsignedp)
4936 return simplify_gen_binary (UMOD, mode, op0, op1);
4937 else
4938 {
4939 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4940 rtx adj = floor_sdiv_adjust (mode, mod, op1);
4941 adj = simplify_gen_unary (NEG, mode,
4942 simplify_gen_binary (MULT, mode, adj, op1),
4943 mode);
4944 return simplify_gen_binary (PLUS, mode, mod, adj);
4945 }
4946
4947 case CEIL_DIV_EXPR:
4948 if (unsignedp)
4949 {
4950 rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
4951 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
4952 rtx adj = ceil_udiv_adjust (mode, mod, op1);
4953 return simplify_gen_binary (PLUS, mode, div, adj);
4954 }
4955 else
4956 {
4957 rtx div = simplify_gen_binary (DIV, mode, op0, op1);
4958 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4959 rtx adj = ceil_sdiv_adjust (mode, mod, op1);
4960 return simplify_gen_binary (PLUS, mode, div, adj);
4961 }
4962
4963 case CEIL_MOD_EXPR:
4964 if (unsignedp)
4965 {
4966 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
4967 rtx adj = ceil_udiv_adjust (mode, mod, op1);
4968 adj = simplify_gen_unary (NEG, mode,
4969 simplify_gen_binary (MULT, mode, adj, op1),
4970 mode);
4971 return simplify_gen_binary (PLUS, mode, mod, adj);
4972 }
4973 else
4974 {
4975 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4976 rtx adj = ceil_sdiv_adjust (mode, mod, op1);
4977 adj = simplify_gen_unary (NEG, mode,
4978 simplify_gen_binary (MULT, mode, adj, op1),
4979 mode);
4980 return simplify_gen_binary (PLUS, mode, mod, adj);
4981 }
4982
4983 case ROUND_DIV_EXPR:
4984 if (unsignedp)
4985 {
4986 rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
4987 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
4988 rtx adj = round_udiv_adjust (mode, mod, op1);
4989 return simplify_gen_binary (PLUS, mode, div, adj);
4990 }
4991 else
4992 {
4993 rtx div = simplify_gen_binary (DIV, mode, op0, op1);
4994 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4995 rtx adj = round_sdiv_adjust (mode, mod, op1);
4996 return simplify_gen_binary (PLUS, mode, div, adj);
4997 }
4998
4999 case ROUND_MOD_EXPR:
5000 if (unsignedp)
5001 {
5002 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5003 rtx adj = round_udiv_adjust (mode, mod, op1);
5004 adj = simplify_gen_unary (NEG, mode,
5005 simplify_gen_binary (MULT, mode, adj, op1),
5006 mode);
5007 return simplify_gen_binary (PLUS, mode, mod, adj);
5008 }
5009 else
5010 {
5011 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5012 rtx adj = round_sdiv_adjust (mode, mod, op1);
5013 adj = simplify_gen_unary (NEG, mode,
5014 simplify_gen_binary (MULT, mode, adj, op1),
5015 mode);
5016 return simplify_gen_binary (PLUS, mode, mod, adj);
5017 }
5018
5019 case LSHIFT_EXPR:
5020 return simplify_gen_binary (ASHIFT, mode, op0, op1);
5021
5022 case RSHIFT_EXPR:
5023 if (unsignedp)
5024 return simplify_gen_binary (LSHIFTRT, mode, op0, op1);
5025 else
5026 return simplify_gen_binary (ASHIFTRT, mode, op0, op1);
5027
5028 case LROTATE_EXPR:
5029 return simplify_gen_binary (ROTATE, mode, op0, op1);
5030
5031 case RROTATE_EXPR:
5032 return simplify_gen_binary (ROTATERT, mode, op0, op1);
5033
5034 case MIN_EXPR:
5035 return simplify_gen_binary (unsignedp ? UMIN : SMIN, mode, op0, op1);
5036
5037 case MAX_EXPR:
5038 return simplify_gen_binary (unsignedp ? UMAX : SMAX, mode, op0, op1);
5039
5040 case BIT_AND_EXPR:
5041 case TRUTH_AND_EXPR:
5042 return simplify_gen_binary (AND, mode, op0, op1);
5043
5044 case BIT_IOR_EXPR:
5045 case TRUTH_OR_EXPR:
5046 return simplify_gen_binary (IOR, mode, op0, op1);
5047
5048 case BIT_XOR_EXPR:
5049 case TRUTH_XOR_EXPR:
5050 return simplify_gen_binary (XOR, mode, op0, op1);
5051
5052 case TRUTH_ANDIF_EXPR:
5053 return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5054
5055 case TRUTH_ORIF_EXPR:
5056 return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5057
5058 case TRUTH_NOT_EXPR:
5059 return simplify_gen_relational (EQ, mode, inner_mode, op0, const0_rtx);
5060
5061 case LT_EXPR:
5062 return simplify_gen_relational (unsignedp ? LTU : LT, mode, inner_mode,
5063 op0, op1);
5064
5065 case LE_EXPR:
5066 return simplify_gen_relational (unsignedp ? LEU : LE, mode, inner_mode,
5067 op0, op1);
5068
5069 case GT_EXPR:
5070 return simplify_gen_relational (unsignedp ? GTU : GT, mode, inner_mode,
5071 op0, op1);
5072
5073 case GE_EXPR:
5074 return simplify_gen_relational (unsignedp ? GEU : GE, mode, inner_mode,
5075 op0, op1);
5076
5077 case EQ_EXPR:
5078 return simplify_gen_relational (EQ, mode, inner_mode, op0, op1);
5079
5080 case NE_EXPR:
5081 return simplify_gen_relational (NE, mode, inner_mode, op0, op1);
5082
5083 case UNORDERED_EXPR:
5084 return simplify_gen_relational (UNORDERED, mode, inner_mode, op0, op1);
5085
5086 case ORDERED_EXPR:
5087 return simplify_gen_relational (ORDERED, mode, inner_mode, op0, op1);
5088
5089 case UNLT_EXPR:
5090 return simplify_gen_relational (UNLT, mode, inner_mode, op0, op1);
5091
5092 case UNLE_EXPR:
5093 return simplify_gen_relational (UNLE, mode, inner_mode, op0, op1);
5094
5095 case UNGT_EXPR:
5096 return simplify_gen_relational (UNGT, mode, inner_mode, op0, op1);
5097
5098 case UNGE_EXPR:
5099 return simplify_gen_relational (UNGE, mode, inner_mode, op0, op1);
5100
5101 case UNEQ_EXPR:
5102 return simplify_gen_relational (UNEQ, mode, inner_mode, op0, op1);
5103
5104 case LTGT_EXPR:
5105 return simplify_gen_relational (LTGT, mode, inner_mode, op0, op1);
5106
5107 case COND_EXPR:
5108 return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5109
5110 case COMPLEX_EXPR:
5111 gcc_assert (COMPLEX_MODE_P (mode));
5112 if (GET_MODE (op0) == VOIDmode)
5113 op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5114 if (GET_MODE (op1) == VOIDmode)
5115 op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5116 return gen_rtx_CONCAT (mode, op0, op1);
5117
5118 case CONJ_EXPR:
5119 if (GET_CODE (op0) == CONCAT)
5120 return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5121 simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5122 XEXP (op0, 1),
5123 GET_MODE_INNER (mode)));
5124 else
5125 {
5126 scalar_mode imode = GET_MODE_INNER (mode);
5127 rtx re, im;
5128
5129 if (MEM_P (op0))
5130 {
5131 re = adjust_address_nv (op0, imode, 0);
5132 im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5133 }
5134 else
5135 {
5136 scalar_int_mode ifmode;
5137 scalar_int_mode ihmode;
5138 rtx halfsize;
5139 if (!int_mode_for_mode (mode).exists (&ifmode)
5140 || !int_mode_for_mode (imode).exists (&ihmode))
5141 return NULL;
5142 halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5143 re = op0;
5144 if (mode != ifmode)
5145 re = gen_rtx_SUBREG (ifmode, re, 0);
5146 re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5147 if (imode != ihmode)
5148 re = gen_rtx_SUBREG (imode, re, 0);
5149 im = copy_rtx (op0);
5150 if (mode != ifmode)
5151 im = gen_rtx_SUBREG (ifmode, im, 0);
5152 im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5153 if (imode != ihmode)
5154 im = gen_rtx_SUBREG (imode, im, 0);
5155 }
5156 im = gen_rtx_NEG (imode, im);
5157 return gen_rtx_CONCAT (mode, re, im);
5158 }
5159
5160 case ADDR_EXPR:
5161 op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5162 if (!op0 || !MEM_P (op0))
5163 {
5164 if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5165 || TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5166 || TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5167 && (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5168 || target_for_debug_bind (TREE_OPERAND (exp, 0))))
5169 return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5170
5171 if (handled_component_p (TREE_OPERAND (exp, 0)))
5172 {
5173 poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5174 bool reverse;
5175 tree decl
5176 = get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5177 &bitsize, &maxsize, &reverse);
5178 if ((VAR_P (decl)
5179 || TREE_CODE (decl) == PARM_DECL
5180 || TREE_CODE (decl) == RESULT_DECL)
5181 && (!TREE_ADDRESSABLE (decl)
5182 || target_for_debug_bind (decl))
5183 && multiple_p (bitoffset, BITS_PER_UNIT, &byteoffset)
5184 && known_gt (bitsize, 0)
5185 && known_eq (bitsize, maxsize))
5186 {
5187 rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5188 return plus_constant (mode, base, byteoffset);
5189 }
5190 }
5191
5192 if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5193 && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5194 == ADDR_EXPR)
5195 {
5196 op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5197 0));
5198 if (op0 != NULL
5199 && (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5200 || (GET_CODE (op0) == PLUS
5201 && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5202 && CONST_INT_P (XEXP (op0, 1)))))
5203 {
5204 op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5205 1));
5206 poly_int64 offset;
5207 if (!op1 || !poly_int_rtx_p (op1, &offset))
5208 return NULL;
5209
5210 return plus_constant (mode, op0, offset);
5211 }
5212 }
5213
5214 return NULL;
5215 }
5216
5217 as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5218 addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5219 op0 = convert_debug_memory_address (addr_mode, XEXP (op0, 0), as);
5220
5221 return op0;
5222
5223 case VECTOR_CST:
5224 {
5225 unsigned HOST_WIDE_INT i, nelts;
5226
5227 if (!VECTOR_CST_NELTS (exp).is_constant (&nelts))
5228 return NULL;
5229
5230 op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5231
5232 for (i = 0; i < nelts; ++i)
5233 {
5234 op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5235 if (!op1)
5236 return NULL;
5237 XVECEXP (op0, 0, i) = op1;
5238 }
5239
5240 return op0;
5241 }
5242
5243 case CONSTRUCTOR:
5244 if (TREE_CLOBBER_P (exp))
5245 return NULL;
5246 else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5247 {
5248 unsigned i;
5249 unsigned HOST_WIDE_INT nelts;
5250 tree val;
5251
5252 if (!TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp)).is_constant (&nelts))
5253 goto flag_unsupported;
5254
5255 op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5256
5257 FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5258 {
5259 op1 = expand_debug_expr (val);
5260 if (!op1)
5261 return NULL;
5262 XVECEXP (op0, 0, i) = op1;
5263 }
5264
5265 if (i < nelts)
5266 {
5267 op1 = expand_debug_expr
5268 (build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5269
5270 if (!op1)
5271 return NULL;
5272
5273 for (; i < nelts; i++)
5274 XVECEXP (op0, 0, i) = op1;
5275 }
5276
5277 return op0;
5278 }
5279 else
5280 goto flag_unsupported;
5281
5282 case CALL_EXPR:
5283 /* ??? Maybe handle some builtins? */
5284 return NULL;
5285
5286 case SSA_NAME:
5287 {
5288 gimple *g = get_gimple_for_ssa_name (exp);
5289 if (g)
5290 {
5291 tree t = NULL_TREE;
5292 if (deep_ter_debug_map)
5293 {
5294 tree *slot = deep_ter_debug_map->get (exp);
5295 if (slot)
5296 t = *slot;
5297 }
5298 if (t == NULL_TREE)
5299 t = gimple_assign_rhs_to_tree (g);
5300 op0 = expand_debug_expr (t);
5301 if (!op0)
5302 return NULL;
5303 }
5304 else
5305 {
5306 /* If this is a reference to an incoming value of
5307 parameter that is never used in the code or where the
5308 incoming value is never used in the code, use
5309 PARM_DECL's DECL_RTL if set. */
5310 if (SSA_NAME_IS_DEFAULT_DEF (exp)
5311 && SSA_NAME_VAR (exp)
5312 && TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5313 && has_zero_uses (exp))
5314 {
5315 op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5316 if (op0)
5317 goto adjust_mode;
5318 op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5319 if (op0)
5320 goto adjust_mode;
5321 }
5322
5323 int part = var_to_partition (SA.map, exp);
5324
5325 if (part == NO_PARTITION)
5326 return NULL;
5327
5328 gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5329
5330 op0 = copy_rtx (SA.partition_to_pseudo[part]);
5331 }
5332 goto adjust_mode;
5333 }
5334
5335 case ERROR_MARK:
5336 return NULL;
5337
5338 /* Vector stuff. For most of the codes we don't have rtl codes. */
5339 case REALIGN_LOAD_EXPR:
5340 case VEC_COND_EXPR:
5341 case VEC_PACK_FIX_TRUNC_EXPR:
5342 case VEC_PACK_FLOAT_EXPR:
5343 case VEC_PACK_SAT_EXPR:
5344 case VEC_PACK_TRUNC_EXPR:
5345 case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5346 case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5347 case VEC_UNPACK_FLOAT_HI_EXPR:
5348 case VEC_UNPACK_FLOAT_LO_EXPR:
5349 case VEC_UNPACK_HI_EXPR:
5350 case VEC_UNPACK_LO_EXPR:
5351 case VEC_WIDEN_MULT_HI_EXPR:
5352 case VEC_WIDEN_MULT_LO_EXPR:
5353 case VEC_WIDEN_MULT_EVEN_EXPR:
5354 case VEC_WIDEN_MULT_ODD_EXPR:
5355 case VEC_WIDEN_LSHIFT_HI_EXPR:
5356 case VEC_WIDEN_LSHIFT_LO_EXPR:
5357 case VEC_PERM_EXPR:
5358 case VEC_DUPLICATE_EXPR:
5359 case VEC_SERIES_EXPR:
5360 case SAD_EXPR:
5361 return NULL;
5362
5363 /* Misc codes. */
5364 case ADDR_SPACE_CONVERT_EXPR:
5365 case FIXED_CONVERT_EXPR:
5366 case OBJ_TYPE_REF:
5367 case WITH_SIZE_EXPR:
5368 case BIT_INSERT_EXPR:
5369 return NULL;
5370
5371 case DOT_PROD_EXPR:
5372 if (SCALAR_INT_MODE_P (GET_MODE (op0))
5373 && SCALAR_INT_MODE_P (mode))
5374 {
5375 op0
5376 = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5377 0)))
5378 ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5379 inner_mode);
5380 op1
5381 = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5382 1)))
5383 ? ZERO_EXTEND : SIGN_EXTEND, mode, op1,
5384 inner_mode);
5385 op0 = simplify_gen_binary (MULT, mode, op0, op1);
5386 return simplify_gen_binary (PLUS, mode, op0, op2);
5387 }
5388 return NULL;
5389
5390 case WIDEN_MULT_EXPR:
5391 case WIDEN_MULT_PLUS_EXPR:
5392 case WIDEN_MULT_MINUS_EXPR:
5393 if (SCALAR_INT_MODE_P (GET_MODE (op0))
5394 && SCALAR_INT_MODE_P (mode))
5395 {
5396 inner_mode = GET_MODE (op0);
5397 if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5398 op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5399 else
5400 op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5401 if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5402 op1 = simplify_gen_unary (ZERO_EXTEND, mode, op1, inner_mode);
5403 else
5404 op1 = simplify_gen_unary (SIGN_EXTEND, mode, op1, inner_mode);
5405 op0 = simplify_gen_binary (MULT, mode, op0, op1);
5406 if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5407 return op0;
5408 else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5409 return simplify_gen_binary (PLUS, mode, op0, op2);
5410 else
5411 return simplify_gen_binary (MINUS, mode, op2, op0);
5412 }
5413 return NULL;
5414
5415 case MULT_HIGHPART_EXPR:
5416 /* ??? Similar to the above. */
5417 return NULL;
5418
5419 case WIDEN_SUM_EXPR:
5420 case WIDEN_LSHIFT_EXPR:
5421 if (SCALAR_INT_MODE_P (GET_MODE (op0))
5422 && SCALAR_INT_MODE_P (mode))
5423 {
5424 op0
5425 = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5426 0)))
5427 ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5428 inner_mode);
5429 return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5430 ? ASHIFT : PLUS, mode, op0, op1);
5431 }
5432 return NULL;
5433
5434 default:
5435 flag_unsupported:
5436 if (flag_checking)
5437 {
5438 debug_tree (exp);
5439 gcc_unreachable ();
5440 }
5441 return NULL;
5442 }
5443 }
5444
5445 /* Return an RTX equivalent to the source bind value of the tree expression
5446 EXP. */
5447
5448 static rtx
expand_debug_source_expr(tree exp)5449 expand_debug_source_expr (tree exp)
5450 {
5451 rtx op0 = NULL_RTX;
5452 machine_mode mode = VOIDmode, inner_mode;
5453
5454 switch (TREE_CODE (exp))
5455 {
5456 case VAR_DECL:
5457 if (DECL_ABSTRACT_ORIGIN (exp))
5458 return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5459 break;
5460 case PARM_DECL:
5461 {
5462 mode = DECL_MODE (exp);
5463 op0 = expand_debug_parm_decl (exp);
5464 if (op0)
5465 break;
5466 /* See if this isn't an argument that has been completely
5467 optimized out. */
5468 if (!DECL_RTL_SET_P (exp)
5469 && !DECL_INCOMING_RTL (exp)
5470 && DECL_ABSTRACT_ORIGIN (current_function_decl))
5471 {
5472 tree aexp = DECL_ORIGIN (exp);
5473 if (DECL_CONTEXT (aexp)
5474 == DECL_ABSTRACT_ORIGIN (current_function_decl))
5475 {
5476 vec<tree, va_gc> **debug_args;
5477 unsigned int ix;
5478 tree ddecl;
5479 debug_args = decl_debug_args_lookup (current_function_decl);
5480 if (debug_args != NULL)
5481 {
5482 for (ix = 0; vec_safe_iterate (*debug_args, ix, &ddecl);
5483 ix += 2)
5484 if (ddecl == aexp)
5485 return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5486 }
5487 }
5488 }
5489 break;
5490 }
5491 default:
5492 break;
5493 }
5494
5495 if (op0 == NULL_RTX)
5496 return NULL_RTX;
5497
5498 inner_mode = GET_MODE (op0);
5499 if (mode == inner_mode)
5500 return op0;
5501
5502 if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5503 {
5504 if (GET_MODE_UNIT_BITSIZE (mode)
5505 == GET_MODE_UNIT_BITSIZE (inner_mode))
5506 op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5507 else if (GET_MODE_UNIT_BITSIZE (mode)
5508 < GET_MODE_UNIT_BITSIZE (inner_mode))
5509 op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5510 else
5511 op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5512 }
5513 else if (FLOAT_MODE_P (mode))
5514 gcc_unreachable ();
5515 else if (FLOAT_MODE_P (inner_mode))
5516 {
5517 if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5518 op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5519 else
5520 op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5521 }
5522 else if (GET_MODE_UNIT_PRECISION (mode)
5523 == GET_MODE_UNIT_PRECISION (inner_mode))
5524 op0 = lowpart_subreg (mode, op0, inner_mode);
5525 else if (GET_MODE_UNIT_PRECISION (mode)
5526 < GET_MODE_UNIT_PRECISION (inner_mode))
5527 op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5528 else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5529 op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5530 else
5531 op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5532
5533 return op0;
5534 }
5535
5536 /* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5537 Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
5538 deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
5539
5540 static void
avoid_complex_debug_insns(rtx_insn * insn,rtx * exp_p,int depth)5541 avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
5542 {
5543 rtx exp = *exp_p;
5544
5545 if (exp == NULL_RTX)
5546 return;
5547
5548 if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
5549 return;
5550
5551 if (depth == 4)
5552 {
5553 /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
5554 rtx dval = make_debug_expr_from_rtl (exp);
5555
5556 /* Emit a debug bind insn before INSN. */
5557 rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
5558 DEBUG_EXPR_TREE_DECL (dval), exp,
5559 VAR_INIT_STATUS_INITIALIZED);
5560
5561 emit_debug_insn_before (bind, insn);
5562 *exp_p = dval;
5563 return;
5564 }
5565
5566 const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
5567 int i, j;
5568 for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
5569 switch (*format_ptr++)
5570 {
5571 case 'e':
5572 avoid_complex_debug_insns (insn, &XEXP (exp, i), depth + 1);
5573 break;
5574
5575 case 'E':
5576 case 'V':
5577 for (j = 0; j < XVECLEN (exp, i); j++)
5578 avoid_complex_debug_insns (insn, &XVECEXP (exp, i, j), depth + 1);
5579 break;
5580
5581 default:
5582 break;
5583 }
5584 }
5585
5586 /* Expand the _LOCs in debug insns. We run this after expanding all
5587 regular insns, so that any variables referenced in the function
5588 will have their DECL_RTLs set. */
5589
5590 static void
expand_debug_locations(void)5591 expand_debug_locations (void)
5592 {
5593 rtx_insn *insn;
5594 rtx_insn *last = get_last_insn ();
5595 int save_strict_alias = flag_strict_aliasing;
5596
5597 /* New alias sets while setting up memory attributes cause
5598 -fcompare-debug failures, even though it doesn't bring about any
5599 codegen changes. */
5600 flag_strict_aliasing = 0;
5601
5602 for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
5603 if (DEBUG_BIND_INSN_P (insn))
5604 {
5605 tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
5606 rtx val;
5607 rtx_insn *prev_insn, *insn2;
5608 machine_mode mode;
5609
5610 if (value == NULL_TREE)
5611 val = NULL_RTX;
5612 else
5613 {
5614 if (INSN_VAR_LOCATION_STATUS (insn)
5615 == VAR_INIT_STATUS_UNINITIALIZED)
5616 val = expand_debug_source_expr (value);
5617 /* The avoid_deep_ter_for_debug function inserts
5618 debug bind stmts after SSA_NAME definition, with the
5619 SSA_NAME as the whole bind location. Disable temporarily
5620 expansion of that SSA_NAME into the DEBUG_EXPR_DECL
5621 being defined in this DEBUG_INSN. */
5622 else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
5623 {
5624 tree *slot = deep_ter_debug_map->get (value);
5625 if (slot)
5626 {
5627 if (*slot == INSN_VAR_LOCATION_DECL (insn))
5628 *slot = NULL_TREE;
5629 else
5630 slot = NULL;
5631 }
5632 val = expand_debug_expr (value);
5633 if (slot)
5634 *slot = INSN_VAR_LOCATION_DECL (insn);
5635 }
5636 else
5637 val = expand_debug_expr (value);
5638 gcc_assert (last == get_last_insn ());
5639 }
5640
5641 if (!val)
5642 val = gen_rtx_UNKNOWN_VAR_LOC ();
5643 else
5644 {
5645 mode = GET_MODE (INSN_VAR_LOCATION (insn));
5646
5647 gcc_assert (mode == GET_MODE (val)
5648 || (GET_MODE (val) == VOIDmode
5649 && (CONST_SCALAR_INT_P (val)
5650 || GET_CODE (val) == CONST_FIXED
5651 || GET_CODE (val) == LABEL_REF)));
5652 }
5653
5654 INSN_VAR_LOCATION_LOC (insn) = val;
5655 prev_insn = PREV_INSN (insn);
5656 for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn2))
5657 avoid_complex_debug_insns (insn2, &INSN_VAR_LOCATION_LOC (insn2), 0);
5658 }
5659
5660 flag_strict_aliasing = save_strict_alias;
5661 }
5662
5663 /* Performs swapping operands of commutative operations to expand
5664 the expensive one first. */
5665
5666 static void
reorder_operands(basic_block bb)5667 reorder_operands (basic_block bb)
5668 {
5669 unsigned int *lattice; /* Hold cost of each statement. */
5670 unsigned int i = 0, n = 0;
5671 gimple_stmt_iterator gsi;
5672 gimple_seq stmts;
5673 gimple *stmt;
5674 bool swap;
5675 tree op0, op1;
5676 ssa_op_iter iter;
5677 use_operand_p use_p;
5678 gimple *def0, *def1;
5679
5680 /* Compute cost of each statement using estimate_num_insns. */
5681 stmts = bb_seq (bb);
5682 for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
5683 {
5684 stmt = gsi_stmt (gsi);
5685 if (!is_gimple_debug (stmt))
5686 gimple_set_uid (stmt, n++);
5687 }
5688 lattice = XNEWVEC (unsigned int, n);
5689 for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
5690 {
5691 unsigned cost;
5692 stmt = gsi_stmt (gsi);
5693 if (is_gimple_debug (stmt))
5694 continue;
5695 cost = estimate_num_insns (stmt, &eni_size_weights);
5696 lattice[i] = cost;
5697 FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
5698 {
5699 tree use = USE_FROM_PTR (use_p);
5700 gimple *def_stmt;
5701 if (TREE_CODE (use) != SSA_NAME)
5702 continue;
5703 def_stmt = get_gimple_for_ssa_name (use);
5704 if (!def_stmt)
5705 continue;
5706 lattice[i] += lattice[gimple_uid (def_stmt)];
5707 }
5708 i++;
5709 if (!is_gimple_assign (stmt)
5710 || !commutative_tree_code (gimple_assign_rhs_code (stmt)))
5711 continue;
5712 op0 = gimple_op (stmt, 1);
5713 op1 = gimple_op (stmt, 2);
5714 if (TREE_CODE (op0) != SSA_NAME
5715 || TREE_CODE (op1) != SSA_NAME)
5716 continue;
5717 /* Swap operands if the second one is more expensive. */
5718 def0 = get_gimple_for_ssa_name (op0);
5719 def1 = get_gimple_for_ssa_name (op1);
5720 if (!def1)
5721 continue;
5722 swap = false;
5723 if (!def0 || lattice[gimple_uid (def1)] > lattice[gimple_uid (def0)])
5724 swap = true;
5725 if (swap)
5726 {
5727 if (dump_file && (dump_flags & TDF_DETAILS))
5728 {
5729 fprintf (dump_file, "Swap operands in stmt:\n");
5730 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
5731 fprintf (dump_file, "Cost left opnd=%d, right opnd=%d\n",
5732 def0 ? lattice[gimple_uid (def0)] : 0,
5733 lattice[gimple_uid (def1)]);
5734 }
5735 swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
5736 gimple_assign_rhs2_ptr (stmt));
5737 }
5738 }
5739 XDELETE (lattice);
5740 }
5741
5742 /* Expand basic block BB from GIMPLE trees to RTL. */
5743
5744 static basic_block
expand_gimple_basic_block(basic_block bb,bool disable_tail_calls)5745 expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
5746 {
5747 gimple_stmt_iterator gsi;
5748 gimple_seq stmts;
5749 gimple *stmt = NULL;
5750 rtx_note *note = NULL;
5751 rtx_insn *last;
5752 edge e;
5753 edge_iterator ei;
5754
5755 if (dump_file)
5756 fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
5757 bb->index);
5758
5759 /* Note that since we are now transitioning from GIMPLE to RTL, we
5760 cannot use the gsi_*_bb() routines because they expect the basic
5761 block to be in GIMPLE, instead of RTL. Therefore, we need to
5762 access the BB sequence directly. */
5763 if (optimize)
5764 reorder_operands (bb);
5765 stmts = bb_seq (bb);
5766 bb->il.gimple.seq = NULL;
5767 bb->il.gimple.phi_nodes = NULL;
5768 rtl_profile_for_bb (bb);
5769 init_rtl_bb_info (bb);
5770 bb->flags |= BB_RTL;
5771
5772 /* Remove the RETURN_EXPR if we may fall though to the exit
5773 instead. */
5774 gsi = gsi_last (stmts);
5775 if (!gsi_end_p (gsi)
5776 && gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
5777 {
5778 greturn *ret_stmt = as_a <greturn *> (gsi_stmt (gsi));
5779
5780 gcc_assert (single_succ_p (bb));
5781 gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
5782
5783 if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
5784 && !gimple_return_retval (ret_stmt))
5785 {
5786 gsi_remove (&gsi, false);
5787 single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
5788 }
5789 }
5790
5791 gsi = gsi_start (stmts);
5792 if (!gsi_end_p (gsi))
5793 {
5794 stmt = gsi_stmt (gsi);
5795 if (gimple_code (stmt) != GIMPLE_LABEL)
5796 stmt = NULL;
5797 }
5798
5799 rtx_code_label **elt = lab_rtx_for_bb->get (bb);
5800
5801 if (stmt || elt)
5802 {
5803 gcc_checking_assert (!note);
5804 last = get_last_insn ();
5805
5806 if (stmt)
5807 {
5808 expand_gimple_stmt (stmt);
5809 gsi_next (&gsi);
5810 }
5811
5812 if (elt)
5813 emit_label (*elt);
5814
5815 BB_HEAD (bb) = NEXT_INSN (last);
5816 if (NOTE_P (BB_HEAD (bb)))
5817 BB_HEAD (bb) = NEXT_INSN (BB_HEAD (bb));
5818 gcc_assert (LABEL_P (BB_HEAD (bb)));
5819 note = emit_note_after (NOTE_INSN_BASIC_BLOCK, BB_HEAD (bb));
5820
5821 maybe_dump_rtl_for_gimple_stmt (stmt, last);
5822 }
5823 else
5824 BB_HEAD (bb) = note = emit_note (NOTE_INSN_BASIC_BLOCK);
5825
5826 if (note)
5827 NOTE_BASIC_BLOCK (note) = bb;
5828
5829 for (; !gsi_end_p (gsi); gsi_next (&gsi))
5830 {
5831 basic_block new_bb;
5832
5833 stmt = gsi_stmt (gsi);
5834
5835 /* If this statement is a non-debug one, and we generate debug
5836 insns, then this one might be the last real use of a TERed
5837 SSA_NAME, but where there are still some debug uses further
5838 down. Expanding the current SSA name in such further debug
5839 uses by their RHS might lead to wrong debug info, as coalescing
5840 might make the operands of such RHS be placed into the same
5841 pseudo as something else. Like so:
5842 a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
5843 use(a_1);
5844 a_2 = ...
5845 #DEBUG ... => a_1
5846 As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
5847 If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
5848 the write to a_2 would actually have clobbered the place which
5849 formerly held a_0.
5850
5851 So, instead of that, we recognize the situation, and generate
5852 debug temporaries at the last real use of TERed SSA names:
5853 a_1 = a_0 + 1;
5854 #DEBUG #D1 => a_1
5855 use(a_1);
5856 a_2 = ...
5857 #DEBUG ... => #D1
5858 */
5859 if (MAY_HAVE_DEBUG_BIND_INSNS
5860 && SA.values
5861 && !is_gimple_debug (stmt))
5862 {
5863 ssa_op_iter iter;
5864 tree op;
5865 gimple *def;
5866
5867 location_t sloc = curr_insn_location ();
5868
5869 /* Look for SSA names that have their last use here (TERed
5870 names always have only one real use). */
5871 FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
5872 if ((def = get_gimple_for_ssa_name (op)))
5873 {
5874 imm_use_iterator imm_iter;
5875 use_operand_p use_p;
5876 bool have_debug_uses = false;
5877
5878 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
5879 {
5880 if (gimple_debug_bind_p (USE_STMT (use_p)))
5881 {
5882 have_debug_uses = true;
5883 break;
5884 }
5885 }
5886
5887 if (have_debug_uses)
5888 {
5889 /* OP is a TERed SSA name, with DEF its defining
5890 statement, and where OP is used in further debug
5891 instructions. Generate a debug temporary, and
5892 replace all uses of OP in debug insns with that
5893 temporary. */
5894 gimple *debugstmt;
5895 tree value = gimple_assign_rhs_to_tree (def);
5896 tree vexpr = make_node (DEBUG_EXPR_DECL);
5897 rtx val;
5898 machine_mode mode;
5899
5900 set_curr_insn_location (gimple_location (def));
5901
5902 DECL_ARTIFICIAL (vexpr) = 1;
5903 TREE_TYPE (vexpr) = TREE_TYPE (value);
5904 if (DECL_P (value))
5905 mode = DECL_MODE (value);
5906 else
5907 mode = TYPE_MODE (TREE_TYPE (value));
5908 SET_DECL_MODE (vexpr, mode);
5909
5910 val = gen_rtx_VAR_LOCATION
5911 (mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5912
5913 emit_debug_insn (val);
5914
5915 FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
5916 {
5917 if (!gimple_debug_bind_p (debugstmt))
5918 continue;
5919
5920 FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
5921 SET_USE (use_p, vexpr);
5922
5923 update_stmt (debugstmt);
5924 }
5925 }
5926 }
5927 set_curr_insn_location (sloc);
5928 }
5929
5930 currently_expanding_gimple_stmt = stmt;
5931
5932 /* Expand this statement, then evaluate the resulting RTL and
5933 fixup the CFG accordingly. */
5934 if (gimple_code (stmt) == GIMPLE_COND)
5935 {
5936 new_bb = expand_gimple_cond (bb, as_a <gcond *> (stmt));
5937 if (new_bb)
5938 return new_bb;
5939 }
5940 else if (is_gimple_debug (stmt))
5941 {
5942 location_t sloc = curr_insn_location ();
5943 gimple_stmt_iterator nsi = gsi;
5944
5945 for (;;)
5946 {
5947 tree var;
5948 tree value = NULL_TREE;
5949 rtx val = NULL_RTX;
5950 machine_mode mode;
5951
5952 if (!gimple_debug_nonbind_marker_p (stmt))
5953 {
5954 if (gimple_debug_bind_p (stmt))
5955 {
5956 var = gimple_debug_bind_get_var (stmt);
5957
5958 if (TREE_CODE (var) != DEBUG_EXPR_DECL
5959 && TREE_CODE (var) != LABEL_DECL
5960 && !target_for_debug_bind (var))
5961 goto delink_debug_stmt;
5962
5963 if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
5964 mode = DECL_MODE (var);
5965 else
5966 mode = TYPE_MODE (TREE_TYPE (var));
5967
5968 if (gimple_debug_bind_has_value_p (stmt))
5969 value = gimple_debug_bind_get_value (stmt);
5970
5971 val = gen_rtx_VAR_LOCATION
5972 (mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5973 }
5974 else if (gimple_debug_source_bind_p (stmt))
5975 {
5976 var = gimple_debug_source_bind_get_var (stmt);
5977
5978 value = gimple_debug_source_bind_get_value (stmt);
5979
5980 if (!VECTOR_TYPE_P (TREE_TYPE (var)))
5981 mode = DECL_MODE (var);
5982 else
5983 mode = TYPE_MODE (TREE_TYPE (var));
5984
5985 val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
5986 VAR_INIT_STATUS_UNINITIALIZED);
5987 }
5988 else
5989 gcc_unreachable ();
5990 }
5991 /* If this function was first compiled with markers
5992 enabled, but they're now disable (e.g. LTO), drop
5993 them on the floor. */
5994 else if (gimple_debug_nonbind_marker_p (stmt)
5995 && !MAY_HAVE_DEBUG_MARKER_INSNS)
5996 goto delink_debug_stmt;
5997 else if (gimple_debug_begin_stmt_p (stmt))
5998 val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
5999 else if (gimple_debug_inline_entry_p (stmt))
6000 val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6001 else
6002 gcc_unreachable ();
6003
6004 last = get_last_insn ();
6005
6006 set_curr_insn_location (gimple_location (stmt));
6007
6008 emit_debug_insn (val);
6009
6010 if (dump_file && (dump_flags & TDF_DETAILS))
6011 {
6012 /* We can't dump the insn with a TREE where an RTX
6013 is expected. */
6014 if (GET_CODE (val) == VAR_LOCATION)
6015 {
6016 gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6017 PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6018 }
6019 maybe_dump_rtl_for_gimple_stmt (stmt, last);
6020 if (GET_CODE (val) == VAR_LOCATION)
6021 PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6022 }
6023
6024 delink_debug_stmt:
6025 /* In order not to generate too many debug temporaries,
6026 we delink all uses of debug statements we already expanded.
6027 Therefore debug statements between definition and real
6028 use of TERed SSA names will continue to use the SSA name,
6029 and not be replaced with debug temps. */
6030 delink_stmt_imm_use (stmt);
6031
6032 gsi = nsi;
6033 gsi_next (&nsi);
6034 if (gsi_end_p (nsi))
6035 break;
6036 stmt = gsi_stmt (nsi);
6037 if (!is_gimple_debug (stmt))
6038 break;
6039 }
6040
6041 set_curr_insn_location (sloc);
6042 }
6043 else
6044 {
6045 gcall *call_stmt = dyn_cast <gcall *> (stmt);
6046 if (call_stmt
6047 && gimple_call_tail_p (call_stmt)
6048 && disable_tail_calls)
6049 gimple_call_set_tail (call_stmt, false);
6050
6051 if (call_stmt && gimple_call_tail_p (call_stmt))
6052 {
6053 bool can_fallthru;
6054 new_bb = expand_gimple_tailcall (bb, call_stmt, &can_fallthru);
6055 if (new_bb)
6056 {
6057 if (can_fallthru)
6058 bb = new_bb;
6059 else
6060 return new_bb;
6061 }
6062 }
6063 else
6064 {
6065 def_operand_p def_p;
6066 def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6067
6068 if (def_p != NULL)
6069 {
6070 /* Ignore this stmt if it is in the list of
6071 replaceable expressions. */
6072 if (SA.values
6073 && bitmap_bit_p (SA.values,
6074 SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6075 continue;
6076 }
6077 last = expand_gimple_stmt (stmt);
6078 maybe_dump_rtl_for_gimple_stmt (stmt, last);
6079 }
6080 }
6081 }
6082
6083 currently_expanding_gimple_stmt = NULL;
6084
6085 /* Expand implicit goto and convert goto_locus. */
6086 FOR_EACH_EDGE (e, ei, bb->succs)
6087 {
6088 if (e->goto_locus != UNKNOWN_LOCATION)
6089 set_curr_insn_location (e->goto_locus);
6090 if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6091 {
6092 emit_jump (label_rtx_for_bb (e->dest));
6093 e->flags &= ~EDGE_FALLTHRU;
6094 }
6095 }
6096
6097 /* Expanded RTL can create a jump in the last instruction of block.
6098 This later might be assumed to be a jump to successor and break edge insertion.
6099 We need to insert dummy move to prevent this. PR41440. */
6100 if (single_succ_p (bb)
6101 && (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6102 && (last = get_last_insn ())
6103 && (JUMP_P (last)
6104 || (DEBUG_INSN_P (last)
6105 && JUMP_P (prev_nondebug_insn (last)))))
6106 {
6107 rtx dummy = gen_reg_rtx (SImode);
6108 emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6109 }
6110
6111 do_pending_stack_adjust ();
6112
6113 /* Find the block tail. The last insn in the block is the insn
6114 before a barrier and/or table jump insn. */
6115 last = get_last_insn ();
6116 if (BARRIER_P (last))
6117 last = PREV_INSN (last);
6118 if (JUMP_TABLE_DATA_P (last))
6119 last = PREV_INSN (PREV_INSN (last));
6120 if (BARRIER_P (last))
6121 last = PREV_INSN (last);
6122 BB_END (bb) = last;
6123
6124 update_bb_for_insn (bb);
6125
6126 return bb;
6127 }
6128
6129
6130 /* Create a basic block for initialization code. */
6131
6132 static basic_block
construct_init_block(void)6133 construct_init_block (void)
6134 {
6135 basic_block init_block, first_block;
6136 edge e = NULL;
6137 int flags;
6138
6139 /* Multiple entry points not supported yet. */
6140 gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6141 init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6142 init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6143 ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6144 EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6145
6146 e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6147
6148 /* When entry edge points to first basic block, we don't need jump,
6149 otherwise we have to jump into proper target. */
6150 if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6151 {
6152 tree label = gimple_block_label (e->dest);
6153
6154 emit_jump (jump_target_rtx (label));
6155 flags = 0;
6156 }
6157 else
6158 flags = EDGE_FALLTHRU;
6159
6160 init_block = create_basic_block (NEXT_INSN (get_insns ()),
6161 get_last_insn (),
6162 ENTRY_BLOCK_PTR_FOR_FN (cfun));
6163 init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6164 add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6165 if (e)
6166 {
6167 first_block = e->dest;
6168 redirect_edge_succ (e, init_block);
6169 make_single_succ_edge (init_block, first_block, flags);
6170 }
6171 else
6172 make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6173 EDGE_FALLTHRU);
6174
6175 update_bb_for_insn (init_block);
6176 return init_block;
6177 }
6178
6179 /* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6180 found in the block tree. */
6181
6182 static void
set_block_levels(tree block,int level)6183 set_block_levels (tree block, int level)
6184 {
6185 while (block)
6186 {
6187 BLOCK_NUMBER (block) = level;
6188 set_block_levels (BLOCK_SUBBLOCKS (block), level + 1);
6189 block = BLOCK_CHAIN (block);
6190 }
6191 }
6192
6193 /* Create a block containing landing pads and similar stuff. */
6194
6195 static void
construct_exit_block(void)6196 construct_exit_block (void)
6197 {
6198 rtx_insn *head = get_last_insn ();
6199 rtx_insn *end;
6200 basic_block exit_block;
6201 edge e, e2;
6202 unsigned ix;
6203 edge_iterator ei;
6204 basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6205 rtx_insn *orig_end = BB_END (prev_bb);
6206
6207 rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6208
6209 /* Make sure the locus is set to the end of the function, so that
6210 epilogue line numbers and warnings are set properly. */
6211 if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6212 input_location = cfun->function_end_locus;
6213
6214 /* Generate rtl for function exit. */
6215 expand_function_end ();
6216
6217 end = get_last_insn ();
6218 if (head == end)
6219 return;
6220 /* While emitting the function end we could move end of the last basic
6221 block. */
6222 BB_END (prev_bb) = orig_end;
6223 while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
6224 head = NEXT_INSN (head);
6225 /* But make sure exit_block starts with RETURN_LABEL, otherwise the
6226 bb count counting will be confused. Any instructions before that
6227 label are emitted for the case where PREV_BB falls through into the
6228 exit block, so append those instructions to prev_bb in that case. */
6229 if (NEXT_INSN (head) != return_label)
6230 {
6231 while (NEXT_INSN (head) != return_label)
6232 {
6233 if (!NOTE_P (NEXT_INSN (head)))
6234 BB_END (prev_bb) = NEXT_INSN (head);
6235 head = NEXT_INSN (head);
6236 }
6237 }
6238 exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
6239 exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6240 add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6241
6242 ix = 0;
6243 while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6244 {
6245 e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6246 if (!(e->flags & EDGE_ABNORMAL))
6247 redirect_edge_succ (e, exit_block);
6248 else
6249 ix++;
6250 }
6251
6252 e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6253 EDGE_FALLTHRU);
6254 FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6255 if (e2 != e)
6256 {
6257 exit_block->count -= e2->count ();
6258 }
6259 update_bb_for_insn (exit_block);
6260 }
6261
6262 /* Helper function for discover_nonconstant_array_refs.
6263 Look for ARRAY_REF nodes with non-constant indexes and mark them
6264 addressable. */
6265
6266 static tree
discover_nonconstant_array_refs_r(tree * tp,int * walk_subtrees,void * data ATTRIBUTE_UNUSED)6267 discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6268 void *data ATTRIBUTE_UNUSED)
6269 {
6270 tree t = *tp;
6271
6272 if (IS_TYPE_OR_DECL_P (t))
6273 *walk_subtrees = 0;
6274 else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6275 {
6276 while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6277 && is_gimple_min_invariant (TREE_OPERAND (t, 1))
6278 && (!TREE_OPERAND (t, 2)
6279 || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6280 || (TREE_CODE (t) == COMPONENT_REF
6281 && (!TREE_OPERAND (t,2)
6282 || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6283 || TREE_CODE (t) == BIT_FIELD_REF
6284 || TREE_CODE (t) == REALPART_EXPR
6285 || TREE_CODE (t) == IMAGPART_EXPR
6286 || TREE_CODE (t) == VIEW_CONVERT_EXPR
6287 || CONVERT_EXPR_P (t))
6288 t = TREE_OPERAND (t, 0);
6289
6290 if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6291 {
6292 t = get_base_address (t);
6293 if (t && DECL_P (t)
6294 && DECL_MODE (t) != BLKmode)
6295 TREE_ADDRESSABLE (t) = 1;
6296 }
6297
6298 *walk_subtrees = 0;
6299 }
6300 /* References of size POLY_INT_CST to a fixed-size object must go
6301 through memory. It's more efficient to force that here than
6302 to create temporary slots on the fly. */
6303 else if ((TREE_CODE (t) == MEM_REF || TREE_CODE (t) == TARGET_MEM_REF)
6304 && TYPE_SIZE (TREE_TYPE (t))
6305 && POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t))))
6306 {
6307 tree base = get_base_address (t);
6308 if (base
6309 && DECL_P (base)
6310 && DECL_MODE (base) != BLKmode
6311 && GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6312 TREE_ADDRESSABLE (base) = 1;
6313 *walk_subtrees = 0;
6314 }
6315
6316 return NULL_TREE;
6317 }
6318
6319 /* If there's a chance to get a pseudo for t then if it would be of float mode
6320 and the actual access is via an integer mode (lowered memcpy or similar
6321 access) then avoid the register expansion if the mode likely is not storage
6322 suitable for raw bits processing (like XFmode on i?86). */
6323
6324 static void
avoid_type_punning_on_regs(tree t)6325 avoid_type_punning_on_regs (tree t)
6326 {
6327 machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6328 if (access_mode != BLKmode
6329 && !SCALAR_INT_MODE_P (access_mode))
6330 return;
6331 tree base = get_base_address (t);
6332 if (DECL_P (base)
6333 && !TREE_ADDRESSABLE (base)
6334 && FLOAT_MODE_P (DECL_MODE (base))
6335 && maybe_lt (GET_MODE_PRECISION (DECL_MODE (base)),
6336 GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6337 /* Double check in the expensive way we really would get a pseudo. */
6338 && use_register_for_decl (base))
6339 TREE_ADDRESSABLE (base) = 1;
6340 }
6341
6342 /* RTL expansion is not able to compile array references with variable
6343 offsets for arrays stored in single register. Discover such
6344 expressions and mark variables as addressable to avoid this
6345 scenario. */
6346
6347 static void
discover_nonconstant_array_refs(void)6348 discover_nonconstant_array_refs (void)
6349 {
6350 basic_block bb;
6351 gimple_stmt_iterator gsi;
6352
6353 FOR_EACH_BB_FN (bb, cfun)
6354 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6355 {
6356 gimple *stmt = gsi_stmt (gsi);
6357 if (!is_gimple_debug (stmt))
6358 {
6359 walk_gimple_op (stmt, discover_nonconstant_array_refs_r, NULL);
6360 gcall *call = dyn_cast <gcall *> (stmt);
6361 if (call && gimple_call_internal_p (call))
6362 switch (gimple_call_internal_fn (call))
6363 {
6364 case IFN_LOAD_LANES:
6365 /* The source must be a MEM. */
6366 mark_addressable (gimple_call_arg (call, 0));
6367 break;
6368 case IFN_STORE_LANES:
6369 /* The destination must be a MEM. */
6370 mark_addressable (gimple_call_lhs (call));
6371 break;
6372 default:
6373 break;
6374 }
6375 if (gimple_vdef (stmt))
6376 {
6377 tree t = gimple_get_lhs (stmt);
6378 if (t && REFERENCE_CLASS_P (t))
6379 avoid_type_punning_on_regs (t);
6380 }
6381 }
6382 }
6383 }
6384
6385 /* This function sets crtl->args.internal_arg_pointer to a virtual
6386 register if DRAP is needed. Local register allocator will replace
6387 virtual_incoming_args_rtx with the virtual register. */
6388
6389 static void
expand_stack_alignment(void)6390 expand_stack_alignment (void)
6391 {
6392 rtx drap_rtx;
6393 unsigned int preferred_stack_boundary;
6394
6395 if (! SUPPORTS_STACK_ALIGNMENT)
6396 return;
6397
6398 if (cfun->calls_alloca
6399 || cfun->has_nonlocal_label
6400 || crtl->has_nonlocal_goto)
6401 crtl->need_drap = true;
6402
6403 /* Call update_stack_boundary here again to update incoming stack
6404 boundary. It may set incoming stack alignment to a different
6405 value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6406 use the minimum incoming stack alignment to check if it is OK
6407 to perform sibcall optimization since sibcall optimization will
6408 only align the outgoing stack to incoming stack boundary. */
6409 if (targetm.calls.update_stack_boundary)
6410 targetm.calls.update_stack_boundary ();
6411
6412 /* The incoming stack frame has to be aligned at least at
6413 parm_stack_boundary. */
6414 gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6415
6416 /* Update crtl->stack_alignment_estimated and use it later to align
6417 stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6418 exceptions since callgraph doesn't collect incoming stack alignment
6419 in this case. */
6420 if (cfun->can_throw_non_call_exceptions
6421 && PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6422 preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6423 else
6424 preferred_stack_boundary = crtl->preferred_stack_boundary;
6425 if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6426 crtl->stack_alignment_estimated = preferred_stack_boundary;
6427 if (preferred_stack_boundary > crtl->stack_alignment_needed)
6428 crtl->stack_alignment_needed = preferred_stack_boundary;
6429
6430 gcc_assert (crtl->stack_alignment_needed
6431 <= crtl->stack_alignment_estimated);
6432
6433 crtl->stack_realign_needed
6434 = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6435 crtl->stack_realign_tried = crtl->stack_realign_needed;
6436
6437 crtl->stack_realign_processed = true;
6438
6439 /* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6440 alignment. */
6441 gcc_assert (targetm.calls.get_drap_rtx != NULL);
6442 drap_rtx = targetm.calls.get_drap_rtx ();
6443
6444 /* stack_realign_drap and drap_rtx must match. */
6445 gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6446
6447 /* Do nothing if NULL is returned, which means DRAP is not needed. */
6448 if (drap_rtx != NULL)
6449 {
6450 crtl->args.internal_arg_pointer = drap_rtx;
6451
6452 /* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6453 needed. */
6454 fixup_tail_calls ();
6455 }
6456 }
6457
6458
6459 static void
expand_main_function(void)6460 expand_main_function (void)
6461 {
6462 #if (defined(INVOKE__main) \
6463 || (!defined(HAS_INIT_SECTION) \
6464 && !defined(INIT_SECTION_ASM_OP) \
6465 && !defined(INIT_ARRAY_SECTION_ASM_OP)))
6466 emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6467 #endif
6468 }
6469
6470
6471 /* Expand code to initialize the stack_protect_guard. This is invoked at
6472 the beginning of a function to be protected. */
6473
6474 static void
stack_protect_prologue(void)6475 stack_protect_prologue (void)
6476 {
6477 tree guard_decl = targetm.stack_protect_guard ();
6478 rtx x, y;
6479
6480 crtl->stack_protect_guard_decl = guard_decl;
6481 x = expand_normal (crtl->stack_protect_guard);
6482
6483 if (targetm.have_stack_protect_combined_set () && guard_decl)
6484 {
6485 gcc_assert (DECL_P (guard_decl));
6486 y = DECL_RTL (guard_decl);
6487
6488 /* Allow the target to compute address of Y and copy it to X without
6489 leaking Y into a register. This combined address + copy pattern
6490 allows the target to prevent spilling of any intermediate results by
6491 splitting it after register allocator. */
6492 if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6493 {
6494 emit_insn (insn);
6495 return;
6496 }
6497 }
6498
6499 if (guard_decl)
6500 y = expand_normal (guard_decl);
6501 else
6502 y = const0_rtx;
6503
6504 /* Allow the target to copy from Y to X without leaking Y into a
6505 register. */
6506 if (targetm.have_stack_protect_set ())
6507 if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
6508 {
6509 emit_insn (insn);
6510 return;
6511 }
6512
6513 /* Otherwise do a straight move. */
6514 emit_move_insn (x, y);
6515 }
6516
6517 /* Translate the intermediate representation contained in the CFG
6518 from GIMPLE trees to RTL.
6519
6520 We do conversion per basic block and preserve/update the tree CFG.
6521 This implies we have to do some magic as the CFG can simultaneously
6522 consist of basic blocks containing RTL and GIMPLE trees. This can
6523 confuse the CFG hooks, so be careful to not manipulate CFG during
6524 the expansion. */
6525
6526 namespace {
6527
6528 const pass_data pass_data_expand =
6529 {
6530 RTL_PASS, /* type */
6531 "expand", /* name */
6532 OPTGROUP_NONE, /* optinfo_flags */
6533 TV_EXPAND, /* tv_id */
6534 ( PROP_ssa | PROP_gimple_leh | PROP_cfg
6535 | PROP_gimple_lcx
6536 | PROP_gimple_lvec
6537 | PROP_gimple_lva), /* properties_required */
6538 PROP_rtl, /* properties_provided */
6539 ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
6540 0, /* todo_flags_start */
6541 0, /* todo_flags_finish */
6542 };
6543
6544 class pass_expand : public rtl_opt_pass
6545 {
6546 public:
pass_expand(gcc::context * ctxt)6547 pass_expand (gcc::context *ctxt)
6548 : rtl_opt_pass (pass_data_expand, ctxt)
6549 {}
6550
6551 /* opt_pass methods: */
6552 virtual unsigned int execute (function *);
6553
6554 }; // class pass_expand
6555
6556 unsigned int
execute(function * fun)6557 pass_expand::execute (function *fun)
6558 {
6559 basic_block bb, init_block;
6560 edge_iterator ei;
6561 edge e;
6562 rtx_insn *var_seq, *var_ret_seq;
6563 unsigned i;
6564
6565 timevar_push (TV_OUT_OF_SSA);
6566 rewrite_out_of_ssa (&SA);
6567 timevar_pop (TV_OUT_OF_SSA);
6568 SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
6569
6570 if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
6571 {
6572 gimple_stmt_iterator gsi;
6573 FOR_EACH_BB_FN (bb, cfun)
6574 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6575 if (gimple_debug_bind_p (gsi_stmt (gsi)))
6576 avoid_deep_ter_for_debug (gsi_stmt (gsi), 0);
6577 }
6578
6579 /* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
6580 discover_nonconstant_array_refs ();
6581
6582 /* Make sure all values used by the optimization passes have sane
6583 defaults. */
6584 reg_renumber = 0;
6585
6586 /* Some backends want to know that we are expanding to RTL. */
6587 currently_expanding_to_rtl = 1;
6588 /* Dominators are not kept up-to-date as we may create new basic-blocks. */
6589 free_dominance_info (CDI_DOMINATORS);
6590
6591 rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
6592
6593 insn_locations_init ();
6594 if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
6595 {
6596 /* Eventually, all FEs should explicitly set function_start_locus. */
6597 if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
6598 set_curr_insn_location
6599 (DECL_SOURCE_LOCATION (current_function_decl));
6600 else
6601 set_curr_insn_location (fun->function_start_locus);
6602 }
6603 else
6604 set_curr_insn_location (UNKNOWN_LOCATION);
6605 prologue_location = curr_insn_location ();
6606
6607 #ifdef INSN_SCHEDULING
6608 init_sched_attrs ();
6609 #endif
6610
6611 /* Make sure first insn is a note even if we don't want linenums.
6612 This makes sure the first insn will never be deleted.
6613 Also, final expects a note to appear there. */
6614 emit_note (NOTE_INSN_DELETED);
6615
6616 targetm.expand_to_rtl_hook ();
6617 crtl->init_stack_alignment ();
6618 fun->cfg->max_jumptable_ents = 0;
6619
6620 /* Resovle the function section. Some targets, like ARM EABI rely on knowledge
6621 of the function section at exapnsion time to predict distance of calls. */
6622 resolve_unique_section (current_function_decl, 0, flag_function_sections);
6623
6624 /* Expand the variables recorded during gimple lowering. */
6625 timevar_push (TV_VAR_EXPAND);
6626 start_sequence ();
6627
6628 var_ret_seq = expand_used_vars ();
6629
6630 var_seq = get_insns ();
6631 end_sequence ();
6632 timevar_pop (TV_VAR_EXPAND);
6633
6634 /* Honor stack protection warnings. */
6635 if (warn_stack_protect)
6636 {
6637 if (fun->calls_alloca)
6638 warning (OPT_Wstack_protector,
6639 "stack protector not protecting local variables: "
6640 "variable length buffer");
6641 if (has_short_buffer && !crtl->stack_protect_guard)
6642 warning (OPT_Wstack_protector,
6643 "stack protector not protecting function: "
6644 "all local arrays are less than %d bytes long",
6645 (int) param_ssp_buffer_size);
6646 }
6647
6648 /* Set up parameters and prepare for return, for the function. */
6649 expand_function_start (current_function_decl);
6650
6651 /* If we emitted any instructions for setting up the variables,
6652 emit them before the FUNCTION_START note. */
6653 if (var_seq)
6654 {
6655 emit_insn_before (var_seq, parm_birth_insn);
6656
6657 /* In expand_function_end we'll insert the alloca save/restore
6658 before parm_birth_insn. We've just insertted an alloca call.
6659 Adjust the pointer to match. */
6660 parm_birth_insn = var_seq;
6661 }
6662
6663 /* Now propagate the RTL assignment of each partition to the
6664 underlying var of each SSA_NAME. */
6665 tree name;
6666
6667 FOR_EACH_SSA_NAME (i, name, cfun)
6668 {
6669 /* We might have generated new SSA names in
6670 update_alias_info_with_stack_vars. They will have a NULL
6671 defining statements, and won't be part of the partitioning,
6672 so ignore those. */
6673 if (!SSA_NAME_DEF_STMT (name))
6674 continue;
6675
6676 adjust_one_expanded_partition_var (name);
6677 }
6678
6679 /* Clean up RTL of variables that straddle across multiple
6680 partitions, and check that the rtl of any PARM_DECLs that are not
6681 cleaned up is that of their default defs. */
6682 FOR_EACH_SSA_NAME (i, name, cfun)
6683 {
6684 int part;
6685
6686 /* We might have generated new SSA names in
6687 update_alias_info_with_stack_vars. They will have a NULL
6688 defining statements, and won't be part of the partitioning,
6689 so ignore those. */
6690 if (!SSA_NAME_DEF_STMT (name))
6691 continue;
6692 part = var_to_partition (SA.map, name);
6693 if (part == NO_PARTITION)
6694 continue;
6695
6696 /* If this decl was marked as living in multiple places, reset
6697 this now to NULL. */
6698 tree var = SSA_NAME_VAR (name);
6699 if (var && DECL_RTL_IF_SET (var) == pc_rtx)
6700 SET_DECL_RTL (var, NULL);
6701 /* Check that the pseudos chosen by assign_parms are those of
6702 the corresponding default defs. */
6703 else if (SSA_NAME_IS_DEFAULT_DEF (name)
6704 && (TREE_CODE (var) == PARM_DECL
6705 || TREE_CODE (var) == RESULT_DECL))
6706 {
6707 rtx in = DECL_RTL_IF_SET (var);
6708 gcc_assert (in);
6709 rtx out = SA.partition_to_pseudo[part];
6710 gcc_assert (in == out);
6711
6712 /* Now reset VAR's RTL to IN, so that the _EXPR attrs match
6713 those expected by debug backends for each parm and for
6714 the result. This is particularly important for stabs,
6715 whose register elimination from parm's DECL_RTL may cause
6716 -fcompare-debug differences as SET_DECL_RTL changes reg's
6717 attrs. So, make sure the RTL already has the parm as the
6718 EXPR, so that it won't change. */
6719 SET_DECL_RTL (var, NULL_RTX);
6720 if (MEM_P (in))
6721 set_mem_attributes (in, var, true);
6722 SET_DECL_RTL (var, in);
6723 }
6724 }
6725
6726 /* If this function is `main', emit a call to `__main'
6727 to run global initializers, etc. */
6728 if (DECL_NAME (current_function_decl)
6729 && MAIN_NAME_P (DECL_NAME (current_function_decl))
6730 && DECL_FILE_SCOPE_P (current_function_decl))
6731 expand_main_function ();
6732
6733 /* Initialize the stack_protect_guard field. This must happen after the
6734 call to __main (if any) so that the external decl is initialized. */
6735 if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
6736 stack_protect_prologue ();
6737
6738 expand_phi_nodes (&SA);
6739
6740 /* Release any stale SSA redirection data. */
6741 redirect_edge_var_map_empty ();
6742
6743 /* Register rtl specific functions for cfg. */
6744 rtl_register_cfg_hooks ();
6745
6746 init_block = construct_init_block ();
6747
6748 /* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
6749 remaining edges later. */
6750 FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
6751 e->flags &= ~EDGE_EXECUTABLE;
6752
6753 /* If the function has too many markers, drop them while expanding. */
6754 if (cfun->debug_marker_count
6755 >= param_max_debug_marker_count)
6756 cfun->debug_nonbind_markers = false;
6757
6758 lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
6759 FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
6760 next_bb)
6761 bb = expand_gimple_basic_block (bb, var_ret_seq != NULL_RTX);
6762
6763 if (MAY_HAVE_DEBUG_BIND_INSNS)
6764 expand_debug_locations ();
6765
6766 if (deep_ter_debug_map)
6767 {
6768 delete deep_ter_debug_map;
6769 deep_ter_debug_map = NULL;
6770 }
6771
6772 /* Free stuff we no longer need after GIMPLE optimizations. */
6773 free_dominance_info (CDI_DOMINATORS);
6774 free_dominance_info (CDI_POST_DOMINATORS);
6775 delete_tree_cfg_annotations (fun);
6776
6777 timevar_push (TV_OUT_OF_SSA);
6778 finish_out_of_ssa (&SA);
6779 timevar_pop (TV_OUT_OF_SSA);
6780
6781 timevar_push (TV_POST_EXPAND);
6782 /* We are no longer in SSA form. */
6783 fun->gimple_df->in_ssa_p = false;
6784 loops_state_clear (LOOP_CLOSED_SSA);
6785
6786 /* Expansion is used by optimization passes too, set maybe_hot_insn_p
6787 conservatively to true until they are all profile aware. */
6788 delete lab_rtx_for_bb;
6789 free_histograms (fun);
6790
6791 construct_exit_block ();
6792 insn_locations_finalize ();
6793
6794 if (var_ret_seq)
6795 {
6796 rtx_insn *after = return_label;
6797 rtx_insn *next = NEXT_INSN (after);
6798 if (next && NOTE_INSN_BASIC_BLOCK_P (next))
6799 after = next;
6800 emit_insn_after (var_ret_seq, after);
6801 }
6802
6803 if (hwasan_sanitize_stack_p ())
6804 hwasan_maybe_emit_frame_base_init ();
6805
6806 /* Zap the tree EH table. */
6807 set_eh_throw_stmt_table (fun, NULL);
6808
6809 /* We need JUMP_LABEL be set in order to redirect jumps, and hence
6810 split edges which edge insertions might do. */
6811 rebuild_jump_labels (get_insns ());
6812
6813 /* If we have a single successor to the entry block, put the pending insns
6814 after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
6815 if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
6816 {
6817 edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
6818 if (e->insns.r)
6819 {
6820 rtx_insn *insns = e->insns.r;
6821 e->insns.r = NULL;
6822 rebuild_jump_labels_chain (insns);
6823 if (NOTE_P (parm_birth_insn)
6824 && NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
6825 emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
6826 else
6827 emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
6828 }
6829 }
6830
6831 /* Otherwise, as well as for other edges, take the usual way. */
6832 commit_edge_insertions ();
6833
6834 /* We're done expanding trees to RTL. */
6835 currently_expanding_to_rtl = 0;
6836
6837 flush_mark_addressable_queue ();
6838
6839 FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
6840 EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
6841 {
6842 edge e;
6843 edge_iterator ei;
6844 for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
6845 {
6846 /* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
6847 e->flags &= ~EDGE_EXECUTABLE;
6848
6849 /* At the moment not all abnormal edges match the RTL
6850 representation. It is safe to remove them here as
6851 find_many_sub_basic_blocks will rediscover them.
6852 In the future we should get this fixed properly. */
6853 if ((e->flags & EDGE_ABNORMAL)
6854 && !(e->flags & EDGE_SIBCALL))
6855 remove_edge (e);
6856 else
6857 ei_next (&ei);
6858 }
6859 }
6860
6861 auto_sbitmap blocks (last_basic_block_for_fn (fun));
6862 bitmap_ones (blocks);
6863 find_many_sub_basic_blocks (blocks);
6864 purge_all_dead_edges ();
6865
6866 /* After initial rtl generation, call back to finish generating
6867 exception support code. We need to do this before cleaning up
6868 the CFG as the code does not expect dead landing pads. */
6869 if (fun->eh->region_tree != NULL)
6870 finish_eh_generation ();
6871
6872 /* Call expand_stack_alignment after finishing all
6873 updates to crtl->preferred_stack_boundary. */
6874 expand_stack_alignment ();
6875
6876 /* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
6877 function. */
6878 if (crtl->tail_call_emit)
6879 fixup_tail_calls ();
6880
6881 HOST_WIDE_INT patch_area_size, patch_area_entry;
6882 parse_and_check_patch_area (flag_patchable_function_entry, false,
6883 &patch_area_size, &patch_area_entry);
6884
6885 tree patchable_function_entry_attr
6886 = lookup_attribute ("patchable_function_entry",
6887 DECL_ATTRIBUTES (cfun->decl));
6888 if (patchable_function_entry_attr)
6889 {
6890 tree pp_val = TREE_VALUE (patchable_function_entry_attr);
6891 tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
6892
6893 patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
6894 patch_area_entry = 0;
6895 if (TREE_CHAIN (pp_val) != NULL_TREE)
6896 {
6897 tree patchable_function_entry_value2
6898 = TREE_VALUE (TREE_CHAIN (pp_val));
6899 patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
6900 }
6901 }
6902
6903 if (patch_area_entry > patch_area_size)
6904 {
6905 if (patch_area_size > 0)
6906 warning (OPT_Wattributes,
6907 "patchable function entry %wu exceeds size %wu",
6908 patch_area_entry, patch_area_size);
6909 patch_area_entry = 0;
6910 }
6911
6912 crtl->patch_area_size = patch_area_size;
6913 crtl->patch_area_entry = patch_area_entry;
6914
6915 /* BB subdivision may have created basic blocks that are only reachable
6916 from unlikely bbs but not marked as such in the profile. */
6917 if (optimize)
6918 propagate_unlikely_bbs_forward ();
6919
6920 /* Remove unreachable blocks, otherwise we cannot compute dominators
6921 which are needed for loop state verification. As a side-effect
6922 this also compacts blocks.
6923 ??? We cannot remove trivially dead insns here as for example
6924 the DRAP reg on i?86 is not magically live at this point.
6925 gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
6926 cleanup_cfg (CLEANUP_NO_INSN_DEL);
6927
6928 checking_verify_flow_info ();
6929
6930 /* Initialize pseudos allocated for hard registers. */
6931 emit_initial_value_sets ();
6932
6933 /* And finally unshare all RTL. */
6934 unshare_all_rtl ();
6935
6936 /* There's no need to defer outputting this function any more; we
6937 know we want to output it. */
6938 DECL_DEFER_OUTPUT (current_function_decl) = 0;
6939
6940 /* Now that we're done expanding trees to RTL, we shouldn't have any
6941 more CONCATs anywhere. */
6942 generating_concat_p = 0;
6943
6944 if (dump_file)
6945 {
6946 fprintf (dump_file,
6947 "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
6948 /* And the pass manager will dump RTL for us. */
6949 }
6950
6951 /* If we're emitting a nested function, make sure its parent gets
6952 emitted as well. Doing otherwise confuses debug info. */
6953 {
6954 tree parent;
6955 for (parent = DECL_CONTEXT (current_function_decl);
6956 parent != NULL_TREE;
6957 parent = get_containing_scope (parent))
6958 if (TREE_CODE (parent) == FUNCTION_DECL)
6959 TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
6960 }
6961
6962 TREE_ASM_WRITTEN (current_function_decl) = 1;
6963
6964 /* After expanding, the return labels are no longer needed. */
6965 return_label = NULL;
6966 naked_return_label = NULL;
6967
6968 /* After expanding, the tm_restart map is no longer needed. */
6969 if (fun->gimple_df->tm_restart)
6970 fun->gimple_df->tm_restart = NULL;
6971
6972 /* Tag the blocks with a depth number so that change_scope can find
6973 the common parent easily. */
6974 set_block_levels (DECL_INITIAL (fun->decl), 0);
6975 default_rtl_profile ();
6976
6977 /* For -dx discard loops now, otherwise IL verify in clean_state will
6978 ICE. */
6979 if (rtl_dump_and_exit)
6980 {
6981 cfun->curr_properties &= ~PROP_loops;
6982 loop_optimizer_finalize ();
6983 }
6984
6985 timevar_pop (TV_POST_EXPAND);
6986
6987 return 0;
6988 }
6989
6990 } // anon namespace
6991
6992 rtl_opt_pass *
make_pass_expand(gcc::context * ctxt)6993 make_pass_expand (gcc::context *ctxt)
6994 {
6995 return new pass_expand (ctxt);
6996 }
6997