1 /* Routines for performing Temporary Expression Replacement (TER) in SSA trees.
2    Copyright (C) 2003-2016 Free Software Foundation, Inc.
3    Contributed by Andrew MacLeod  <amacleod@redhat.com>
4 
5 This file is part of GCC.
6 
7 GCC is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11 
12 GCC is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 GNU General Public License for more details.
16 
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3.  If not see
19 <http://www.gnu.org/licenses/>.  */
20 
21 
22 #include "config.h"
23 #include "system.h"
24 #include "coretypes.h"
25 #include "backend.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "ssa.h"
29 #include "gimple-pretty-print.h"
30 #include "gimple-iterator.h"
31 #include "dumpfile.h"
32 #include "tree-ssa-live.h"
33 #include "tree-ssa-ter.h"
34 #include "tree-outof-ssa.h"
35 #include "gimple-walk.h"
36 
37 
38 /* Temporary Expression Replacement (TER)
39 
40    Replace SSA version variables during out-of-ssa with their defining
41    expression if there is only one use of the variable.
42 
43    This pass is required in order for the RTL expansion pass to see larger
44    chunks of code.  This allows it to make better choices on RTL pattern
45    selection.  When expand is rewritten and merged with out-of-ssa, and
46    understands SSA, this should be eliminated.
47 
48    A pass is made through the function, one block at a time.  No cross block
49    information is tracked.
50 
51    Variables which only have one use, and whose defining stmt is considered
52    a replaceable expression (see ssa_is_replaceable_p) are tracked to see whether
53    they can be replaced at their use location.
54 
55    n_12 = C * 10
56    a_2 = b_5 + 6
57    v_9 = a_2 * n_12
58 
59    if there are the only use of n_12 and a_2, TER will substitute in their
60    expressions in v_9, and end up with:
61 
62    v_9 = (b_5 + 6) * (C * 10)
63 
64    which will then have the ssa_name assigned to regular variables, and the
65    resulting code which will be passed to the expander looks something like:
66 
67    v = (b + 6) * (C * 10)
68 
69 
70    This requires ensuring that none of the variables used by the expression
71    change between the def point and where it is used.  Furthermore, if any
72    of the ssa_names used in this expression are themselves replaceable, we
73    have to ensure none of that expressions' arguments change either.
74    Although SSA_NAMES themselves don't change, this pass is performed after
75    coalescing has coalesced different SSA_NAMES together, so there could be a
76    definition of an SSA_NAME which is coalesced with a use that causes a
77    problem, i.e.,
78 
79    PHI b_5 = <b_8(2), b_14(1)>
80    <...>
81    a_2 = b_5 + 6
82    b_8 = c_4 + 4
83    v_9 = a_2 * n_12
84    <...>
85 
86    If b_5, b_8 and b_14 are all coalesced together...
87    The expression b_5 + 6 CANNOT replace the use in the statement defining v_9
88    because b_8 is in fact killing the value of b_5 since they share a partition
89    and will be assigned the same memory or register location.
90 
91    TER implements this but stepping through the instructions in a block and
92    tracking potential expressions for replacement, and the partitions they are
93    dependent on.  Expressions are represented by the SSA_NAME_VERSION of the
94    DEF on the LHS of a GIMPLE_ASSIGN and the expression is the RHS.
95 
96    When a stmt is determined to be a possible replacement expression, the
97    following steps are taken:
98 
99    EXPR_DECL_UID bitmap is allocated and set to the base variable UID of the
100    def and any uses in the expression.  non-NULL means the expression is being
101    tracked.  The UID's themselves are used to prevent TER substitution into
102    accumulating sequences, i.e.,
103 
104    x = x + y
105    x = x + z
106    x = x + w
107    etc.
108    this can result in very large expressions which don't accomplish anything
109    see PR tree-optimization/17549.
110 
111    PARTITION_DEPENDENCIES is another bitmap array, and it has a bit set for any
112    partition which is used in the expression.  This is primarily used to remove
113    an expression from the partition kill lists when a decision is made whether
114    to replace it or not.  This is indexed by ssa version number as well, and
115    indicates a partition number.  virtual operands are not tracked individually,
116    but they are summarized by an artificial partition called VIRTUAL_PARTITION.
117    This means a MAY or MUST def will kill *ALL* expressions that are dependent
118    on a virtual operand.
119    Note that the EXPR_DECL_UID and this bitmap represent very similar
120    information, but the info in one is not easy to obtain from the other.
121 
122    KILL_LIST is yet another bitmap array, this time it is indexed by partition
123    number, and represents a list of active expressions which will no
124    longer be valid if a definition into this partition takes place.
125 
126    PARTITION_IN_USE is simply a bitmap which is used to track which partitions
127    currently have something in their kill list.  This is used at the end of
128    a block to clear out the KILL_LIST bitmaps at the end of each block.
129 
130    NEW_REPLACEABLE_DEPENDENCIES is used as a temporary place to store
131    dependencies which will be reused by the current definition. All the uses
132    on an expression are processed before anything else is done. If a use is
133    determined to be a replaceable expression AND the current stmt is also going
134    to be replaceable, all the dependencies of this replaceable use will be
135    picked up by the current stmt's expression. Rather than recreate them, they
136    are simply copied here and then copied into the new expression when it is
137    processed.
138 
139    a_2 = b_5 + 6
140    v_8 = a_2 + c_4
141 
142    a_2's expression 'b_5 + 6' is determined to be replaceable at the use
143    location. It is dependent on the partition 'b_5' is in. This is cached into
144    the NEW_REPLACEABLE_DEPENDENCIES bitmap, and when v_8 is examined for
145    replaceability, it is a candidate, and it is dependent on the partition
146    b_5 is in *NOT* a_2, as well as c_4's partition.
147 
148    if v_8 is also replaceable:
149 
150    x_9 = v_8 * 5
151 
152    x_9 is dependent on partitions b_5, and c_4
153 
154    if a statement is found which has either of those partitions written to
155    before x_9 is used, then x_9 itself is NOT replaceable.  */
156 
157 
158 /* Temporary Expression Replacement (TER) table information.  */
159 
160 struct temp_expr_table
161 {
162   var_map map;
163   bitmap *partition_dependencies;	/* Partitions expr is dependent on.  */
164   bitmap replaceable_expressions;	/* Replacement expression table.  */
165   bitmap *expr_decl_uids;		/* Base uids of exprs.  */
166   bitmap *kill_list;			/* Expr's killed by a partition.  */
167   int virtual_partition;		/* Pseudo partition for virtual ops.  */
168   bitmap partition_in_use;		/* Partitions with kill entries.  */
169   bitmap new_replaceable_dependencies;	/* Holding place for pending dep's.  */
170   int *num_in_part;			/* # of ssa_names in a partition.  */
171   int *call_cnt;			/* Call count at definition.  */
172   int *reg_vars_cnt;			/* Number of register variable
173 					   definitions encountered.  */
174 };
175 
176 /* Used to indicate a dependency on VDEFs.  */
177 #define VIRTUAL_PARTITION(table)	(table->virtual_partition)
178 
179 /* A place for the many, many bitmaps we create.  */
180 static bitmap_obstack ter_bitmap_obstack;
181 
182 extern void debug_ter (FILE *, temp_expr_table *);
183 
184 
185 /* Create a new TER table for MAP.  */
186 
187 static temp_expr_table *
new_temp_expr_table(var_map map)188 new_temp_expr_table (var_map map)
189 {
190   unsigned x;
191 
192   temp_expr_table *t = XNEW (struct temp_expr_table);
193   t->map = map;
194 
195   t->partition_dependencies = XCNEWVEC (bitmap, num_ssa_names + 1);
196   t->expr_decl_uids = XCNEWVEC (bitmap, num_ssa_names + 1);
197   t->kill_list = XCNEWVEC (bitmap, num_var_partitions (map) + 1);
198 
199   t->partition_in_use = BITMAP_ALLOC (&ter_bitmap_obstack);
200 
201   t->virtual_partition = num_var_partitions (map);
202   t->new_replaceable_dependencies = BITMAP_ALLOC (&ter_bitmap_obstack);
203 
204   t->replaceable_expressions = NULL;
205   t->num_in_part = XCNEWVEC (int, num_var_partitions (map));
206   for (x = 1; x < num_ssa_names; x++)
207     {
208       int p;
209       tree name = ssa_name (x);
210       if (!name)
211         continue;
212       p = var_to_partition (map, name);
213       if (p != NO_PARTITION)
214         t->num_in_part[p]++;
215     }
216   t->call_cnt = XCNEWVEC (int, num_ssa_names + 1);
217   t->reg_vars_cnt = XCNEWVEC (int, num_ssa_names + 1);
218 
219   return t;
220 }
221 
222 
223 /* Free TER table T.  If there are valid replacements, return the expression
224    vector.  */
225 
226 static bitmap
free_temp_expr_table(temp_expr_table * t)227 free_temp_expr_table (temp_expr_table *t)
228 {
229   bitmap ret = NULL;
230 
231   if (flag_checking)
232     {
233       for (unsigned x = 0; x <= num_var_partitions (t->map); x++)
234 	gcc_assert (!t->kill_list[x]);
235       for (unsigned x = 0; x < num_ssa_names; x++)
236 	{
237 	  gcc_assert (t->expr_decl_uids[x] == NULL);
238 	  gcc_assert (t->partition_dependencies[x] == NULL);
239 	}
240     }
241 
242   BITMAP_FREE (t->partition_in_use);
243   BITMAP_FREE (t->new_replaceable_dependencies);
244 
245   free (t->expr_decl_uids);
246   free (t->kill_list);
247   free (t->partition_dependencies);
248   free (t->num_in_part);
249   free (t->call_cnt);
250   free (t->reg_vars_cnt);
251 
252   if (t->replaceable_expressions)
253     ret = t->replaceable_expressions;
254 
255   free (t);
256   return ret;
257 }
258 
259 
260 /* Return TRUE if VERSION is to be replaced by an expression in TAB.  */
261 
262 static inline bool
version_to_be_replaced_p(temp_expr_table * tab,int version)263 version_to_be_replaced_p (temp_expr_table *tab, int version)
264 {
265   if (!tab->replaceable_expressions)
266     return false;
267   return bitmap_bit_p (tab->replaceable_expressions, version);
268 }
269 
270 
271 /* Add partition P to the list if partitions VERSION is dependent on.  TAB is
272    the expression table */
273 
274 static inline void
make_dependent_on_partition(temp_expr_table * tab,int version,int p)275 make_dependent_on_partition (temp_expr_table *tab, int version, int p)
276 {
277   if (!tab->partition_dependencies[version])
278     tab->partition_dependencies[version] = BITMAP_ALLOC (&ter_bitmap_obstack);
279 
280   bitmap_set_bit (tab->partition_dependencies[version], p);
281 }
282 
283 
284 /* Add VER to the kill list for P.  TAB is the expression table */
285 
286 static inline void
add_to_partition_kill_list(temp_expr_table * tab,int p,int ver)287 add_to_partition_kill_list (temp_expr_table *tab, int p, int ver)
288 {
289   if (!tab->kill_list[p])
290     {
291       tab->kill_list[p] = BITMAP_ALLOC (&ter_bitmap_obstack);
292       bitmap_set_bit (tab->partition_in_use, p);
293     }
294   bitmap_set_bit (tab->kill_list[p], ver);
295 }
296 
297 
298 /* Remove VER from the partition kill list for P.  TAB is the expression
299    table.  */
300 
301 static inline void
remove_from_partition_kill_list(temp_expr_table * tab,int p,int version)302 remove_from_partition_kill_list (temp_expr_table *tab, int p, int version)
303 {
304   gcc_checking_assert (tab->kill_list[p]);
305   bitmap_clear_bit (tab->kill_list[p], version);
306   if (bitmap_empty_p (tab->kill_list[p]))
307     {
308       bitmap_clear_bit (tab->partition_in_use, p);
309       BITMAP_FREE (tab->kill_list[p]);
310     }
311 }
312 
313 
314 /* Add a dependency between the def of ssa VERSION and VAR.  If VAR is
315    replaceable by an expression, add a dependence each of the elements of the
316    expression.  These are contained in the new_replaceable list.  TAB is the
317    expression table.  */
318 
319 static void
add_dependence(temp_expr_table * tab,int version,tree var)320 add_dependence (temp_expr_table *tab, int version, tree var)
321 {
322   int i;
323   bitmap_iterator bi;
324   unsigned x;
325 
326   i = SSA_NAME_VERSION (var);
327   if (version_to_be_replaced_p (tab, i))
328     {
329       if (!bitmap_empty_p (tab->new_replaceable_dependencies))
330         {
331 	  /* Version will now be killed by a write to any partition the
332 	     substituted expression would have been killed by.  */
333 	  EXECUTE_IF_SET_IN_BITMAP (tab->new_replaceable_dependencies, 0, x, bi)
334 	    add_to_partition_kill_list (tab, x, version);
335 
336 	  /* Rather than set partition_dependencies and in_use lists bit by
337 	     bit, simply OR in the new_replaceable_dependencies bits.  */
338 	  if (!tab->partition_dependencies[version])
339 	    tab->partition_dependencies[version] =
340 	      BITMAP_ALLOC (&ter_bitmap_obstack);
341 	  bitmap_ior_into (tab->partition_dependencies[version],
342 			   tab->new_replaceable_dependencies);
343 	  bitmap_ior_into (tab->partition_in_use,
344 			   tab->new_replaceable_dependencies);
345 	  /* It is only necessary to add these once.  */
346 	  bitmap_clear (tab->new_replaceable_dependencies);
347 	}
348     }
349   else
350     {
351       i = var_to_partition (tab->map, var);
352       gcc_checking_assert (i != NO_PARTITION);
353       gcc_checking_assert (tab->num_in_part[i] != 0);
354       /* Only dependencies on ssa_names which are coalesced with something need
355          to be tracked.  Partitions with containing only a single SSA_NAME
356 	 *cannot* have their value changed.  */
357       if (tab->num_in_part[i] > 1)
358         {
359 	  add_to_partition_kill_list (tab, i, version);
360 	  make_dependent_on_partition (tab, version, i);
361 	}
362     }
363 }
364 
365 
366 /* This function will remove the expression for VERSION from replacement
367    consideration in table TAB.  If FREE_EXPR is true, then remove the
368    expression from consideration as well by freeing the decl uid bitmap.  */
369 
370 static void
finished_with_expr(temp_expr_table * tab,int version,bool free_expr)371 finished_with_expr (temp_expr_table *tab, int version, bool free_expr)
372 {
373   unsigned i;
374   bitmap_iterator bi;
375 
376   /* Remove this expression from its dependent lists.  The partition dependence
377      list is retained and transferred later to whomever uses this version.  */
378   if (tab->partition_dependencies[version])
379     {
380       EXECUTE_IF_SET_IN_BITMAP (tab->partition_dependencies[version], 0, i, bi)
381 	remove_from_partition_kill_list (tab, i, version);
382       BITMAP_FREE (tab->partition_dependencies[version]);
383     }
384   if (free_expr)
385     BITMAP_FREE (tab->expr_decl_uids[version]);
386 }
387 
388 
389 /* Return TRUE if expression STMT is suitable for replacement.
390    In addition to ssa_is_replaceable_p, require the same bb, and for -O0
391    same locus and same BLOCK), Considers memory loads as replaceable if aliasing
392    is available.  */
393 
394 static inline bool
ter_is_replaceable_p(gimple * stmt)395 ter_is_replaceable_p (gimple *stmt)
396 {
397 
398   if (ssa_is_replaceable_p (stmt))
399     {
400       use_operand_p use_p;
401       tree def;
402       gimple *use_stmt;
403       location_t locus1, locus2;
404       tree block1, block2;
405 
406       /* Only consider definitions which have a single use.  ssa_is_replaceable_p
407 	 already performed this check, but the use stmt pointer is required for
408 	 further checks.  */
409       def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF);
410       if (!single_imm_use (def, &use_p, &use_stmt))
411 	  return false;
412 
413       /* If the use isn't in this block, it wont be replaced either.  */
414       if (gimple_bb (use_stmt) != gimple_bb (stmt))
415         return false;
416 
417       locus1 = gimple_location (stmt);
418       block1 = LOCATION_BLOCK (locus1);
419       locus1 = LOCATION_LOCUS (locus1);
420 
421       if (gphi *phi = dyn_cast <gphi *> (use_stmt))
422 	locus2 = gimple_phi_arg_location (phi,
423 					  PHI_ARG_INDEX_FROM_USE (use_p));
424       else
425 	locus2 = gimple_location (use_stmt);
426       block2 = LOCATION_BLOCK (locus2);
427       locus2 = LOCATION_LOCUS (locus2);
428 
429       if ((!optimize || optimize_debug)
430 	  && ((locus1 != UNKNOWN_LOCATION && locus1 != locus2)
431 	      || (block1 != NULL_TREE && block1 != block2)))
432 	return false;
433 
434       return true;
435     }
436   return false;
437 }
438 
439 
440 /* Create an expression entry for a replaceable expression.  */
441 
442 static void
process_replaceable(temp_expr_table * tab,gimple * stmt,int call_cnt,int reg_vars_cnt)443 process_replaceable (temp_expr_table *tab, gimple *stmt, int call_cnt,
444 		     int reg_vars_cnt)
445 {
446   tree var, def, basevar;
447   int version;
448   ssa_op_iter iter;
449   bitmap def_vars, use_vars;
450 
451   gcc_checking_assert (ter_is_replaceable_p (stmt));
452 
453   def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF);
454   version = SSA_NAME_VERSION (def);
455   def_vars = BITMAP_ALLOC (&ter_bitmap_obstack);
456 
457   basevar = SSA_NAME_VAR (def);
458   if (basevar)
459     bitmap_set_bit (def_vars, DECL_UID (basevar));
460 
461   /* Add this expression to the dependency list for each use partition.  */
462   FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
463     {
464       int var_version = SSA_NAME_VERSION (var);
465 
466       use_vars = tab->expr_decl_uids[var_version];
467       add_dependence (tab, version, var);
468       if (use_vars)
469         {
470 	  bitmap_ior_into (def_vars, use_vars);
471 	  BITMAP_FREE (tab->expr_decl_uids[var_version]);
472 	}
473       else if (SSA_NAME_VAR (var))
474 	bitmap_set_bit (def_vars, DECL_UID (SSA_NAME_VAR (var)));
475     }
476   tab->expr_decl_uids[version] = def_vars;
477 
478   /* If there are VUSES, add a dependence on virtual defs.  */
479   if (gimple_vuse (stmt))
480     {
481       make_dependent_on_partition (tab, version, VIRTUAL_PARTITION (tab));
482       add_to_partition_kill_list (tab, VIRTUAL_PARTITION (tab), version);
483     }
484 
485   tab->call_cnt[version] = call_cnt;
486   tab->reg_vars_cnt[version] = reg_vars_cnt;
487 }
488 
489 
490 /* This function removes any expression in TAB which is dependent on PARTITION
491    from consideration, making it not replaceable.  */
492 
493 static inline void
kill_expr(temp_expr_table * tab,int partition)494 kill_expr (temp_expr_table *tab, int partition)
495 {
496   unsigned version;
497 
498   /* Mark every active expr dependent on this var as not replaceable.
499      finished_with_expr can modify the bitmap, so we can't execute over it.  */
500   while (tab->kill_list[partition])
501     {
502       version = bitmap_first_set_bit (tab->kill_list[partition]);
503       finished_with_expr (tab, version, true);
504     }
505 
506   gcc_checking_assert (!tab->kill_list[partition]);
507 }
508 
509 
510 /* This function kills all expressions in TAB which are dependent on virtual
511    partitions.  */
512 
513 static inline void
kill_virtual_exprs(temp_expr_table * tab)514 kill_virtual_exprs (temp_expr_table *tab)
515 {
516   kill_expr (tab, VIRTUAL_PARTITION (tab));
517 }
518 
519 
520 /* Mark the expression associated with VAR as replaceable, and enter
521    the defining stmt into the partition_dependencies table TAB.  If
522    MORE_REPLACING is true, accumulate the pending partition dependencies.  */
523 
524 static void
mark_replaceable(temp_expr_table * tab,tree var,bool more_replacing)525 mark_replaceable (temp_expr_table *tab, tree var, bool more_replacing)
526 {
527   int version = SSA_NAME_VERSION (var);
528 
529   /* Move the dependence list to the pending listpending.  */
530   if (more_replacing && tab->partition_dependencies[version])
531     bitmap_ior_into (tab->new_replaceable_dependencies,
532 		     tab->partition_dependencies[version]);
533 
534   finished_with_expr (tab, version, !more_replacing);
535 
536   /* Set the replaceable expression.
537      The bitmap for this "escapes" from this file so it's allocated
538      on the default obstack.  */
539   if (!tab->replaceable_expressions)
540     tab->replaceable_expressions = BITMAP_ALLOC (NULL);
541   bitmap_set_bit (tab->replaceable_expressions, version);
542 }
543 
544 
545 /* Helper function for find_ssaname_in_stores.  Called via walk_tree to
546    find a SSA_NAME DATA somewhere in *TP.  */
547 
548 static tree
find_ssaname(tree * tp,int * walk_subtrees,void * data)549 find_ssaname (tree *tp, int *walk_subtrees, void *data)
550 {
551   tree var = (tree) data;
552   if (*tp == var)
553     return var;
554   else if (IS_TYPE_OR_DECL_P (*tp))
555     *walk_subtrees = 0;
556   return NULL_TREE;
557 }
558 
559 /* Helper function for find_replaceable_in_bb.  Return true if SSA_NAME DATA
560    is used somewhere in T, which is a store in the statement.  Called via
561    walk_stmt_load_store_addr_ops.  */
562 
563 static bool
find_ssaname_in_store(gimple *,tree,tree t,void * data)564 find_ssaname_in_store (gimple *, tree, tree t, void *data)
565 {
566   return walk_tree (&t, find_ssaname, data, NULL) != NULL_TREE;
567 }
568 
569 /* This function processes basic block BB, and looks for variables which can
570    be replaced by their expressions.  Results are stored in the table TAB.  */
571 
572 static void
find_replaceable_in_bb(temp_expr_table * tab,basic_block bb)573 find_replaceable_in_bb (temp_expr_table *tab, basic_block bb)
574 {
575   gimple_stmt_iterator bsi;
576   gimple *stmt;
577   tree def, use, fndecl;
578   int partition;
579   var_map map = tab->map;
580   ssa_op_iter iter;
581   bool stmt_replaceable;
582   int cur_call_cnt = 0;
583   int cur_reg_vars_cnt = 0;
584 
585   for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
586     {
587       stmt = gsi_stmt (bsi);
588 
589       if (is_gimple_debug (stmt))
590 	continue;
591 
592       stmt_replaceable = ter_is_replaceable_p (stmt);
593 
594       /* Determine if this stmt finishes an existing expression.  */
595       FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
596 	{
597 	  unsigned ver = SSA_NAME_VERSION (use);
598 
599 	  /* If this use is a potential replacement variable, process it.  */
600 	  if (tab->expr_decl_uids[ver])
601 	    {
602 	      bool same_root_var = false;
603 	      ssa_op_iter iter2;
604 	      bitmap vars = tab->expr_decl_uids[ver];
605 
606 	      /* See if the root variables are the same.  If they are, we
607 		 do not want to do the replacement to avoid problems with
608 		 code size, see PR tree-optimization/17549.  */
609 	      if (!bitmap_empty_p (vars))
610 		FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter2, SSA_OP_DEF)
611 		  {
612 		    if (SSA_NAME_VAR (def)
613 			&& bitmap_bit_p (vars, DECL_UID (SSA_NAME_VAR (def))))
614 		      {
615 			same_root_var = true;
616 			break;
617 		      }
618 		  }
619 
620 	      /* If the stmt does a memory store and the replacement
621 	         is a load aliasing it avoid creating overlapping
622 		 assignments which we cannot expand correctly.  */
623 	      if (gimple_vdef (stmt))
624 		{
625 		  gimple *def_stmt = SSA_NAME_DEF_STMT (use);
626 		  while (is_gimple_assign (def_stmt)
627 			 && gimple_assign_rhs_code (def_stmt) == SSA_NAME)
628 		    def_stmt
629 		      = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
630 		  if (gimple_vuse (def_stmt)
631 		      && gimple_assign_single_p (def_stmt)
632 		      && stmt_may_clobber_ref_p (stmt,
633 						 gimple_assign_rhs1 (def_stmt)))
634 		    {
635 		      /* For calls, it is not a problem if USE is among
636 			 call's arguments or say OBJ_TYPE_REF argument,
637 			 all those necessarily need to be evaluated before
638 			 the call that may clobber the memory.  But if
639 			 LHS of the call refers to USE, expansion might
640 			 evaluate it after the call, prevent TER in that
641 			 case.
642 			 For inline asm, allow TER of loads into input
643 			 arguments, but disallow TER for USEs that occur
644 			 somewhere in outputs.  */
645 		      if (is_gimple_call (stmt)
646 			  || gimple_code (stmt) == GIMPLE_ASM)
647 			{
648 			  if (walk_stmt_load_store_ops (stmt, use, NULL,
649 							find_ssaname_in_store))
650 			    same_root_var = true;
651 			}
652 		      else
653 			same_root_var = true;
654 		    }
655 		}
656 
657 	      /* Mark expression as replaceable unless stmt is volatile, or the
658 		 def variable has the same root variable as something in the
659 		 substitution list, or the def and use span a call such that
660 		 we'll expand lifetimes across a call.  We also don't want to
661 		 replace across these expressions that may call libcalls that
662 		 clobber the register involved.  See PR 70184.  */
663 	      if (gimple_has_volatile_ops (stmt) || same_root_var
664 		  || (tab->call_cnt[ver] != cur_call_cnt
665 		      && SINGLE_SSA_USE_OPERAND (SSA_NAME_DEF_STMT (use), SSA_OP_USE)
666 			 == NULL_USE_OPERAND_P)
667 		  || tab->reg_vars_cnt[ver] != cur_reg_vars_cnt)
668 		finished_with_expr (tab, ver, true);
669 	      else
670 		mark_replaceable (tab, use, stmt_replaceable);
671 	    }
672 	}
673 
674       /* Next, see if this stmt kills off an active expression.  */
675       FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_DEF)
676 	{
677 	  partition = var_to_partition (map, def);
678 	  if (partition != NO_PARTITION && tab->kill_list[partition])
679 	    kill_expr (tab, partition);
680 	}
681 
682       /* Increment counter if this is a non BUILT_IN call. We allow
683 	 replacement over BUILT_IN calls since many will expand to inline
684 	 insns instead of a true call.  */
685       if (is_gimple_call (stmt)
686 	  && !((fndecl = gimple_call_fndecl (stmt))
687 	       && DECL_BUILT_IN (fndecl)))
688 	cur_call_cnt++;
689 
690       /* Increment counter if this statement sets a local
691 	 register variable.  */
692       if (gimple_assign_single_p (stmt)
693 	  && (TREE_CODE (gimple_assign_lhs (stmt)) == VAR_DECL
694 	  && DECL_HARD_REGISTER (gimple_assign_lhs (stmt))))
695 	cur_reg_vars_cnt++;
696 
697       /* Now see if we are creating a new expression or not.  */
698       if (stmt_replaceable)
699 	process_replaceable (tab, stmt, cur_call_cnt, cur_reg_vars_cnt);
700 
701       /* Free any unused dependency lists.  */
702       bitmap_clear (tab->new_replaceable_dependencies);
703 
704       /* A V_{MAY,MUST}_DEF kills any expression using a virtual operand,
705 	 including the current stmt.  */
706       if (gimple_vdef (stmt))
707         kill_virtual_exprs (tab);
708     }
709 }
710 
711 
712 /* This function is the driver routine for replacement of temporary expressions
713    in the SSA->normal phase, operating on MAP.  If there are replaceable
714    expressions, a table is returned which maps SSA versions to the
715    expressions they should be replaced with.  A NULL_TREE indicates no
716    replacement should take place.  If there are no replacements at all,
717    NULL is returned by the function, otherwise an expression vector indexed
718    by SSA_NAME version numbers.  */
719 
720 bitmap
find_replaceable_exprs(var_map map)721 find_replaceable_exprs (var_map map)
722 {
723   basic_block bb;
724   temp_expr_table *table;
725   bitmap ret;
726 
727   bitmap_obstack_initialize (&ter_bitmap_obstack);
728   table = new_temp_expr_table (map);
729   FOR_EACH_BB_FN (bb, cfun)
730     {
731       find_replaceable_in_bb (table, bb);
732       gcc_checking_assert (bitmap_empty_p (table->partition_in_use));
733     }
734   ret = free_temp_expr_table (table);
735   bitmap_obstack_release (&ter_bitmap_obstack);
736   return ret;
737 }
738 
739 /* Dump TER expression table EXPR to file F.  */
740 
741 void
dump_replaceable_exprs(FILE * f,bitmap expr)742 dump_replaceable_exprs (FILE *f, bitmap expr)
743 {
744   tree var;
745   unsigned x;
746 
747   fprintf (f, "\nReplacing Expressions\n");
748   for (x = 0; x < num_ssa_names; x++)
749     if (bitmap_bit_p (expr, x))
750       {
751 	var = ssa_name (x);
752 	print_generic_expr (f, var, TDF_SLIM);
753 	fprintf (f, " replace with --> ");
754 	print_gimple_stmt (f, SSA_NAME_DEF_STMT (var), 0, TDF_SLIM);
755 	fprintf (f, "\n");
756       }
757   fprintf (f, "\n");
758 }
759 
760 
761 /* Dump the status of the various tables in the expression table.  This is used
762    exclusively to debug TER.  F is the place to send debug info and T is the
763    table being debugged.  */
764 
765 DEBUG_FUNCTION void
debug_ter(FILE * f,temp_expr_table * t)766 debug_ter (FILE *f, temp_expr_table *t)
767 {
768   unsigned x, y;
769   bitmap_iterator bi;
770 
771   fprintf (f, "\nDumping current state of TER\n virtual partition = %d\n",
772 	   VIRTUAL_PARTITION (t));
773   if (t->replaceable_expressions)
774     dump_replaceable_exprs (f, t->replaceable_expressions);
775   fprintf (f, "Currently tracking the following expressions:\n");
776 
777   for (x = 1; x < num_ssa_names; x++)
778     if (t->expr_decl_uids[x])
779       {
780         print_generic_expr (f, ssa_name (x), TDF_SLIM);
781         fprintf (f, " dep-parts : ");
782 	if (t->partition_dependencies[x]
783 	    && !bitmap_empty_p (t->partition_dependencies[x]))
784 	  {
785 	    EXECUTE_IF_SET_IN_BITMAP (t->partition_dependencies[x], 0, y, bi)
786 	      fprintf (f, "P%d ",y);
787 	  }
788 	fprintf (f, "   basedecls: ");
789 	EXECUTE_IF_SET_IN_BITMAP (t->expr_decl_uids[x], 0, y, bi)
790 	  fprintf (f, "%d ",y);
791 	fprintf (f, "   call_cnt : %d",t->call_cnt[x]);
792 	fprintf (f, "\n");
793       }
794 
795   bitmap_print (f, t->partition_in_use, "Partitions in use ",
796 		"\npartition KILL lists:\n");
797 
798   for (x = 0; x <= num_var_partitions (t->map); x++)
799     if (t->kill_list[x])
800       {
801         fprintf (f, "Partition %d : ", x);
802 	EXECUTE_IF_SET_IN_BITMAP (t->kill_list[x], 0, y, bi)
803 	  fprintf (f, "_%d ",y);
804       }
805 
806   fprintf (f, "\n----------\n");
807 }
808