1 /* Callgraph clones
2 Copyright (C) 2003-2021 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 /* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
24
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
27 clones
28
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
32
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
38
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
42
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
49 a given function.
50
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
54
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
61
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
66
67 #include "config.h"
68 #include "system.h"
69 #include "coretypes.h"
70 #include "backend.h"
71 #include "target.h"
72 #include "rtl.h"
73 #include "tree.h"
74 #include "gimple.h"
75 #include "stringpool.h"
76 #include "cgraph.h"
77 #include "lto-streamer.h"
78 #include "tree-eh.h"
79 #include "tree-cfg.h"
80 #include "tree-inline.h"
81 #include "dumpfile.h"
82 #include "gimple-pretty-print.h"
83 #include "alloc-pool.h"
84 #include "symbol-summary.h"
85 #include "tree-vrp.h"
86 #include "ipa-prop.h"
87 #include "ipa-fnsummary.h"
88 #include "symtab-thunks.h"
89 #include "symtab-clones.h"
90
91 /* Create clone of edge in the node N represented by CALL_EXPR
92 the callgraph. */
93
94 cgraph_edge *
clone(cgraph_node * n,gcall * call_stmt,unsigned stmt_uid,profile_count num,profile_count den,bool update_original)95 cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid,
96 profile_count num, profile_count den,
97 bool update_original)
98 {
99 cgraph_edge *new_edge;
100 profile_count::adjust_for_ipa_scaling (&num, &den);
101 profile_count prof_count = count.apply_scale (num, den);
102
103 if (indirect_unknown_callee)
104 {
105 tree decl;
106
107 if (call_stmt && (decl = gimple_call_fndecl (call_stmt))
108 /* When the call is speculative, we need to resolve it
109 via cgraph_resolve_speculation and not here. */
110 && !speculative)
111 {
112 cgraph_node *callee = cgraph_node::get (decl);
113 gcc_checking_assert (callee);
114 new_edge = n->create_edge (callee, call_stmt, prof_count, true);
115 }
116 else
117 {
118 new_edge = n->create_indirect_edge (call_stmt,
119 indirect_info->ecf_flags,
120 prof_count, true);
121 *new_edge->indirect_info = *indirect_info;
122 }
123 }
124 else
125 {
126 new_edge = n->create_edge (callee, call_stmt, prof_count, true);
127 if (indirect_info)
128 {
129 new_edge->indirect_info
130 = ggc_cleared_alloc<cgraph_indirect_call_info> ();
131 *new_edge->indirect_info = *indirect_info;
132 }
133 }
134
135 new_edge->inline_failed = inline_failed;
136 new_edge->indirect_inlining_edge = indirect_inlining_edge;
137 if (!call_stmt)
138 new_edge->lto_stmt_uid = stmt_uid;
139 new_edge->speculative_id = speculative_id;
140 /* Clone flags that depend on call_stmt availability manually. */
141 new_edge->can_throw_external = can_throw_external;
142 new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p;
143 new_edge->speculative = speculative;
144 new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor;
145
146 /* Update IPA profile. Local profiles need no updating in original. */
147 if (update_original)
148 count = count.combine_with_ipa_count_within (count.ipa ()
149 - new_edge->count.ipa (),
150 caller->count);
151 symtab->call_edge_duplication_hooks (this, new_edge);
152 return new_edge;
153 }
154
155 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
156 clone or its thunk. */
157
158 static void
set_new_clone_decl_and_node_flags(cgraph_node * new_node)159 set_new_clone_decl_and_node_flags (cgraph_node *new_node)
160 {
161 DECL_EXTERNAL (new_node->decl) = 0;
162 TREE_PUBLIC (new_node->decl) = 0;
163 DECL_COMDAT (new_node->decl) = 0;
164 DECL_WEAK (new_node->decl) = 0;
165 DECL_VIRTUAL_P (new_node->decl) = 0;
166 DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0;
167 DECL_STATIC_DESTRUCTOR (new_node->decl) = 0;
168 DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0);
169 DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0);
170 DECL_IS_REPLACEABLE_OPERATOR (new_node->decl) = 0;
171
172 new_node->externally_visible = 0;
173 new_node->local = 1;
174 new_node->lowered = true;
175 }
176
177 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
178 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
179 Function can return NODE if no thunk is necessary, which can happen when
180 thunk is this_adjusting but we are removing this parameter. */
181
182 static cgraph_node *
duplicate_thunk_for_node(cgraph_node * thunk,cgraph_node * node)183 duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node)
184 {
185 cgraph_node *new_thunk, *thunk_of;
186 thunk_of = thunk->callees->callee->ultimate_alias_target ();
187
188 if (thunk_of->thunk)
189 node = duplicate_thunk_for_node (thunk_of, node);
190
191 if (!DECL_ARGUMENTS (thunk->decl))
192 thunk->get_untransformed_body ();
193
194 thunk_info *i = thunk_info::get (thunk);
195 cgraph_edge *cs;
196 for (cs = node->callers; cs; cs = cs->next_caller)
197 if (cs->caller->thunk)
198 {
199 thunk_info *i2 = thunk_info::get (cs->caller);
200 if (*i2 == *i)
201 return cs->caller;
202 }
203
204 tree new_decl;
205 clone_info *info = clone_info::get (node);
206 if (info && info->param_adjustments)
207 {
208 /* We do not need to duplicate this_adjusting thunks if we have removed
209 this. */
210 if (i->this_adjusting
211 && !info->param_adjustments->first_param_intact_p ())
212 return node;
213
214 new_decl = copy_node (thunk->decl);
215 ipa_param_body_adjustments body_adj (info->param_adjustments,
216 new_decl);
217 body_adj.modify_formal_parameters ();
218 }
219 else
220 new_decl = copy_node (thunk->decl);
221
222 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl));
223 gcc_checking_assert (!DECL_INITIAL (new_decl));
224 gcc_checking_assert (!DECL_RESULT (new_decl));
225 gcc_checking_assert (!DECL_RTL_SET_P (new_decl));
226
227 DECL_NAME (new_decl) = clone_function_name_numbered (thunk->decl,
228 "artificial_thunk");
229 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
230
231 /* We need to force DECL_IGNORED_P because the new thunk is created after
232 early debug was run. */
233 DECL_IGNORED_P (new_decl) = 1;
234
235 new_thunk = cgraph_node::create (new_decl);
236 set_new_clone_decl_and_node_flags (new_thunk);
237 new_thunk->definition = true;
238 new_thunk->can_change_signature = node->can_change_signature;
239 new_thunk->thunk = thunk->thunk;
240 new_thunk->unique_name = in_lto_p;
241 new_thunk->former_clone_of = thunk->decl;
242 if (info && info->param_adjustments)
243 clone_info::get_create (new_thunk)->param_adjustments
244 = info->param_adjustments;
245 new_thunk->unit_id = thunk->unit_id;
246 new_thunk->merged_comdat = thunk->merged_comdat;
247 new_thunk->merged_extern_inline = thunk->merged_extern_inline;
248
249 cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count);
250 symtab->call_edge_duplication_hooks (thunk->callees, e);
251 symtab->call_cgraph_duplication_hooks (thunk, new_thunk);
252 return new_thunk;
253 }
254
255 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
256 one or more equivalent thunks for N and redirect E to the first in the
257 chain. Note that it is then necessary to call
258 n->expand_all_artificial_thunks once all callers are redirected. */
259
260 void
redirect_callee_duplicating_thunks(cgraph_node * n)261 cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n)
262 {
263 cgraph_node *orig_to = callee->ultimate_alias_target ();
264 if (orig_to->thunk)
265 n = duplicate_thunk_for_node (orig_to, n);
266
267 redirect_callee (n);
268 }
269
270 /* Call expand_thunk on all callers that are thunks and if analyze those nodes
271 that were expanded. */
272
273 void
expand_all_artificial_thunks()274 cgraph_node::expand_all_artificial_thunks ()
275 {
276 cgraph_edge *e;
277 for (e = callers; e;)
278 if (e->caller->thunk)
279 {
280 cgraph_node *thunk = e->caller;
281
282 e = e->next_caller;
283 if (expand_thunk (thunk, false, false))
284 {
285 thunk->thunk = false;
286 thunk->analyze ();
287 ipa_analyze_node (thunk);
288 inline_analyze_function (thunk);
289 }
290 thunk->expand_all_artificial_thunks ();
291 }
292 else
293 e = e->next_caller;
294 }
295
296 void
dump_callgraph_transformation(const cgraph_node * original,const cgraph_node * clone,const char * suffix)297 dump_callgraph_transformation (const cgraph_node *original,
298 const cgraph_node *clone,
299 const char *suffix)
300 {
301 if (symtab->ipa_clones_dump_file)
302 {
303 fprintf (symtab->ipa_clones_dump_file,
304 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
305 original->asm_name (), original->order,
306 DECL_SOURCE_FILE (original->decl),
307 DECL_SOURCE_LINE (original->decl),
308 DECL_SOURCE_COLUMN (original->decl), clone->asm_name (),
309 clone->order, DECL_SOURCE_FILE (clone->decl),
310 DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl),
311 suffix);
312
313 symtab->cloned_nodes.add (original);
314 symtab->cloned_nodes.add (clone);
315 }
316 }
317
318 /* Turn profile of N to local profile. */
319
320 static void
localize_profile(cgraph_node * n)321 localize_profile (cgraph_node *n)
322 {
323 n->count = n->count.guessed_local ();
324 for (cgraph_edge *e = n->callees; e; e=e->next_callee)
325 {
326 e->count = e->count.guessed_local ();
327 if (!e->inline_failed)
328 localize_profile (e->callee);
329 }
330 for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee)
331 e->count = e->count.guessed_local ();
332 }
333
334 /* Create node representing clone of N executed COUNT times. Decrease
335 the execution counts from original node too.
336 The new clone will have decl set to DECL that may or may not be the same
337 as decl of N.
338
339 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
340 function's profile to reflect the fact that part of execution is handled
341 by node.
342 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
343 the new clone. Otherwise the caller is responsible for doing so later.
344
345 If the new node is being inlined into another one, NEW_INLINED_TO should be
346 the outline function the new one is (even indirectly) inlined to. All hooks
347 will see this in node's inlined_to, when invoked. Can be NULL if the
348 node is not inlined.
349
350 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
351 will be overwritten by the new structure. Otherwise the new node will
352 share parameter manipulation information with the original node. */
353
354 cgraph_node *
create_clone(tree new_decl,profile_count prof_count,bool update_original,vec<cgraph_edge * > redirect_callers,bool call_duplication_hook,cgraph_node * new_inlined_to,ipa_param_adjustments * param_adjustments,const char * suffix)355 cgraph_node::create_clone (tree new_decl, profile_count prof_count,
356 bool update_original,
357 vec<cgraph_edge *> redirect_callers,
358 bool call_duplication_hook,
359 cgraph_node *new_inlined_to,
360 ipa_param_adjustments *param_adjustments,
361 const char *suffix)
362 {
363 cgraph_node *new_node = symtab->create_empty ();
364 cgraph_edge *e;
365 unsigned i;
366 profile_count old_count = count;
367 bool nonzero = count.ipa ().nonzero_p ();
368
369 if (new_inlined_to)
370 dump_callgraph_transformation (this, new_inlined_to, "inlining to");
371
372 /* When inlining we scale precisely to prof_count, when cloning we can
373 preserve local profile. */
374 if (!new_inlined_to)
375 prof_count = count.combine_with_ipa_count (prof_count);
376 new_node->count = prof_count;
377
378 /* Update IPA profile. Local profiles need no updating in original. */
379 if (update_original)
380 {
381 if (inlined_to)
382 count = count.combine_with_ipa_count_within (count.ipa ()
383 - prof_count.ipa (),
384 inlined_to->count);
385 else
386 count = count.combine_with_ipa_count (count.ipa () - prof_count.ipa ());
387 }
388 new_node->decl = new_decl;
389 new_node->register_symbol ();
390 new_node->lto_file_data = lto_file_data;
391 new_node->analyzed = analyzed;
392 new_node->definition = definition;
393 new_node->versionable = versionable;
394 new_node->can_change_signature = can_change_signature;
395 new_node->redefined_extern_inline = redefined_extern_inline;
396 new_node->tm_may_enter_irr = tm_may_enter_irr;
397 new_node->externally_visible = false;
398 new_node->no_reorder = no_reorder;
399 new_node->local = true;
400 new_node->inlined_to = new_inlined_to;
401 new_node->rtl = rtl;
402 new_node->frequency = frequency;
403 new_node->tp_first_run = tp_first_run;
404 new_node->tm_clone = tm_clone;
405 new_node->icf_merged = icf_merged;
406 new_node->thunk = thunk;
407 new_node->unit_id = unit_id;
408 new_node->merged_comdat = merged_comdat;
409 new_node->merged_extern_inline = merged_extern_inline;
410 clone_info *info = clone_info::get (this);
411
412 if (param_adjustments)
413 clone_info::get_create (new_node)->param_adjustments = param_adjustments;
414 else if (info && info->param_adjustments)
415 clone_info::get_create (new_node)->param_adjustments
416 = info->param_adjustments;
417 if (info && info->performed_splits)
418 clone_info::get_create (new_node)->performed_splits
419 = vec_safe_copy (info->performed_splits);
420 new_node->split_part = split_part;
421
422 FOR_EACH_VEC_ELT (redirect_callers, i, e)
423 {
424 /* Redirect calls to the old version node to point to its new
425 version. The only exception is when the edge was proved to
426 be unreachable during the cloning procedure. */
427 if (!e->callee
428 || !fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE))
429 e->redirect_callee_duplicating_thunks (new_node);
430 }
431 new_node->expand_all_artificial_thunks ();
432
433 for (e = callees;e; e=e->next_callee)
434 e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count,
435 update_original);
436
437 for (e = indirect_calls; e; e = e->next_callee)
438 e->clone (new_node, e->call_stmt, e->lto_stmt_uid,
439 new_node->count, old_count, update_original);
440 new_node->clone_references (this);
441
442 new_node->next_sibling_clone = clones;
443 if (clones)
444 clones->prev_sibling_clone = new_node;
445 clones = new_node;
446 new_node->clone_of = this;
447
448 if (call_duplication_hook)
449 symtab->call_cgraph_duplication_hooks (this, new_node);
450 /* With partial train run we do not want to assume that original's
451 count is zero whenever we redurect all executed edges to clone.
452 Simply drop profile to local one in this case. */
453 if (update_original
454 && opt_for_fn (decl, flag_profile_partial_training)
455 && nonzero
456 && count.ipa_p ()
457 && !count.ipa ().nonzero_p ()
458 && !inlined_to)
459 localize_profile (this);
460
461 if (!new_inlined_to)
462 dump_callgraph_transformation (this, new_node, suffix);
463
464 return new_node;
465 }
466
467 static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids;
468
469 /* Return a new assembler name for a clone of decl named NAME. Apart
470 from the string SUFFIX, the new name will end with a unique (for
471 each NAME) unspecified number. If clone numbering is not needed
472 then the two argument clone_function_name should be used instead.
473 Should not be called directly except for by
474 lto-partition.c:privatize_symbol_name_1. */
475
476 tree
clone_function_name_numbered(const char * name,const char * suffix)477 clone_function_name_numbered (const char *name, const char *suffix)
478 {
479 /* Initialize the function->counter mapping the first time it's
480 needed. */
481 if (!clone_fn_ids)
482 clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (64);
483 unsigned int &suffix_counter = clone_fn_ids->get_or_insert (
484 IDENTIFIER_POINTER (get_identifier (name)));
485 return clone_function_name (name, suffix, suffix_counter++);
486 }
487
488 /* Return a new assembler name for a clone of DECL. Apart from string
489 SUFFIX, the new name will end with a unique (for each DECL
490 assembler name) unspecified number. If clone numbering is not
491 needed then the two argument clone_function_name should be used
492 instead. */
493
494 tree
clone_function_name_numbered(tree decl,const char * suffix)495 clone_function_name_numbered (tree decl, const char *suffix)
496 {
497 tree name = DECL_ASSEMBLER_NAME (decl);
498 return clone_function_name_numbered (IDENTIFIER_POINTER (name),
499 suffix);
500 }
501
502 /* Return a new assembler name for a clone of decl named NAME. Apart
503 from the string SUFFIX, the new name will end with the specified
504 NUMBER. If clone numbering is not needed then the two argument
505 clone_function_name should be used instead. */
506
507 tree
clone_function_name(const char * name,const char * suffix,unsigned long number)508 clone_function_name (const char *name, const char *suffix,
509 unsigned long number)
510 {
511 size_t len = strlen (name);
512 char *tmp_name, *prefix;
513
514 prefix = XALLOCAVEC (char, len + strlen (suffix) + 2);
515 memcpy (prefix, name, len);
516 strcpy (prefix + len + 1, suffix);
517 prefix[len] = symbol_table::symbol_suffix_separator ();
518 ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number);
519 return get_identifier (tmp_name);
520 }
521
522 /* Return a new assembler name for a clone of DECL. Apart from the
523 string SUFFIX, the new name will end with the specified NUMBER. If
524 clone numbering is not needed then the two argument
525 clone_function_name should be used instead. */
526
527 tree
clone_function_name(tree decl,const char * suffix,unsigned long number)528 clone_function_name (tree decl, const char *suffix,
529 unsigned long number)
530 {
531 return clone_function_name (
532 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number);
533 }
534
535 /* Return a new assembler name ending with the string SUFFIX for a
536 clone of DECL. */
537
538 tree
clone_function_name(tree decl,const char * suffix)539 clone_function_name (tree decl, const char *suffix)
540 {
541 tree identifier = DECL_ASSEMBLER_NAME (decl);
542 /* For consistency this needs to behave the same way as
543 ASM_FORMAT_PRIVATE_NAME does, but without the final number
544 suffix. */
545 char *separator = XALLOCAVEC (char, 2);
546 separator[0] = symbol_table::symbol_suffix_separator ();
547 separator[1] = 0;
548 #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
549 const char *prefix = "__";
550 #else
551 const char *prefix = "";
552 #endif
553 char *result = ACONCAT ((prefix,
554 IDENTIFIER_POINTER (identifier),
555 separator,
556 suffix,
557 (char*)0));
558 return get_identifier (result);
559 }
560
561
562 /* Create callgraph node clone with new declaration. The actual body will be
563 copied later at compilation stage. The name of the new clone will be
564 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
565
566 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
567 bitmap interface.
568 */
569 cgraph_node *
create_virtual_clone(vec<cgraph_edge * > redirect_callers,vec<ipa_replace_map *,va_gc> * tree_map,ipa_param_adjustments * param_adjustments,const char * suffix,unsigned num_suffix)570 cgraph_node::create_virtual_clone (vec<cgraph_edge *> redirect_callers,
571 vec<ipa_replace_map *, va_gc> *tree_map,
572 ipa_param_adjustments *param_adjustments,
573 const char * suffix, unsigned num_suffix)
574 {
575 tree old_decl = decl;
576 cgraph_node *new_node = NULL;
577 tree new_decl;
578 size_t len, i;
579 ipa_replace_map *map;
580 char *name;
581
582 gcc_checking_assert (versionable);
583 /* TODO: It would be nice if we could recognize that param_adjustments do not
584 actually perform any changes, but at the moment let's require it simply
585 does not exist. */
586 gcc_assert (can_change_signature || !param_adjustments);
587
588 /* Make a new FUNCTION_DECL tree node */
589 if (!param_adjustments)
590 new_decl = copy_node (old_decl);
591 else
592 new_decl = param_adjustments->adjust_decl (old_decl);
593
594 /* These pointers represent function body and will be populated only when clone
595 is materialized. */
596 gcc_assert (new_decl != old_decl);
597 DECL_STRUCT_FUNCTION (new_decl) = NULL;
598 DECL_ARGUMENTS (new_decl) = NULL;
599 DECL_INITIAL (new_decl) = NULL;
600 DECL_RESULT (new_decl) = NULL;
601 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
602 sometimes storing only clone decl instead of original. */
603
604 /* Generate a new name for the new version. */
605 len = IDENTIFIER_LENGTH (DECL_NAME (old_decl));
606 name = XALLOCAVEC (char, len + strlen (suffix) + 2);
607 memcpy (name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), len);
608 strcpy (name + len + 1, suffix);
609 name[len] = '.';
610 DECL_NAME (new_decl) = get_identifier (name);
611 SET_DECL_ASSEMBLER_NAME (new_decl,
612 clone_function_name (old_decl, suffix, num_suffix));
613 SET_DECL_RTL (new_decl, NULL);
614
615 new_node = create_clone (new_decl, count, false,
616 redirect_callers, false, NULL, param_adjustments,
617 suffix);
618
619 /* Update the properties.
620 Make clone visible only within this translation unit. Make sure
621 that is not weak also.
622 ??? We cannot use COMDAT linkage because there is no
623 ABI support for this. */
624 set_new_clone_decl_and_node_flags (new_node);
625 new_node->ipcp_clone = ipcp_clone;
626 if (tree_map)
627 clone_info::get_create (new_node)->tree_map = tree_map;
628 if (!implicit_section)
629 new_node->set_section (*this);
630
631 /* Clones of global symbols or symbols with unique names are unique. */
632 if ((TREE_PUBLIC (old_decl)
633 && !DECL_EXTERNAL (old_decl)
634 && !DECL_WEAK (old_decl)
635 && !DECL_COMDAT (old_decl))
636 || in_lto_p)
637 new_node->unique_name = true;
638 FOR_EACH_VEC_SAFE_ELT (tree_map, i, map)
639 new_node->maybe_create_reference (map->new_tree, NULL);
640
641 if (ipa_transforms_to_apply.exists ())
642 new_node->ipa_transforms_to_apply
643 = ipa_transforms_to_apply.copy ();
644
645 symtab->call_cgraph_duplication_hooks (this, new_node);
646
647 return new_node;
648 }
649
650 /* callgraph node being removed from symbol table; see if its entry can be
651 replaced by other inline clone.
652 INFO is clone info to attach to the new root. */
653 cgraph_node *
find_replacement(clone_info * info)654 cgraph_node::find_replacement (clone_info *info)
655 {
656 cgraph_node *next_inline_clone, *replacement;
657
658 for (next_inline_clone = clones;
659 next_inline_clone
660 && next_inline_clone->decl != decl;
661 next_inline_clone = next_inline_clone->next_sibling_clone)
662 ;
663
664 /* If there is inline clone of the node being removed, we need
665 to put it into the position of removed node and reorganize all
666 other clones to be based on it. */
667 if (next_inline_clone)
668 {
669 cgraph_node *n;
670 cgraph_node *new_clones;
671
672 replacement = next_inline_clone;
673
674 /* Unlink inline clone from the list of clones of removed node. */
675 if (next_inline_clone->next_sibling_clone)
676 next_inline_clone->next_sibling_clone->prev_sibling_clone
677 = next_inline_clone->prev_sibling_clone;
678 if (next_inline_clone->prev_sibling_clone)
679 {
680 gcc_assert (clones != next_inline_clone);
681 next_inline_clone->prev_sibling_clone->next_sibling_clone
682 = next_inline_clone->next_sibling_clone;
683 }
684 else
685 {
686 gcc_assert (clones == next_inline_clone);
687 clones = next_inline_clone->next_sibling_clone;
688 }
689
690 new_clones = clones;
691 clones = NULL;
692
693 /* Copy clone info. */
694 if (info)
695 *clone_info::get_create (next_inline_clone) = *info;
696
697 /* Now place it into clone tree at same level at NODE. */
698 next_inline_clone->clone_of = clone_of;
699 next_inline_clone->prev_sibling_clone = NULL;
700 next_inline_clone->next_sibling_clone = NULL;
701 if (clone_of)
702 {
703 if (clone_of->clones)
704 clone_of->clones->prev_sibling_clone = next_inline_clone;
705 next_inline_clone->next_sibling_clone = clone_of->clones;
706 clone_of->clones = next_inline_clone;
707 }
708
709 /* Merge the clone list. */
710 if (new_clones)
711 {
712 if (!next_inline_clone->clones)
713 next_inline_clone->clones = new_clones;
714 else
715 {
716 n = next_inline_clone->clones;
717 while (n->next_sibling_clone)
718 n = n->next_sibling_clone;
719 n->next_sibling_clone = new_clones;
720 new_clones->prev_sibling_clone = n;
721 }
722 }
723
724 /* Update clone_of pointers. */
725 n = new_clones;
726 while (n)
727 {
728 n->clone_of = next_inline_clone;
729 n = n->next_sibling_clone;
730 }
731
732 /* Update order in order to be able to find a LTO section
733 with function body. */
734 replacement->order = order;
735
736 return replacement;
737 }
738 else
739 return NULL;
740 }
741
742 /* Like cgraph_set_call_stmt but walk the clone tree and update all
743 clones sharing the same function body.
744 When WHOLE_SPECULATIVE_EDGES is true, all three components of
745 speculative edge gets updated. Otherwise we update only direct
746 call. */
747
748 void
set_call_stmt_including_clones(gimple * old_stmt,gcall * new_stmt,bool update_speculative)749 cgraph_node::set_call_stmt_including_clones (gimple *old_stmt,
750 gcall *new_stmt,
751 bool update_speculative)
752 {
753 cgraph_node *node;
754 cgraph_edge *master_edge = get_edge (old_stmt);
755
756 if (master_edge)
757 cgraph_edge::set_call_stmt (master_edge, new_stmt, update_speculative);
758
759 node = clones;
760 if (node)
761 while (node != this)
762 {
763 cgraph_edge *edge = node->get_edge (old_stmt);
764 if (edge)
765 {
766 edge = cgraph_edge::set_call_stmt (edge, new_stmt,
767 update_speculative);
768 /* If UPDATE_SPECULATIVE is false, it means that we are turning
769 speculative call into a real code sequence. Update the
770 callgraph edges. */
771 if (edge->speculative && !update_speculative)
772 {
773 cgraph_edge *indirect = edge->speculative_call_indirect_edge ();
774
775 for (cgraph_edge *next, *direct
776 = edge->first_speculative_call_target ();
777 direct;
778 direct = next)
779 {
780 next = direct->next_speculative_call_target ();
781 direct->speculative_call_target_ref ()->speculative = false;
782 direct->speculative = false;
783 }
784 indirect->speculative = false;
785 }
786 }
787 if (node->clones)
788 node = node->clones;
789 else if (node->next_sibling_clone)
790 node = node->next_sibling_clone;
791 else
792 {
793 while (node != this && !node->next_sibling_clone)
794 node = node->clone_of;
795 if (node != this)
796 node = node->next_sibling_clone;
797 }
798 }
799 }
800
801 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
802 same function body. If clones already have edge for OLD_STMT; only
803 update the edge same way as cgraph_set_call_stmt_including_clones does.
804
805 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
806 frequencies of the clones. */
807
808 void
create_edge_including_clones(cgraph_node * callee,gimple * old_stmt,gcall * stmt,profile_count count,cgraph_inline_failed_t reason)809 cgraph_node::create_edge_including_clones (cgraph_node *callee,
810 gimple *old_stmt, gcall *stmt,
811 profile_count count,
812 cgraph_inline_failed_t reason)
813 {
814 cgraph_node *node;
815
816 if (!get_edge (stmt))
817 {
818 cgraph_edge *edge = create_edge (callee, stmt, count);
819 edge->inline_failed = reason;
820 }
821
822 node = clones;
823 if (node)
824 while (node != this)
825 /* Thunk clones do not get updated while copying inline function body. */
826 if (!node->thunk)
827 {
828 cgraph_edge *edge = node->get_edge (old_stmt);
829
830 /* It is possible that clones already contain the edge while
831 master didn't. Either we promoted indirect call into direct
832 call in the clone or we are processing clones of unreachable
833 master where edges has been removed. */
834 if (edge)
835 edge = cgraph_edge::set_call_stmt (edge, stmt);
836 else if (! node->get_edge (stmt))
837 {
838 edge = node->create_edge (callee, stmt, count);
839 edge->inline_failed = reason;
840 }
841
842 if (node->clones)
843 node = node->clones;
844 else if (node->next_sibling_clone)
845 node = node->next_sibling_clone;
846 else
847 {
848 while (node != this && !node->next_sibling_clone)
849 node = node->clone_of;
850 if (node != this)
851 node = node->next_sibling_clone;
852 }
853 }
854 }
855
856 /* Remove the node from cgraph and all inline clones inlined into it.
857 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
858 removed. This allows to call the function from outer loop walking clone
859 tree. */
860
861 bool
remove_symbol_and_inline_clones(cgraph_node * forbidden_node)862 cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node)
863 {
864 cgraph_edge *e, *next;
865 bool found = false;
866
867 if (this == forbidden_node)
868 {
869 cgraph_edge::remove (callers);
870 return true;
871 }
872 for (e = callees; e; e = next)
873 {
874 next = e->next_callee;
875 if (!e->inline_failed)
876 found |= e->callee->remove_symbol_and_inline_clones (forbidden_node);
877 }
878 remove ();
879 return found;
880 }
881
882 /* The edges representing the callers of the NEW_VERSION node were
883 fixed by cgraph_function_versioning (), now the call_expr in their
884 respective tree code should be updated to call the NEW_VERSION. */
885
886 static void
update_call_expr(cgraph_node * new_version)887 update_call_expr (cgraph_node *new_version)
888 {
889 cgraph_edge *e;
890
891 gcc_assert (new_version);
892
893 /* Update the call expr on the edges to call the new version. */
894 for (e = new_version->callers; e; e = e->next_caller)
895 {
896 function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl);
897 gimple_call_set_fndecl (e->call_stmt, new_version->decl);
898 maybe_clean_eh_stmt_fn (inner_function, e->call_stmt);
899 }
900 }
901
902
903 /* Create a new cgraph node which is the new version of
904 callgraph node. REDIRECT_CALLERS holds the callers
905 edges which should be redirected to point to
906 NEW_VERSION. ALL the callees edges of the node
907 are cloned to the new version node. Return the new
908 version node.
909
910 If non-NULL BLOCK_TO_COPY determine what basic blocks
911 was copied to prevent duplications of calls that are dead
912 in the clone. */
913
914 cgraph_node *
create_version_clone(tree new_decl,vec<cgraph_edge * > redirect_callers,bitmap bbs_to_copy,const char * suffix)915 cgraph_node::create_version_clone (tree new_decl,
916 vec<cgraph_edge *> redirect_callers,
917 bitmap bbs_to_copy,
918 const char *suffix)
919 {
920 cgraph_node *new_version;
921 cgraph_edge *e;
922 unsigned i;
923
924 new_version = cgraph_node::create (new_decl);
925
926 new_version->analyzed = analyzed;
927 new_version->definition = definition;
928 new_version->local = local;
929 new_version->externally_visible = false;
930 new_version->no_reorder = no_reorder;
931 new_version->local = new_version->definition;
932 new_version->inlined_to = inlined_to;
933 new_version->rtl = rtl;
934 new_version->count = count;
935 new_version->unit_id = unit_id;
936 new_version->merged_comdat = merged_comdat;
937 new_version->merged_extern_inline = merged_extern_inline;
938
939 for (e = callees; e; e=e->next_callee)
940 if (!bbs_to_copy
941 || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
942 e->clone (new_version, e->call_stmt,
943 e->lto_stmt_uid, count, count,
944 true);
945 for (e = indirect_calls; e; e=e->next_callee)
946 if (!bbs_to_copy
947 || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
948 e->clone (new_version, e->call_stmt,
949 e->lto_stmt_uid, count, count,
950 true);
951 FOR_EACH_VEC_ELT (redirect_callers, i, e)
952 {
953 /* Redirect calls to the old version node to point to its new
954 version. */
955 e->redirect_callee (new_version);
956 }
957
958 dump_callgraph_transformation (this, new_version, suffix);
959
960 return new_version;
961 }
962
963 /* Perform function versioning.
964 Function versioning includes copying of the tree and
965 a callgraph update (creating a new cgraph node and updating
966 its callees and callers).
967
968 REDIRECT_CALLERS varray includes the edges to be redirected
969 to the new version.
970
971 TREE_MAP is a mapping of tree nodes we want to replace with
972 new ones (according to results of prior analysis).
973
974 If non-NULL ARGS_TO_SKIP determine function parameters to remove
975 from new version.
976 If SKIP_RETURN is true, the new version will return void.
977 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
978 If non_NULL NEW_ENTRY determine new entry BB of the clone.
979
980 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
981 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
982 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
983 of the declaration.
984
985 Return the new version's cgraph node. */
986
987 cgraph_node *
create_version_clone_with_body(vec<cgraph_edge * > redirect_callers,vec<ipa_replace_map *,va_gc> * tree_map,ipa_param_adjustments * param_adjustments,bitmap bbs_to_copy,basic_block new_entry_block,const char * suffix,tree target_attributes)988 cgraph_node::create_version_clone_with_body
989 (vec<cgraph_edge *> redirect_callers,
990 vec<ipa_replace_map *, va_gc> *tree_map,
991 ipa_param_adjustments *param_adjustments,
992 bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix,
993 tree target_attributes)
994 {
995 tree old_decl = decl;
996 cgraph_node *new_version_node = NULL;
997 tree new_decl;
998
999 if (!tree_versionable_function_p (old_decl))
1000 return NULL;
1001
1002 /* TODO: Restore an assert that we do not change signature if
1003 can_change_signature is false. We cannot just check that
1004 param_adjustments is NULL because unfortunately ipa-split removes return
1005 values from such functions. */
1006
1007 /* Make a new FUNCTION_DECL tree node for the new version. */
1008 if (param_adjustments)
1009 new_decl = param_adjustments->adjust_decl (old_decl);
1010 else
1011 new_decl = copy_node (old_decl);
1012
1013 /* Generate a new name for the new version. */
1014 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl, suffix);
1015 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
1016 SET_DECL_RTL (new_decl, NULL);
1017
1018 DECL_VIRTUAL_P (new_decl) = 0;
1019
1020 if (target_attributes)
1021 {
1022 DECL_ATTRIBUTES (new_decl) = target_attributes;
1023
1024 location_t saved_loc = input_location;
1025 tree v = TREE_VALUE (target_attributes);
1026 input_location = DECL_SOURCE_LOCATION (new_decl);
1027 bool r = targetm.target_option.valid_attribute_p (new_decl, NULL, v, 1);
1028 input_location = saved_loc;
1029 if (!r)
1030 return NULL;
1031 }
1032
1033 /* When the old decl was a con-/destructor make sure the clone isn't. */
1034 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
1035 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
1036 DECL_SET_IS_OPERATOR_NEW (new_decl, 0);
1037 DECL_SET_IS_OPERATOR_DELETE (new_decl, 0);
1038 DECL_IS_REPLACEABLE_OPERATOR (new_decl) = 0;
1039
1040 /* Create the new version's call-graph node.
1041 and update the edges of the new node. */
1042 new_version_node = create_version_clone (new_decl, redirect_callers,
1043 bbs_to_copy, suffix);
1044
1045 if (ipa_transforms_to_apply.exists ())
1046 new_version_node->ipa_transforms_to_apply
1047 = ipa_transforms_to_apply.copy ();
1048 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1049 tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments,
1050 false, bbs_to_copy, new_entry_block);
1051
1052 /* Update the new version's properties.
1053 Make The new version visible only within this translation unit. Make sure
1054 that is not weak also.
1055 ??? We cannot use COMDAT linkage because there is no
1056 ABI support for this. */
1057 new_version_node->make_decl_local ();
1058 DECL_VIRTUAL_P (new_version_node->decl) = 0;
1059 new_version_node->externally_visible = 0;
1060 new_version_node->local = 1;
1061 new_version_node->lowered = true;
1062 if (!implicit_section)
1063 new_version_node->set_section (*this);
1064 /* Clones of global symbols or symbols with unique names are unique. */
1065 if ((TREE_PUBLIC (old_decl)
1066 && !DECL_EXTERNAL (old_decl)
1067 && !DECL_WEAK (old_decl)
1068 && !DECL_COMDAT (old_decl))
1069 || in_lto_p)
1070 new_version_node->unique_name = true;
1071
1072 /* Update the call_expr on the edges to call the new version node. */
1073 update_call_expr (new_version_node);
1074
1075 symtab->call_cgraph_insertion_hooks (new_version_node);
1076 return new_version_node;
1077 }
1078
1079 /* Remove the node from the tree of virtual and inline clones and make it a
1080 standalone node - not a clone any more. */
1081
remove_from_clone_tree()1082 void cgraph_node::remove_from_clone_tree ()
1083 {
1084 if (next_sibling_clone)
1085 next_sibling_clone->prev_sibling_clone = prev_sibling_clone;
1086 if (prev_sibling_clone)
1087 prev_sibling_clone->next_sibling_clone = next_sibling_clone;
1088 else
1089 clone_of->clones = next_sibling_clone;
1090 next_sibling_clone = NULL;
1091 prev_sibling_clone = NULL;
1092 clone_of = NULL;
1093 }
1094
1095 /* Given virtual clone, turn it into actual clone. */
1096
1097 void
materialize_clone()1098 cgraph_node::materialize_clone ()
1099 {
1100 clone_info *info = clone_info::get (this);
1101 clone_of->get_untransformed_body ();
1102 former_clone_of = clone_of->decl;
1103 if (clone_of->former_clone_of)
1104 former_clone_of = clone_of->former_clone_of;
1105 if (symtab->dump_file)
1106 {
1107 fprintf (symtab->dump_file, "cloning %s to %s\n",
1108 clone_of->dump_name (),
1109 dump_name ());
1110 if (info && info->tree_map)
1111 {
1112 fprintf (symtab->dump_file, " replace map:");
1113 for (unsigned int i = 0;
1114 i < vec_safe_length (info->tree_map);
1115 i++)
1116 {
1117 ipa_replace_map *replace_info;
1118 replace_info = (*info->tree_map)[i];
1119 fprintf (symtab->dump_file, "%s %i -> ",
1120 i ? "," : "", replace_info->parm_num);
1121 print_generic_expr (symtab->dump_file,
1122 replace_info->new_tree);
1123 }
1124 fprintf (symtab->dump_file, "\n");
1125 }
1126 if (info && info->param_adjustments)
1127 info->param_adjustments->dump (symtab->dump_file);
1128 }
1129 clear_stmts_in_references ();
1130 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1131 tree_function_versioning (clone_of->decl, decl,
1132 info ? info->tree_map : NULL,
1133 info ? info->param_adjustments : NULL,
1134 true, NULL, NULL);
1135 if (symtab->dump_file)
1136 {
1137 dump_function_to_file (clone_of->decl, symtab->dump_file,
1138 dump_flags);
1139 dump_function_to_file (decl, symtab->dump_file, dump_flags);
1140 }
1141
1142 cgraph_node *this_clone_of = clone_of;
1143 /* Function is no longer clone. */
1144 remove_from_clone_tree ();
1145 if (!this_clone_of->analyzed && !this_clone_of->clones)
1146 this_clone_of->release_body ();
1147 }
1148
1149 #include "gt-cgraphclones.h"
1150