1 /* Calculate branch probabilities, and basic block execution counts.
2 Copyright (C) 1990-2021 Free Software Foundation, Inc.
3 Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
4 based on some ideas from Dain Samples of UC Berkeley.
5 Further mangling by Bob Manson, Cygnus Support.
6 Converted to use trees by Dale Johannesen, Apple Computer.
7
8 This file is part of GCC.
9
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
13 version.
14
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
23
24 /* Generate basic block profile instrumentation and auxiliary files.
25 Tree-based version. See profile.c for overview. */
26
27 #include "config.h"
28 #include "system.h"
29 #include "coretypes.h"
30 #include "memmodel.h"
31 #include "backend.h"
32 #include "target.h"
33 #include "tree.h"
34 #include "gimple.h"
35 #include "cfghooks.h"
36 #include "tree-pass.h"
37 #include "ssa.h"
38 #include "cgraph.h"
39 #include "coverage.h"
40 #include "diagnostic-core.h"
41 #include "fold-const.h"
42 #include "varasm.h"
43 #include "tree-nested.h"
44 #include "gimplify.h"
45 #include "gimple-iterator.h"
46 #include "gimplify-me.h"
47 #include "tree-cfg.h"
48 #include "tree-into-ssa.h"
49 #include "value-prof.h"
50 #include "profile.h"
51 #include "tree-cfgcleanup.h"
52 #include "stringpool.h"
53 #include "attribs.h"
54 #include "tree-pretty-print.h"
55 #include "langhooks.h"
56 #include "stor-layout.h"
57 #include "xregex.h"
58 #include "alloc-pool.h"
59 #include "symbol-summary.h"
60 #include "symtab-thunks.h"
61
62 static GTY(()) tree gcov_type_node;
63 static GTY(()) tree tree_interval_profiler_fn;
64 static GTY(()) tree tree_pow2_profiler_fn;
65 static GTY(()) tree tree_topn_values_profiler_fn;
66 static GTY(()) tree tree_indirect_call_profiler_fn;
67 static GTY(()) tree tree_average_profiler_fn;
68 static GTY(()) tree tree_ior_profiler_fn;
69 static GTY(()) tree tree_time_profiler_counter;
70
71
72 static GTY(()) tree ic_tuple_var;
73 static GTY(()) tree ic_tuple_counters_field;
74 static GTY(()) tree ic_tuple_callee_field;
75
76 /* Do initialization work for the edge profiler. */
77
78 /* Add code:
79 __thread gcov* __gcov_indirect_call.counters; // pointer to actual counter
80 __thread void* __gcov_indirect_call.callee; // actual callee address
81 __thread int __gcov_function_counter; // time profiler function counter
82 */
83 static void
init_ic_make_global_vars(void)84 init_ic_make_global_vars (void)
85 {
86 tree gcov_type_ptr;
87
88 gcov_type_ptr = build_pointer_type (get_gcov_type ());
89
90 tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
91
92 /* callee */
93 ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
94 ptr_type_node);
95
96 /* counters */
97 ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
98 NULL_TREE, gcov_type_ptr);
99 DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
100
101 finish_builtin_struct (tuple_type, "indirect_call_tuple",
102 ic_tuple_counters_field, NULL_TREE);
103
104 ic_tuple_var
105 = build_decl (UNKNOWN_LOCATION, VAR_DECL,
106 get_identifier ("__gcov_indirect_call"), tuple_type);
107 TREE_PUBLIC (ic_tuple_var) = 1;
108 DECL_ARTIFICIAL (ic_tuple_var) = 1;
109 DECL_INITIAL (ic_tuple_var) = NULL;
110 DECL_EXTERNAL (ic_tuple_var) = 1;
111 if (targetm.have_tls)
112 set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
113 }
114
115 /* Create the type and function decls for the interface with gcov. */
116
117 void
gimple_init_gcov_profiler(void)118 gimple_init_gcov_profiler (void)
119 {
120 tree interval_profiler_fn_type;
121 tree pow2_profiler_fn_type;
122 tree topn_values_profiler_fn_type;
123 tree gcov_type_ptr;
124 tree ic_profiler_fn_type;
125 tree average_profiler_fn_type;
126 const char *fn_name;
127
128 if (!gcov_type_node)
129 {
130 const char *fn_suffix
131 = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
132
133 gcov_type_node = get_gcov_type ();
134 gcov_type_ptr = build_pointer_type (gcov_type_node);
135
136 /* void (*) (gcov_type *, gcov_type, int, unsigned) */
137 interval_profiler_fn_type
138 = build_function_type_list (void_type_node,
139 gcov_type_ptr, gcov_type_node,
140 integer_type_node,
141 unsigned_type_node, NULL_TREE);
142 fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
143 tree_interval_profiler_fn = build_fn_decl (fn_name,
144 interval_profiler_fn_type);
145 free (CONST_CAST (char *, fn_name));
146 TREE_NOTHROW (tree_interval_profiler_fn) = 1;
147 DECL_ATTRIBUTES (tree_interval_profiler_fn)
148 = tree_cons (get_identifier ("leaf"), NULL,
149 DECL_ATTRIBUTES (tree_interval_profiler_fn));
150
151 /* void (*) (gcov_type *, gcov_type) */
152 pow2_profiler_fn_type
153 = build_function_type_list (void_type_node,
154 gcov_type_ptr, gcov_type_node,
155 NULL_TREE);
156 fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
157 tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
158 free (CONST_CAST (char *, fn_name));
159 TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
160 DECL_ATTRIBUTES (tree_pow2_profiler_fn)
161 = tree_cons (get_identifier ("leaf"), NULL,
162 DECL_ATTRIBUTES (tree_pow2_profiler_fn));
163
164 /* void (*) (gcov_type *, gcov_type) */
165 topn_values_profiler_fn_type
166 = build_function_type_list (void_type_node,
167 gcov_type_ptr, gcov_type_node,
168 NULL_TREE);
169 fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
170 tree_topn_values_profiler_fn
171 = build_fn_decl (fn_name, topn_values_profiler_fn_type);
172 free (CONST_CAST (char *, fn_name));
173
174 TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
175 DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
176 = tree_cons (get_identifier ("leaf"), NULL,
177 DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
178
179 init_ic_make_global_vars ();
180
181 /* void (*) (gcov_type, void *) */
182 ic_profiler_fn_type
183 = build_function_type_list (void_type_node,
184 gcov_type_node,
185 ptr_type_node,
186 NULL_TREE);
187 fn_name = concat ("__gcov_indirect_call_profiler_v4", fn_suffix, NULL);
188 tree_indirect_call_profiler_fn
189 = build_fn_decl (fn_name, ic_profiler_fn_type);
190 free (CONST_CAST (char *, fn_name));
191
192 TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
193 DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
194 = tree_cons (get_identifier ("leaf"), NULL,
195 DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
196
197 tree_time_profiler_counter
198 = build_decl (UNKNOWN_LOCATION, VAR_DECL,
199 get_identifier ("__gcov_time_profiler_counter"),
200 get_gcov_type ());
201 TREE_PUBLIC (tree_time_profiler_counter) = 1;
202 DECL_EXTERNAL (tree_time_profiler_counter) = 1;
203 TREE_STATIC (tree_time_profiler_counter) = 1;
204 DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
205 DECL_INITIAL (tree_time_profiler_counter) = NULL;
206
207 /* void (*) (gcov_type *, gcov_type) */
208 average_profiler_fn_type
209 = build_function_type_list (void_type_node,
210 gcov_type_ptr, gcov_type_node, NULL_TREE);
211 fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
212 tree_average_profiler_fn = build_fn_decl (fn_name,
213 average_profiler_fn_type);
214 free (CONST_CAST (char *, fn_name));
215 TREE_NOTHROW (tree_average_profiler_fn) = 1;
216 DECL_ATTRIBUTES (tree_average_profiler_fn)
217 = tree_cons (get_identifier ("leaf"), NULL,
218 DECL_ATTRIBUTES (tree_average_profiler_fn));
219 fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
220 tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
221 free (CONST_CAST (char *, fn_name));
222 TREE_NOTHROW (tree_ior_profiler_fn) = 1;
223 DECL_ATTRIBUTES (tree_ior_profiler_fn)
224 = tree_cons (get_identifier ("leaf"), NULL,
225 DECL_ATTRIBUTES (tree_ior_profiler_fn));
226
227 /* LTO streamer needs assembler names. Because we create these decls
228 late, we need to initialize them by hand. */
229 DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
230 DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
231 DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
232 DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
233 DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
234 DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
235 }
236 }
237
238 /* Output instructions as GIMPLE trees to increment the edge
239 execution count, and insert them on E. We rely on
240 gsi_insert_on_edge to preserve the order. */
241
242 void
gimple_gen_edge_profiler(int edgeno,edge e)243 gimple_gen_edge_profiler (int edgeno, edge e)
244 {
245 tree one;
246
247 one = build_int_cst (gcov_type_node, 1);
248
249 if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
250 {
251 /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
252 tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
253 tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
254 ? BUILT_IN_ATOMIC_FETCH_ADD_8:
255 BUILT_IN_ATOMIC_FETCH_ADD_4);
256 gcall *stmt = gimple_build_call (f, 3, addr, one,
257 build_int_cst (integer_type_node,
258 MEMMODEL_RELAXED));
259 gsi_insert_on_edge (e, stmt);
260 }
261 else
262 {
263 tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
264 tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
265 NULL, "PROF_edge_counter");
266 gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
267 gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
268 NULL, "PROF_edge_counter");
269 gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
270 gimple_assign_lhs (stmt1), one);
271 gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
272 gimple_assign_lhs (stmt2));
273 gsi_insert_on_edge (e, stmt1);
274 gsi_insert_on_edge (e, stmt2);
275 gsi_insert_on_edge (e, stmt3);
276 }
277 }
278
279 /* Emits code to get VALUE to instrument at GSI, and returns the
280 variable containing the value. */
281
282 static tree
prepare_instrumented_value(gimple_stmt_iterator * gsi,histogram_value value)283 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
284 {
285 tree val = value->hvalue.value;
286 if (POINTER_TYPE_P (TREE_TYPE (val)))
287 val = fold_convert (build_nonstandard_integer_type
288 (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
289 return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
290 true, NULL_TREE, true, GSI_SAME_STMT);
291 }
292
293 /* Output instructions as GIMPLE trees to increment the interval histogram
294 counter. VALUE is the expression whose value is profiled. TAG is the
295 tag of the section for counters, BASE is offset of the counter position. */
296
297 void
gimple_gen_interval_profiler(histogram_value value,unsigned tag)298 gimple_gen_interval_profiler (histogram_value value, unsigned tag)
299 {
300 gimple *stmt = value->hvalue.stmt;
301 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
302 tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr;
303 gcall *call;
304 tree val;
305 tree start = build_int_cst_type (integer_type_node,
306 value->hdata.intvl.int_start);
307 tree steps = build_int_cst_type (unsigned_type_node,
308 value->hdata.intvl.steps);
309
310 ref_ptr = force_gimple_operand_gsi (&gsi,
311 build_addr (ref),
312 true, NULL_TREE, true, GSI_SAME_STMT);
313 val = prepare_instrumented_value (&gsi, value);
314 call = gimple_build_call (tree_interval_profiler_fn, 4,
315 ref_ptr, val, start, steps);
316 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
317 }
318
319 /* Output instructions as GIMPLE trees to increment the power of two histogram
320 counter. VALUE is the expression whose value is profiled. TAG is the tag
321 of the section for counters. */
322
323 void
gimple_gen_pow2_profiler(histogram_value value,unsigned tag)324 gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
325 {
326 gimple *stmt = value->hvalue.stmt;
327 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
328 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
329 gcall *call;
330 tree val;
331
332 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
333 true, NULL_TREE, true, GSI_SAME_STMT);
334 val = prepare_instrumented_value (&gsi, value);
335 call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
336 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
337 }
338
339 /* Output instructions as GIMPLE trees for code to find the most N common
340 values. VALUE is the expression whose value is profiled. TAG is the tag
341 of the section for counters. */
342
343 void
gimple_gen_topn_values_profiler(histogram_value value,unsigned tag)344 gimple_gen_topn_values_profiler (histogram_value value, unsigned tag)
345 {
346 gimple *stmt = value->hvalue.stmt;
347 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
348 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
349 gcall *call;
350 tree val;
351
352 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
353 true, NULL_TREE, true, GSI_SAME_STMT);
354 val = prepare_instrumented_value (&gsi, value);
355 call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
356 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
357 }
358
359
360 /* Output instructions as GIMPLE trees for code to find the most
361 common called function in indirect call.
362 VALUE is the call expression whose indirect callee is profiled.
363 TAG is the tag of the section for counters. */
364
365 void
gimple_gen_ic_profiler(histogram_value value,unsigned tag)366 gimple_gen_ic_profiler (histogram_value value, unsigned tag)
367 {
368 tree tmp1;
369 gassign *stmt1, *stmt2, *stmt3;
370 gimple *stmt = value->hvalue.stmt;
371 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
372 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
373
374 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
375 true, NULL_TREE, true, GSI_SAME_STMT);
376
377 /* Insert code:
378
379 stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
380 stmt2: tmp1 = (void *) (indirect call argument value)
381 stmt3: __gcov_indirect_call.callee = tmp1;
382
383 Example:
384 f_1 = foo;
385 __gcov_indirect_call.counters = &__gcov4.main[0];
386 PROF_9 = f_1;
387 __gcov_indirect_call.callee = PROF_9;
388 _4 = f_1 ();
389 */
390
391 tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
392
393 tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
394 ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
395
396 stmt1 = gimple_build_assign (counter_ref, ref_ptr);
397 tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF");
398 stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
399 tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
400 ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
401 stmt3 = gimple_build_assign (callee_ref, tmp1);
402
403 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
404 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
405 gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
406 }
407
408
409 /* Output instructions as GIMPLE trees for code to find the most
410 common called function in indirect call. Insert instructions at the
411 beginning of every possible called function.
412 */
413
414 void
gimple_gen_ic_func_profiler(void)415 gimple_gen_ic_func_profiler (void)
416 {
417 struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
418 gcall *stmt1;
419 tree tree_uid, cur_func, void0;
420
421 if (c_node->only_called_directly_p ())
422 return;
423
424 gimple_init_gcov_profiler ();
425
426 basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
427 basic_block cond_bb = split_edge (single_succ_edge (entry));
428 basic_block update_bb = split_edge (single_succ_edge (cond_bb));
429
430 /* We need to do an extra split in order to not create an input
431 for a possible PHI node. */
432 split_edge (single_succ_edge (update_bb));
433
434 edge true_edge = single_succ_edge (cond_bb);
435 true_edge->flags = EDGE_TRUE_VALUE;
436
437 profile_probability probability;
438 if (DECL_VIRTUAL_P (current_function_decl))
439 probability = profile_probability::very_likely ();
440 else
441 probability = profile_probability::unlikely ();
442
443 true_edge->probability = probability;
444 edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
445 EDGE_FALSE_VALUE);
446 e->probability = true_edge->probability.invert ();
447
448 /* Insert code:
449
450 if (__gcov_indirect_call.callee != NULL)
451 __gcov_indirect_call_profiler_v3 (profile_id, ¤t_function_decl);
452
453 The function __gcov_indirect_call_profiler_v3 is responsible for
454 resetting __gcov_indirect_call.callee to NULL. */
455
456 gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
457 void0 = build_int_cst (ptr_type_node, 0);
458
459 tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
460 ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
461
462 tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
463 true, GSI_SAME_STMT);
464
465 gcond *cond = gimple_build_cond (NE_EXPR, ref,
466 void0, NULL, NULL);
467 gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
468
469 gsi = gsi_after_labels (update_bb);
470
471 cur_func = force_gimple_operand_gsi (&gsi,
472 build_addr (current_function_decl),
473 true, NULL_TREE,
474 true, GSI_SAME_STMT);
475 tree_uid = build_int_cst
476 (gcov_type_node,
477 cgraph_node::get (current_function_decl)->profile_id);
478 stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
479 tree_uid, cur_func);
480 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
481 }
482
483 /* Output instructions as GIMPLE tree at the beginning for each function.
484 TAG is the tag of the section for counters, BASE is offset of the
485 counter position and GSI is the iterator we place the counter. */
486
487 void
gimple_gen_time_profiler(unsigned tag)488 gimple_gen_time_profiler (unsigned tag)
489 {
490 tree type = get_gcov_type ();
491 basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
492 basic_block cond_bb = split_edge (single_succ_edge (entry));
493 basic_block update_bb = split_edge (single_succ_edge (cond_bb));
494
495 /* We need to do an extra split in order to not create an input
496 for a possible PHI node. */
497 split_edge (single_succ_edge (update_bb));
498
499 edge true_edge = single_succ_edge (cond_bb);
500 true_edge->flags = EDGE_TRUE_VALUE;
501 true_edge->probability = profile_probability::unlikely ();
502 edge e
503 = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
504 e->probability = true_edge->probability.invert ();
505
506 gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
507 tree original_ref = tree_coverage_counter_ref (tag, 0);
508 tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
509 true, GSI_SAME_STMT);
510 tree one = build_int_cst (type, 1);
511
512 /* Emit: if (counters[0] != 0). */
513 gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
514 NULL, NULL);
515 gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
516
517 gsi = gsi_start_bb (update_bb);
518
519 /* Emit: counters[0] = ++__gcov_time_profiler_counter. */
520 if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
521 {
522 tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
523 "time_profiler_counter_ptr");
524 tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
525 tree_time_profiler_counter);
526 gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
527 gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
528 tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
529 ? BUILT_IN_ATOMIC_ADD_FETCH_8:
530 BUILT_IN_ATOMIC_ADD_FETCH_4);
531 gcall *stmt = gimple_build_call (f, 3, ptr, one,
532 build_int_cst (integer_type_node,
533 MEMMODEL_RELAXED));
534 tree result_type = TREE_TYPE (TREE_TYPE (f));
535 tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
536 gimple_set_lhs (stmt, tmp);
537 gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
538 tmp = make_temp_ssa_name (type, NULL, "time_profile");
539 assign = gimple_build_assign (tmp, NOP_EXPR,
540 gimple_call_lhs (stmt));
541 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
542 assign = gimple_build_assign (original_ref, tmp);
543 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
544 }
545 else
546 {
547 tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
548 gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
549 gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
550
551 tmp = make_temp_ssa_name (type, NULL, "time_profile");
552 assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
553 one);
554 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
555 assign = gimple_build_assign (original_ref, tmp);
556 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
557 assign = gimple_build_assign (tree_time_profiler_counter, tmp);
558 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
559 }
560 }
561
562 /* Output instructions as GIMPLE trees to increment the average histogram
563 counter. VALUE is the expression whose value is profiled. TAG is the
564 tag of the section for counters, BASE is offset of the counter position. */
565
566 void
gimple_gen_average_profiler(histogram_value value,unsigned tag)567 gimple_gen_average_profiler (histogram_value value, unsigned tag)
568 {
569 gimple *stmt = value->hvalue.stmt;
570 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
571 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
572 gcall *call;
573 tree val;
574
575 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
576 true, NULL_TREE,
577 true, GSI_SAME_STMT);
578 val = prepare_instrumented_value (&gsi, value);
579 call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
580 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
581 }
582
583 /* Output instructions as GIMPLE trees to increment the ior histogram
584 counter. VALUE is the expression whose value is profiled. TAG is the
585 tag of the section for counters, BASE is offset of the counter position. */
586
587 void
gimple_gen_ior_profiler(histogram_value value,unsigned tag)588 gimple_gen_ior_profiler (histogram_value value, unsigned tag)
589 {
590 gimple *stmt = value->hvalue.stmt;
591 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
592 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
593 gcall *call;
594 tree val;
595
596 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
597 true, NULL_TREE, true, GSI_SAME_STMT);
598 val = prepare_instrumented_value (&gsi, value);
599 call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
600 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
601 }
602
603 static vec<regex_t> profile_filter_files;
604 static vec<regex_t> profile_exclude_files;
605
606 /* Parse list of provided REGEX (separated with semi-collon) and
607 create expressions (of type regex_t) and save them into V vector.
608 If there is a regular expression parsing error, error message is
609 printed for FLAG_NAME. */
610
611 static void
parse_profile_filter(const char * regex,vec<regex_t> * v,const char * flag_name)612 parse_profile_filter (const char *regex, vec<regex_t> *v,
613 const char *flag_name)
614 {
615 v->create (4);
616 if (regex != NULL)
617 {
618 char *str = xstrdup (regex);
619 for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
620 {
621 regex_t r;
622 if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
623 {
624 error ("invalid regular expression %qs in %qs",
625 p, flag_name);
626 return;
627 }
628
629 v->safe_push (r);
630 }
631 }
632 }
633
634 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files
635 options. */
636
637 static void
parse_profile_file_filtering()638 parse_profile_file_filtering ()
639 {
640 parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
641 "-fprofile-filter-files");
642 parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
643 "-fprofile-exclude-files");
644 }
645
646 /* Parse vectors of regular expressions. */
647
648 static void
release_profile_file_filtering()649 release_profile_file_filtering ()
650 {
651 profile_filter_files.release ();
652 profile_exclude_files.release ();
653 }
654
655 /* Return true when FILENAME should be instrumented based on
656 -fprofile-filter-files and -fprofile-exclude-files options. */
657
658 static bool
include_source_file_for_profile(const char * filename)659 include_source_file_for_profile (const char *filename)
660 {
661 /* First check whether file is included in flag_profile_exclude_files. */
662 for (unsigned i = 0; i < profile_exclude_files.length (); i++)
663 if (regexec (&profile_exclude_files[i],
664 filename, 0, NULL, 0) == REG_NOERROR)
665 return false;
666
667 /* For non-empty flag_profile_filter_files include only files matching a
668 regex in the flag. */
669 if (profile_filter_files.is_empty ())
670 return true;
671
672 for (unsigned i = 0; i < profile_filter_files.length (); i++)
673 if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
674 return true;
675
676 return false;
677 }
678
679 #ifndef HAVE_sync_compare_and_swapsi
680 #define HAVE_sync_compare_and_swapsi 0
681 #endif
682 #ifndef HAVE_atomic_compare_and_swapsi
683 #define HAVE_atomic_compare_and_swapsi 0
684 #endif
685
686 #ifndef HAVE_sync_compare_and_swapdi
687 #define HAVE_sync_compare_and_swapdi 0
688 #endif
689 #ifndef HAVE_atomic_compare_and_swapdi
690 #define HAVE_atomic_compare_and_swapdi 0
691 #endif
692
693 /* Profile all functions in the callgraph. */
694
695 static unsigned int
tree_profiling(void)696 tree_profiling (void)
697 {
698 struct cgraph_node *node;
699
700 /* Verify whether we can utilize atomic update operations. */
701 bool can_support_atomic = false;
702 unsigned HOST_WIDE_INT gcov_type_size
703 = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
704 if (gcov_type_size == 4)
705 can_support_atomic
706 = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
707 else if (gcov_type_size == 8)
708 can_support_atomic
709 = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
710
711 if (flag_profile_update == PROFILE_UPDATE_ATOMIC
712 && !can_support_atomic)
713 {
714 warning (0, "target does not support atomic profile update, "
715 "single mode is selected");
716 flag_profile_update = PROFILE_UPDATE_SINGLE;
717 }
718 else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
719 flag_profile_update = can_support_atomic
720 ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
721
722 /* This is a small-ipa pass that gets called only once, from
723 cgraphunit.c:ipa_passes(). */
724 gcc_assert (symtab->state == IPA_SSA);
725
726 init_node_map (true);
727 parse_profile_file_filtering ();
728
729 FOR_EACH_DEFINED_FUNCTION (node)
730 {
731 bool thunk = false;
732 if (!gimple_has_body_p (node->decl) && !node->thunk)
733 continue;
734
735 /* Don't profile functions produced for builtin stuff. */
736 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
737 continue;
738
739 if (lookup_attribute ("no_profile_instrument_function",
740 DECL_ATTRIBUTES (node->decl)))
741 continue;
742 /* Do not instrument extern inline functions when testing coverage.
743 While this is not perfectly consistent (early inlined extern inlines
744 will get acocunted), testsuite expects that. */
745 if (DECL_EXTERNAL (node->decl)
746 && flag_test_coverage)
747 continue;
748
749 const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
750 if (!include_source_file_for_profile (file))
751 continue;
752
753 if (node->thunk)
754 {
755 /* We cannot expand variadic thunks to Gimple. */
756 if (stdarg_p (TREE_TYPE (node->decl)))
757 continue;
758 thunk = true;
759 /* When generate profile, expand thunk to gimple so it can be
760 instrumented same way as other functions. */
761 if (profile_arc_flag)
762 expand_thunk (node, false, true);
763 /* Read cgraph profile but keep function as thunk at profile-use
764 time. */
765 else
766 {
767 read_thunk_profile (node);
768 continue;
769 }
770 }
771
772 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
773
774 if (dump_file)
775 dump_function_header (dump_file, cfun->decl, dump_flags);
776
777 /* Local pure-const may imply need to fixup the cfg. */
778 if (gimple_has_body_p (node->decl)
779 && (execute_fixup_cfg () & TODO_cleanup_cfg))
780 cleanup_tree_cfg ();
781
782 branch_prob (thunk);
783
784 if (! flag_branch_probabilities
785 && flag_profile_values)
786 gimple_gen_ic_func_profiler ();
787
788 if (flag_branch_probabilities
789 && !thunk
790 && flag_profile_values
791 && flag_value_profile_transformations
792 && profile_status_for_fn (cfun) == PROFILE_READ)
793 gimple_value_profile_transformations ();
794
795 /* The above could hose dominator info. Currently there is
796 none coming in, this is a safety valve. It should be
797 easy to adjust it, if and when there is some. */
798 free_dominance_info (CDI_DOMINATORS);
799 free_dominance_info (CDI_POST_DOMINATORS);
800 pop_cfun ();
801 }
802
803 release_profile_file_filtering ();
804
805 /* Drop pure/const flags from instrumented functions. */
806 if (profile_arc_flag || flag_test_coverage)
807 FOR_EACH_DEFINED_FUNCTION (node)
808 {
809 if (!gimple_has_body_p (node->decl)
810 || !(!node->clone_of
811 || node->decl != node->clone_of->decl))
812 continue;
813
814 /* Don't profile functions produced for builtin stuff. */
815 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
816 continue;
817
818 node->set_const_flag (false, false);
819 node->set_pure_flag (false, false);
820 }
821
822 /* Update call statements and rebuild the cgraph. */
823 FOR_EACH_DEFINED_FUNCTION (node)
824 {
825 basic_block bb;
826
827 if (!gimple_has_body_p (node->decl)
828 || !(!node->clone_of
829 || node->decl != node->clone_of->decl))
830 continue;
831
832 /* Don't profile functions produced for builtin stuff. */
833 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
834 continue;
835
836 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
837
838 FOR_EACH_BB_FN (bb, cfun)
839 {
840 gimple_stmt_iterator gsi;
841 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
842 {
843 gimple *stmt = gsi_stmt (gsi);
844 if (is_gimple_call (stmt))
845 update_stmt (stmt);
846 }
847 }
848
849 /* re-merge split blocks. */
850 cleanup_tree_cfg ();
851 update_ssa (TODO_update_ssa);
852
853 cgraph_edge::rebuild_edges ();
854
855 pop_cfun ();
856 }
857
858 handle_missing_profiles ();
859
860 del_node_map ();
861 return 0;
862 }
863
864 namespace {
865
866 const pass_data pass_data_ipa_tree_profile =
867 {
868 SIMPLE_IPA_PASS, /* type */
869 "profile", /* name */
870 OPTGROUP_NONE, /* optinfo_flags */
871 TV_IPA_PROFILE, /* tv_id */
872 0, /* properties_required */
873 0, /* properties_provided */
874 0, /* properties_destroyed */
875 0, /* todo_flags_start */
876 TODO_dump_symtab, /* todo_flags_finish */
877 };
878
879 class pass_ipa_tree_profile : public simple_ipa_opt_pass
880 {
881 public:
pass_ipa_tree_profile(gcc::context * ctxt)882 pass_ipa_tree_profile (gcc::context *ctxt)
883 : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
884 {}
885
886 /* opt_pass methods: */
887 virtual bool gate (function *);
execute(function *)888 virtual unsigned int execute (function *) { return tree_profiling (); }
889
890 }; // class pass_ipa_tree_profile
891
892 bool
gate(function *)893 pass_ipa_tree_profile::gate (function *)
894 {
895 /* When profile instrumentation, use or test coverage shall be performed.
896 But for AutoFDO, this there is no instrumentation, thus this pass is
897 disabled. */
898 return (!in_lto_p && !flag_auto_profile
899 && (flag_branch_probabilities || flag_test_coverage
900 || profile_arc_flag));
901 }
902
903 } // anon namespace
904
905 simple_ipa_opt_pass *
make_pass_ipa_tree_profile(gcc::context * ctxt)906 make_pass_ipa_tree_profile (gcc::context *ctxt)
907 {
908 return new pass_ipa_tree_profile (ctxt);
909 }
910
911 #include "gt-tree-profile.h"
912