1 | /* Calculate branch probabilities, and basic block execution counts. |
2 | Copyright (C) 1990-2023 Free Software Foundation, Inc. |
3 | Contributed by James E. Wilson, UC Berkeley/Cygnus Support; |
4 | based on some ideas from Dain Samples of UC Berkeley. |
5 | Further mangling by Bob Manson, Cygnus Support. |
6 | Converted to use trees by Dale Johannesen, Apple Computer. |
7 | |
8 | This file is part of GCC. |
9 | |
10 | GCC is free software; you can redistribute it and/or modify it under |
11 | the terms of the GNU General Public License as published by the Free |
12 | Software Foundation; either version 3, or (at your option) any later |
13 | version. |
14 | |
15 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
16 | WARRANTY; without even the implied warranty of MERCHANTABILITY or |
17 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
18 | for more details. |
19 | |
20 | You should have received a copy of the GNU General Public License |
21 | along with GCC; see the file COPYING3. If not see |
22 | <http://www.gnu.org/licenses/>. */ |
23 | |
24 | /* Generate basic block profile instrumentation and auxiliary files. |
25 | Tree-based version. See profile.cc for overview. */ |
26 | |
27 | #include "config.h" |
28 | #include "system.h" |
29 | #include "coretypes.h" |
30 | #include "memmodel.h" |
31 | #include "backend.h" |
32 | #include "target.h" |
33 | #include "tree.h" |
34 | #include "gimple.h" |
35 | #include "cfghooks.h" |
36 | #include "tree-pass.h" |
37 | #include "ssa.h" |
38 | #include "cgraph.h" |
39 | #include "coverage.h" |
40 | #include "diagnostic-core.h" |
41 | #include "fold-const.h" |
42 | #include "varasm.h" |
43 | #include "tree-nested.h" |
44 | #include "gimplify.h" |
45 | #include "gimple-iterator.h" |
46 | #include "gimplify-me.h" |
47 | #include "tree-cfg.h" |
48 | #include "tree-into-ssa.h" |
49 | #include "value-prof.h" |
50 | #include "profile.h" |
51 | #include "tree-cfgcleanup.h" |
52 | #include "stringpool.h" |
53 | #include "attribs.h" |
54 | #include "tree-pretty-print.h" |
55 | #include "langhooks.h" |
56 | #include "stor-layout.h" |
57 | #include "xregex.h" |
58 | #include "alloc-pool.h" |
59 | #include "symbol-summary.h" |
60 | #include "symtab-thunks.h" |
61 | |
62 | static GTY(()) tree gcov_type_node; |
63 | static GTY(()) tree tree_interval_profiler_fn; |
64 | static GTY(()) tree tree_pow2_profiler_fn; |
65 | static GTY(()) tree tree_topn_values_profiler_fn; |
66 | static GTY(()) tree tree_indirect_call_profiler_fn; |
67 | static GTY(()) tree tree_average_profiler_fn; |
68 | static GTY(()) tree tree_ior_profiler_fn; |
69 | static GTY(()) tree tree_time_profiler_counter; |
70 | |
71 | |
72 | static GTY(()) tree ic_tuple_var; |
73 | static GTY(()) tree ic_tuple_counters_field; |
74 | static GTY(()) tree ic_tuple_callee_field; |
75 | |
76 | /* Do initialization work for the edge profiler. */ |
77 | |
78 | /* Add code: |
79 | __thread gcov* __gcov_indirect_call.counters; // pointer to actual counter |
80 | __thread void* __gcov_indirect_call.callee; // actual callee address |
81 | __thread int __gcov_function_counter; // time profiler function counter |
82 | */ |
83 | static void |
84 | init_ic_make_global_vars (void) |
85 | { |
86 | tree gcov_type_ptr; |
87 | |
88 | gcov_type_ptr = build_pointer_type (get_gcov_type ()); |
89 | |
90 | tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE); |
91 | |
92 | /* callee */ |
93 | ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE, |
94 | ptr_type_node); |
95 | |
96 | /* counters */ |
97 | ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, |
98 | NULL_TREE, gcov_type_ptr); |
99 | DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field; |
100 | |
101 | finish_builtin_struct (tuple_type, "indirect_call_tuple" , |
102 | ic_tuple_counters_field, NULL_TREE); |
103 | |
104 | ic_tuple_var |
105 | = build_decl (UNKNOWN_LOCATION, VAR_DECL, |
106 | get_identifier ("__gcov_indirect_call" ), tuple_type); |
107 | TREE_PUBLIC (ic_tuple_var) = 1; |
108 | DECL_ARTIFICIAL (ic_tuple_var) = 1; |
109 | DECL_INITIAL (ic_tuple_var) = NULL; |
110 | DECL_EXTERNAL (ic_tuple_var) = 1; |
111 | if (targetm.have_tls) |
112 | set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var)); |
113 | } |
114 | |
115 | /* Create the type and function decls for the interface with gcov. */ |
116 | |
117 | void |
118 | gimple_init_gcov_profiler (void) |
119 | { |
120 | tree interval_profiler_fn_type; |
121 | tree pow2_profiler_fn_type; |
122 | tree topn_values_profiler_fn_type; |
123 | tree gcov_type_ptr; |
124 | tree ic_profiler_fn_type; |
125 | tree average_profiler_fn_type; |
126 | const char *fn_name; |
127 | |
128 | if (!gcov_type_node) |
129 | { |
130 | const char *fn_suffix |
131 | = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "" ; |
132 | |
133 | gcov_type_node = get_gcov_type (); |
134 | gcov_type_ptr = build_pointer_type (gcov_type_node); |
135 | |
136 | /* void (*) (gcov_type *, gcov_type, int, unsigned) */ |
137 | interval_profiler_fn_type |
138 | = build_function_type_list (void_type_node, |
139 | gcov_type_ptr, gcov_type_node, |
140 | integer_type_node, |
141 | unsigned_type_node, NULL_TREE); |
142 | fn_name = concat ("__gcov_interval_profiler" , fn_suffix, NULL); |
143 | tree_interval_profiler_fn = build_fn_decl (fn_name, |
144 | interval_profiler_fn_type); |
145 | free (CONST_CAST (char *, fn_name)); |
146 | TREE_NOTHROW (tree_interval_profiler_fn) = 1; |
147 | DECL_ATTRIBUTES (tree_interval_profiler_fn) |
148 | = tree_cons (get_identifier ("leaf" ), NULL, |
149 | DECL_ATTRIBUTES (tree_interval_profiler_fn)); |
150 | |
151 | /* void (*) (gcov_type *, gcov_type) */ |
152 | pow2_profiler_fn_type |
153 | = build_function_type_list (void_type_node, |
154 | gcov_type_ptr, gcov_type_node, |
155 | NULL_TREE); |
156 | fn_name = concat ("__gcov_pow2_profiler" , fn_suffix, NULL); |
157 | tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type); |
158 | free (CONST_CAST (char *, fn_name)); |
159 | TREE_NOTHROW (tree_pow2_profiler_fn) = 1; |
160 | DECL_ATTRIBUTES (tree_pow2_profiler_fn) |
161 | = tree_cons (get_identifier ("leaf" ), NULL, |
162 | DECL_ATTRIBUTES (tree_pow2_profiler_fn)); |
163 | |
164 | /* void (*) (gcov_type *, gcov_type) */ |
165 | topn_values_profiler_fn_type |
166 | = build_function_type_list (void_type_node, |
167 | gcov_type_ptr, gcov_type_node, |
168 | NULL_TREE); |
169 | fn_name = concat ("__gcov_topn_values_profiler" , fn_suffix, NULL); |
170 | tree_topn_values_profiler_fn |
171 | = build_fn_decl (fn_name, topn_values_profiler_fn_type); |
172 | free (CONST_CAST (char *, fn_name)); |
173 | |
174 | TREE_NOTHROW (tree_topn_values_profiler_fn) = 1; |
175 | DECL_ATTRIBUTES (tree_topn_values_profiler_fn) |
176 | = tree_cons (get_identifier ("leaf" ), NULL, |
177 | DECL_ATTRIBUTES (tree_topn_values_profiler_fn)); |
178 | |
179 | init_ic_make_global_vars (); |
180 | |
181 | /* void (*) (gcov_type, void *) */ |
182 | ic_profiler_fn_type |
183 | = build_function_type_list (void_type_node, |
184 | gcov_type_node, |
185 | ptr_type_node, |
186 | NULL_TREE); |
187 | fn_name = concat ("__gcov_indirect_call_profiler_v4" , fn_suffix, NULL); |
188 | tree_indirect_call_profiler_fn |
189 | = build_fn_decl (fn_name, ic_profiler_fn_type); |
190 | free (CONST_CAST (char *, fn_name)); |
191 | |
192 | TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1; |
193 | DECL_ATTRIBUTES (tree_indirect_call_profiler_fn) |
194 | = tree_cons (get_identifier ("leaf" ), NULL, |
195 | DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)); |
196 | |
197 | tree_time_profiler_counter |
198 | = build_decl (UNKNOWN_LOCATION, VAR_DECL, |
199 | get_identifier ("__gcov_time_profiler_counter" ), |
200 | get_gcov_type ()); |
201 | TREE_PUBLIC (tree_time_profiler_counter) = 1; |
202 | DECL_EXTERNAL (tree_time_profiler_counter) = 1; |
203 | TREE_STATIC (tree_time_profiler_counter) = 1; |
204 | DECL_ARTIFICIAL (tree_time_profiler_counter) = 1; |
205 | DECL_INITIAL (tree_time_profiler_counter) = NULL; |
206 | |
207 | /* void (*) (gcov_type *, gcov_type) */ |
208 | average_profiler_fn_type |
209 | = build_function_type_list (void_type_node, |
210 | gcov_type_ptr, gcov_type_node, NULL_TREE); |
211 | fn_name = concat ("__gcov_average_profiler" , fn_suffix, NULL); |
212 | tree_average_profiler_fn = build_fn_decl (fn_name, |
213 | average_profiler_fn_type); |
214 | free (CONST_CAST (char *, fn_name)); |
215 | TREE_NOTHROW (tree_average_profiler_fn) = 1; |
216 | DECL_ATTRIBUTES (tree_average_profiler_fn) |
217 | = tree_cons (get_identifier ("leaf" ), NULL, |
218 | DECL_ATTRIBUTES (tree_average_profiler_fn)); |
219 | fn_name = concat ("__gcov_ior_profiler" , fn_suffix, NULL); |
220 | tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type); |
221 | free (CONST_CAST (char *, fn_name)); |
222 | TREE_NOTHROW (tree_ior_profiler_fn) = 1; |
223 | DECL_ATTRIBUTES (tree_ior_profiler_fn) |
224 | = tree_cons (get_identifier ("leaf" ), NULL, |
225 | DECL_ATTRIBUTES (tree_ior_profiler_fn)); |
226 | |
227 | /* LTO streamer needs assembler names. Because we create these decls |
228 | late, we need to initialize them by hand. */ |
229 | DECL_ASSEMBLER_NAME (tree_interval_profiler_fn); |
230 | DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn); |
231 | DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn); |
232 | DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn); |
233 | DECL_ASSEMBLER_NAME (tree_average_profiler_fn); |
234 | DECL_ASSEMBLER_NAME (tree_ior_profiler_fn); |
235 | } |
236 | } |
237 | |
238 | /* Output instructions as GIMPLE trees to increment the edge |
239 | execution count, and insert them on E. We rely on |
240 | gsi_insert_on_edge to preserve the order. */ |
241 | |
242 | void |
243 | gimple_gen_edge_profiler (int edgeno, edge e) |
244 | { |
245 | tree one; |
246 | |
247 | one = build_int_cst (gcov_type_node, 1); |
248 | |
249 | if (flag_profile_update == PROFILE_UPDATE_ATOMIC) |
250 | { |
251 | /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */ |
252 | tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno); |
253 | tree f = builtin_decl_explicit (TYPE_PRECISION (gcov_type_node) > 32 |
254 | ? BUILT_IN_ATOMIC_FETCH_ADD_8: |
255 | BUILT_IN_ATOMIC_FETCH_ADD_4); |
256 | gcall *stmt = gimple_build_call (f, 3, addr, one, |
257 | build_int_cst (integer_type_node, |
258 | MEMMODEL_RELAXED)); |
259 | gsi_insert_on_edge (e, stmt); |
260 | } |
261 | else |
262 | { |
263 | tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno); |
264 | tree gcov_type_tmp_var = make_temp_ssa_name (type: gcov_type_node, |
265 | NULL, name: "PROF_edge_counter" ); |
266 | gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref); |
267 | gcov_type_tmp_var = make_temp_ssa_name (type: gcov_type_node, |
268 | NULL, name: "PROF_edge_counter" ); |
269 | gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR, |
270 | gimple_assign_lhs (gs: stmt1), one); |
271 | gassign *stmt3 = gimple_build_assign (unshare_expr (ref), |
272 | gimple_assign_lhs (gs: stmt2)); |
273 | gsi_insert_on_edge (e, stmt1); |
274 | gsi_insert_on_edge (e, stmt2); |
275 | gsi_insert_on_edge (e, stmt3); |
276 | } |
277 | } |
278 | |
279 | /* Emits code to get VALUE to instrument at GSI, and returns the |
280 | variable containing the value. */ |
281 | |
282 | static tree |
283 | prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value) |
284 | { |
285 | tree val = value->hvalue.value; |
286 | if (POINTER_TYPE_P (TREE_TYPE (val))) |
287 | val = fold_convert (build_nonstandard_integer_type |
288 | (TYPE_PRECISION (TREE_TYPE (val)), 1), val); |
289 | return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val), |
290 | true, NULL_TREE, true, GSI_SAME_STMT); |
291 | } |
292 | |
293 | /* Output instructions as GIMPLE trees to increment the interval histogram |
294 | counter. VALUE is the expression whose value is profiled. TAG is the |
295 | tag of the section for counters, BASE is offset of the counter position. */ |
296 | |
297 | void |
298 | gimple_gen_interval_profiler (histogram_value value, unsigned tag) |
299 | { |
300 | gimple *stmt = value->hvalue.stmt; |
301 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
302 | tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr; |
303 | gcall *call; |
304 | tree val; |
305 | tree start = build_int_cst_type (integer_type_node, |
306 | value->hdata.intvl.int_start); |
307 | tree steps = build_int_cst_type (unsigned_type_node, |
308 | value->hdata.intvl.steps); |
309 | |
310 | ref_ptr = force_gimple_operand_gsi (&gsi, |
311 | build_addr (ref), |
312 | true, NULL_TREE, true, GSI_SAME_STMT); |
313 | val = prepare_instrumented_value (gsi: &gsi, value); |
314 | call = gimple_build_call (tree_interval_profiler_fn, 4, |
315 | ref_ptr, val, start, steps); |
316 | gsi_insert_before (&gsi, call, GSI_NEW_STMT); |
317 | } |
318 | |
319 | /* Output instructions as GIMPLE trees to increment the power of two histogram |
320 | counter. VALUE is the expression whose value is profiled. TAG is the tag |
321 | of the section for counters. */ |
322 | |
323 | void |
324 | gimple_gen_pow2_profiler (histogram_value value, unsigned tag) |
325 | { |
326 | gimple *stmt = value->hvalue.stmt; |
327 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
328 | tree ref_ptr = tree_coverage_counter_addr (tag, 0); |
329 | gcall *call; |
330 | tree val; |
331 | |
332 | ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, |
333 | true, NULL_TREE, true, GSI_SAME_STMT); |
334 | val = prepare_instrumented_value (gsi: &gsi, value); |
335 | call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val); |
336 | gsi_insert_before (&gsi, call, GSI_NEW_STMT); |
337 | } |
338 | |
339 | /* Output instructions as GIMPLE trees for code to find the most N common |
340 | values. VALUE is the expression whose value is profiled. TAG is the tag |
341 | of the section for counters. */ |
342 | |
343 | void |
344 | gimple_gen_topn_values_profiler (histogram_value value, unsigned tag) |
345 | { |
346 | gimple *stmt = value->hvalue.stmt; |
347 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
348 | tree ref_ptr = tree_coverage_counter_addr (tag, 0); |
349 | gcall *call; |
350 | tree val; |
351 | |
352 | ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, |
353 | true, NULL_TREE, true, GSI_SAME_STMT); |
354 | val = prepare_instrumented_value (gsi: &gsi, value); |
355 | call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val); |
356 | gsi_insert_before (&gsi, call, GSI_NEW_STMT); |
357 | } |
358 | |
359 | |
360 | /* Output instructions as GIMPLE trees for code to find the most |
361 | common called function in indirect call. |
362 | VALUE is the call expression whose indirect callee is profiled. |
363 | TAG is the tag of the section for counters. */ |
364 | |
365 | void |
366 | gimple_gen_ic_profiler (histogram_value value, unsigned tag) |
367 | { |
368 | tree tmp1; |
369 | gassign *stmt1, *stmt2, *stmt3; |
370 | gimple *stmt = value->hvalue.stmt; |
371 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
372 | tree ref_ptr = tree_coverage_counter_addr (tag, 0); |
373 | |
374 | ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, |
375 | true, NULL_TREE, true, GSI_SAME_STMT); |
376 | |
377 | /* Insert code: |
378 | |
379 | stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr (); |
380 | stmt2: tmp1 = (void *) (indirect call argument value) |
381 | stmt3: __gcov_indirect_call.callee = tmp1; |
382 | |
383 | Example: |
384 | f_1 = foo; |
385 | __gcov_indirect_call.counters = &__gcov4.main[0]; |
386 | PROF_fn_9 = f_1; |
387 | __gcov_indirect_call.callee = PROF_fn_9; |
388 | _4 = f_1 (); |
389 | */ |
390 | |
391 | tree gcov_type_ptr = build_pointer_type (get_gcov_type ()); |
392 | |
393 | tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr, |
394 | ic_tuple_var, ic_tuple_counters_field, NULL_TREE); |
395 | |
396 | stmt1 = gimple_build_assign (counter_ref, ref_ptr); |
397 | tmp1 = make_temp_ssa_name (ptr_type_node, NULL, name: "PROF_fn" ); |
398 | stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value)); |
399 | tree callee_ref = build3 (COMPONENT_REF, ptr_type_node, |
400 | ic_tuple_var, ic_tuple_callee_field, NULL_TREE); |
401 | stmt3 = gimple_build_assign (callee_ref, tmp1); |
402 | |
403 | gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT); |
404 | gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT); |
405 | gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT); |
406 | } |
407 | |
408 | |
409 | /* Output instructions as GIMPLE trees for code to find the most |
410 | common called function in indirect call. Insert instructions at the |
411 | beginning of every possible called function. |
412 | */ |
413 | |
414 | void |
415 | gimple_gen_ic_func_profiler (void) |
416 | { |
417 | struct cgraph_node * c_node = cgraph_node::get (decl: current_function_decl); |
418 | gcall *stmt1; |
419 | tree tree_uid, cur_func, void0; |
420 | |
421 | if (c_node->only_called_directly_p ()) |
422 | return; |
423 | |
424 | gimple_init_gcov_profiler (); |
425 | |
426 | basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun); |
427 | basic_block cond_bb = split_edge (single_succ_edge (bb: entry)); |
428 | basic_block update_bb = split_edge (single_succ_edge (bb: cond_bb)); |
429 | |
430 | /* We need to do an extra split in order to not create an input |
431 | for a possible PHI node. */ |
432 | split_edge (single_succ_edge (bb: update_bb)); |
433 | |
434 | edge true_edge = single_succ_edge (bb: cond_bb); |
435 | true_edge->flags = EDGE_TRUE_VALUE; |
436 | |
437 | profile_probability probability; |
438 | if (DECL_VIRTUAL_P (current_function_decl)) |
439 | probability = profile_probability::very_likely (); |
440 | else |
441 | probability = profile_probability::unlikely (); |
442 | |
443 | true_edge->probability = probability; |
444 | edge e = make_edge (cond_bb, single_succ_edge (bb: update_bb)->dest, |
445 | EDGE_FALSE_VALUE); |
446 | e->probability = true_edge->probability.invert (); |
447 | |
448 | /* Insert code: |
449 | |
450 | if (__gcov_indirect_call.callee != NULL) |
451 | __gcov_indirect_call_profiler_v3 (profile_id, ¤t_function_decl); |
452 | |
453 | The function __gcov_indirect_call_profiler_v3 is responsible for |
454 | resetting __gcov_indirect_call.callee to NULL. */ |
455 | |
456 | gimple_stmt_iterator gsi = gsi_start_bb (bb: cond_bb); |
457 | void0 = build_int_cst (ptr_type_node, 0); |
458 | |
459 | tree callee_ref = build3 (COMPONENT_REF, ptr_type_node, |
460 | ic_tuple_var, ic_tuple_callee_field, NULL_TREE); |
461 | |
462 | tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE, |
463 | true, GSI_SAME_STMT); |
464 | |
465 | gcond *cond = gimple_build_cond (NE_EXPR, ref, |
466 | void0, NULL, NULL); |
467 | gsi_insert_before (&gsi, cond, GSI_NEW_STMT); |
468 | |
469 | gsi = gsi_after_labels (bb: update_bb); |
470 | |
471 | cur_func = force_gimple_operand_gsi (&gsi, |
472 | build_addr (current_function_decl), |
473 | true, NULL_TREE, |
474 | true, GSI_SAME_STMT); |
475 | tree_uid = build_int_cst |
476 | (gcov_type_node, |
477 | cgraph_node::get (decl: current_function_decl)->profile_id); |
478 | stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2, |
479 | tree_uid, cur_func); |
480 | gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT); |
481 | } |
482 | |
483 | /* Output instructions as GIMPLE tree at the beginning for each function. |
484 | TAG is the tag of the section for counters, BASE is offset of the |
485 | counter position and GSI is the iterator we place the counter. */ |
486 | |
487 | void |
488 | gimple_gen_time_profiler (unsigned tag) |
489 | { |
490 | tree type = get_gcov_type (); |
491 | basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun); |
492 | basic_block cond_bb = split_edge (single_succ_edge (bb: entry)); |
493 | basic_block update_bb = split_edge (single_succ_edge (bb: cond_bb)); |
494 | |
495 | /* We need to do an extra split in order to not create an input |
496 | for a possible PHI node. */ |
497 | split_edge (single_succ_edge (bb: update_bb)); |
498 | |
499 | edge true_edge = single_succ_edge (bb: cond_bb); |
500 | true_edge->flags = EDGE_TRUE_VALUE; |
501 | true_edge->probability = profile_probability::unlikely (); |
502 | edge e |
503 | = make_edge (cond_bb, single_succ_edge (bb: update_bb)->dest, EDGE_FALSE_VALUE); |
504 | e->probability = true_edge->probability.invert (); |
505 | |
506 | gimple_stmt_iterator gsi = gsi_start_bb (bb: cond_bb); |
507 | tree original_ref = tree_coverage_counter_ref (tag, 0); |
508 | tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE, |
509 | true, GSI_SAME_STMT); |
510 | tree one = build_int_cst (type, 1); |
511 | |
512 | /* Emit: if (counters[0] != 0). */ |
513 | gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0), |
514 | NULL, NULL); |
515 | gsi_insert_before (&gsi, cond, GSI_NEW_STMT); |
516 | |
517 | gsi = gsi_start_bb (bb: update_bb); |
518 | |
519 | /* Emit: counters[0] = ++__gcov_time_profiler_counter. */ |
520 | if (flag_profile_update == PROFILE_UPDATE_ATOMIC) |
521 | { |
522 | tree ptr = make_temp_ssa_name (type: build_pointer_type (type), NULL, |
523 | name: "PROF_time_profiler_counter_ptr" ); |
524 | tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr), |
525 | tree_time_profiler_counter); |
526 | gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr); |
527 | gsi_insert_before (&gsi, assign, GSI_NEW_STMT); |
528 | tree f = builtin_decl_explicit (TYPE_PRECISION (gcov_type_node) > 32 |
529 | ? BUILT_IN_ATOMIC_ADD_FETCH_8: |
530 | BUILT_IN_ATOMIC_ADD_FETCH_4); |
531 | gcall *stmt = gimple_build_call (f, 3, ptr, one, |
532 | build_int_cst (integer_type_node, |
533 | MEMMODEL_RELAXED)); |
534 | tree result_type = TREE_TYPE (TREE_TYPE (f)); |
535 | tree tmp = make_temp_ssa_name (type: result_type, NULL, name: "PROF_time_profile" ); |
536 | gimple_set_lhs (stmt, tmp); |
537 | gsi_insert_after (&gsi, stmt, GSI_NEW_STMT); |
538 | tmp = make_temp_ssa_name (type, NULL, name: "PROF_time_profile" ); |
539 | assign = gimple_build_assign (tmp, NOP_EXPR, |
540 | gimple_call_lhs (gs: stmt)); |
541 | gsi_insert_after (&gsi, assign, GSI_NEW_STMT); |
542 | assign = gimple_build_assign (original_ref, tmp); |
543 | gsi_insert_after (&gsi, assign, GSI_NEW_STMT); |
544 | } |
545 | else |
546 | { |
547 | tree tmp = make_temp_ssa_name (type, NULL, name: "PROF_time_profile" ); |
548 | gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter); |
549 | gsi_insert_before (&gsi, assign, GSI_NEW_STMT); |
550 | |
551 | tmp = make_temp_ssa_name (type, NULL, name: "PROF_time_profile" ); |
552 | assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (gs: assign), |
553 | one); |
554 | gsi_insert_after (&gsi, assign, GSI_NEW_STMT); |
555 | assign = gimple_build_assign (original_ref, tmp); |
556 | gsi_insert_after (&gsi, assign, GSI_NEW_STMT); |
557 | assign = gimple_build_assign (tree_time_profiler_counter, tmp); |
558 | gsi_insert_after (&gsi, assign, GSI_NEW_STMT); |
559 | } |
560 | } |
561 | |
562 | /* Output instructions as GIMPLE trees to increment the average histogram |
563 | counter. VALUE is the expression whose value is profiled. TAG is the |
564 | tag of the section for counters, BASE is offset of the counter position. */ |
565 | |
566 | void |
567 | gimple_gen_average_profiler (histogram_value value, unsigned tag) |
568 | { |
569 | gimple *stmt = value->hvalue.stmt; |
570 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
571 | tree ref_ptr = tree_coverage_counter_addr (tag, 0); |
572 | gcall *call; |
573 | tree val; |
574 | |
575 | ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, |
576 | true, NULL_TREE, |
577 | true, GSI_SAME_STMT); |
578 | val = prepare_instrumented_value (gsi: &gsi, value); |
579 | call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val); |
580 | gsi_insert_before (&gsi, call, GSI_NEW_STMT); |
581 | } |
582 | |
583 | /* Output instructions as GIMPLE trees to increment the ior histogram |
584 | counter. VALUE is the expression whose value is profiled. TAG is the |
585 | tag of the section for counters, BASE is offset of the counter position. */ |
586 | |
587 | void |
588 | gimple_gen_ior_profiler (histogram_value value, unsigned tag) |
589 | { |
590 | gimple *stmt = value->hvalue.stmt; |
591 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
592 | tree ref_ptr = tree_coverage_counter_addr (tag, 0); |
593 | gcall *call; |
594 | tree val; |
595 | |
596 | ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, |
597 | true, NULL_TREE, true, GSI_SAME_STMT); |
598 | val = prepare_instrumented_value (gsi: &gsi, value); |
599 | call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val); |
600 | gsi_insert_before (&gsi, call, GSI_NEW_STMT); |
601 | } |
602 | |
603 | static vec<regex_t> profile_filter_files; |
604 | static vec<regex_t> profile_exclude_files; |
605 | |
606 | /* Parse list of provided REGEX (separated with semi-collon) and |
607 | create expressions (of type regex_t) and save them into V vector. |
608 | If there is a regular expression parsing error, error message is |
609 | printed for FLAG_NAME. */ |
610 | |
611 | static void |
612 | parse_profile_filter (const char *regex, vec<regex_t> *v, |
613 | const char *flag_name) |
614 | { |
615 | v->create (nelems: 4); |
616 | if (regex != NULL) |
617 | { |
618 | char *str = xstrdup (regex); |
619 | for (char *p = strtok (s: str, delim: ";" ); p != NULL; p = strtok (NULL, delim: ";" )) |
620 | { |
621 | regex_t r; |
622 | if (regcomp (preg: &r, pattern: p, REG_EXTENDED | REG_NOSUB) != 0) |
623 | { |
624 | error ("invalid regular expression %qs in %qs" , |
625 | p, flag_name); |
626 | return; |
627 | } |
628 | |
629 | v->safe_push (obj: r); |
630 | } |
631 | } |
632 | } |
633 | |
634 | /* Parse values of -fprofile-filter-files and -fprofile-exclude-files |
635 | options. */ |
636 | |
637 | static void |
638 | parse_profile_file_filtering () |
639 | { |
640 | parse_profile_filter (flag_profile_filter_files, v: &profile_filter_files, |
641 | flag_name: "-fprofile-filter-files" ); |
642 | parse_profile_filter (flag_profile_exclude_files, v: &profile_exclude_files, |
643 | flag_name: "-fprofile-exclude-files" ); |
644 | } |
645 | |
646 | /* Parse vectors of regular expressions. */ |
647 | |
648 | static void |
649 | release_profile_file_filtering () |
650 | { |
651 | profile_filter_files.release (); |
652 | profile_exclude_files.release (); |
653 | } |
654 | |
655 | /* Return true when FILENAME should be instrumented based on |
656 | -fprofile-filter-files and -fprofile-exclude-files options. */ |
657 | |
658 | static bool |
659 | include_source_file_for_profile (const char *filename) |
660 | { |
661 | /* First check whether file is included in flag_profile_exclude_files. */ |
662 | for (unsigned i = 0; i < profile_exclude_files.length (); i++) |
663 | if (regexec (preg: &profile_exclude_files[i], |
664 | string: filename, nmatch: 0, NULL, eflags: 0) == REG_NOERROR) |
665 | return false; |
666 | |
667 | /* For non-empty flag_profile_filter_files include only files matching a |
668 | regex in the flag. */ |
669 | if (profile_filter_files.is_empty ()) |
670 | return true; |
671 | |
672 | for (unsigned i = 0; i < profile_filter_files.length (); i++) |
673 | if (regexec (preg: &profile_filter_files[i], string: filename, nmatch: 0, NULL, eflags: 0) == REG_NOERROR) |
674 | return true; |
675 | |
676 | return false; |
677 | } |
678 | |
679 | #ifndef HAVE_sync_compare_and_swapsi |
680 | #define HAVE_sync_compare_and_swapsi 0 |
681 | #endif |
682 | #ifndef HAVE_atomic_compare_and_swapsi |
683 | #define HAVE_atomic_compare_and_swapsi 0 |
684 | #endif |
685 | |
686 | #ifndef HAVE_sync_compare_and_swapdi |
687 | #define HAVE_sync_compare_and_swapdi 0 |
688 | #endif |
689 | #ifndef HAVE_atomic_compare_and_swapdi |
690 | #define HAVE_atomic_compare_and_swapdi 0 |
691 | #endif |
692 | |
693 | /* Profile all functions in the callgraph. */ |
694 | |
695 | static unsigned int |
696 | tree_profiling (void) |
697 | { |
698 | struct cgraph_node *node; |
699 | |
700 | /* Verify whether we can utilize atomic update operations. */ |
701 | bool can_support_atomic = false; |
702 | unsigned HOST_WIDE_INT gcov_type_size |
703 | = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ())); |
704 | if (gcov_type_size == 4) |
705 | can_support_atomic |
706 | = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi; |
707 | else if (gcov_type_size == 8) |
708 | can_support_atomic |
709 | = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi; |
710 | |
711 | if (flag_profile_update == PROFILE_UPDATE_ATOMIC |
712 | && !can_support_atomic) |
713 | { |
714 | warning (0, "target does not support atomic profile update, " |
715 | "single mode is selected" ); |
716 | flag_profile_update = PROFILE_UPDATE_SINGLE; |
717 | } |
718 | else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC) |
719 | flag_profile_update = can_support_atomic |
720 | ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE; |
721 | |
722 | /* This is a small-ipa pass that gets called only once, from |
723 | cgraphunit.cc:ipa_passes(). */ |
724 | gcc_assert (symtab->state == IPA_SSA); |
725 | |
726 | init_node_map (true); |
727 | parse_profile_file_filtering (); |
728 | |
729 | FOR_EACH_DEFINED_FUNCTION (node) |
730 | { |
731 | bool thunk = false; |
732 | if (!gimple_has_body_p (node->decl) && !node->thunk) |
733 | continue; |
734 | |
735 | /* Don't profile functions produced for builtin stuff. */ |
736 | if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION) |
737 | continue; |
738 | |
739 | if (lookup_attribute (attr_name: "no_profile_instrument_function" , |
740 | DECL_ATTRIBUTES (node->decl))) |
741 | continue; |
742 | /* Do not instrument extern inline functions when testing coverage. |
743 | While this is not perfectly consistent (early inlined extern inlines |
744 | will get acocunted), testsuite expects that. */ |
745 | if (DECL_EXTERNAL (node->decl) |
746 | && flag_test_coverage) |
747 | continue; |
748 | |
749 | const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl)); |
750 | if (!include_source_file_for_profile (filename: file)) |
751 | continue; |
752 | |
753 | if (node->thunk) |
754 | { |
755 | /* We cannot expand variadic thunks to Gimple. */ |
756 | if (stdarg_p (TREE_TYPE (node->decl))) |
757 | continue; |
758 | thunk = true; |
759 | /* When generate profile, expand thunk to gimple so it can be |
760 | instrumented same way as other functions. */ |
761 | if (profile_arc_flag) |
762 | expand_thunk (node, false, true); |
763 | /* Read cgraph profile but keep function as thunk at profile-use |
764 | time. */ |
765 | else |
766 | { |
767 | read_thunk_profile (node); |
768 | continue; |
769 | } |
770 | } |
771 | |
772 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); |
773 | |
774 | if (dump_file) |
775 | dump_function_header (dump_file, cfun->decl, dump_flags); |
776 | |
777 | /* Local pure-const may imply need to fixup the cfg. */ |
778 | if (gimple_has_body_p (node->decl) |
779 | && (execute_fixup_cfg () & TODO_cleanup_cfg)) |
780 | cleanup_tree_cfg (); |
781 | |
782 | branch_prob (thunk); |
783 | |
784 | if (! flag_branch_probabilities |
785 | && flag_profile_values) |
786 | gimple_gen_ic_func_profiler (); |
787 | |
788 | if (flag_branch_probabilities |
789 | && !thunk |
790 | && flag_profile_values |
791 | && flag_value_profile_transformations |
792 | && profile_status_for_fn (cfun) == PROFILE_READ) |
793 | gimple_value_profile_transformations (); |
794 | |
795 | /* The above could hose dominator info. Currently there is |
796 | none coming in, this is a safety valve. It should be |
797 | easy to adjust it, if and when there is some. */ |
798 | free_dominance_info (CDI_DOMINATORS); |
799 | free_dominance_info (CDI_POST_DOMINATORS); |
800 | pop_cfun (); |
801 | } |
802 | |
803 | release_profile_file_filtering (); |
804 | |
805 | /* Drop pure/const flags from instrumented functions. */ |
806 | if (profile_arc_flag || flag_test_coverage) |
807 | FOR_EACH_DEFINED_FUNCTION (node) |
808 | { |
809 | if (!gimple_has_body_p (node->decl) |
810 | || !(!node->clone_of |
811 | || node->decl != node->clone_of->decl)) |
812 | continue; |
813 | |
814 | /* Don't profile functions produced for builtin stuff. */ |
815 | if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION) |
816 | continue; |
817 | |
818 | node->set_const_flag (set_const: false, looping: false); |
819 | node->set_pure_flag (pure: false, looping: false); |
820 | } |
821 | |
822 | /* Update call statements and rebuild the cgraph. */ |
823 | FOR_EACH_DEFINED_FUNCTION (node) |
824 | { |
825 | basic_block bb; |
826 | |
827 | if (!gimple_has_body_p (node->decl) |
828 | || !(!node->clone_of |
829 | || node->decl != node->clone_of->decl)) |
830 | continue; |
831 | |
832 | /* Don't profile functions produced for builtin stuff. */ |
833 | if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION) |
834 | continue; |
835 | |
836 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); |
837 | |
838 | if (profile_arc_flag || flag_test_coverage) |
839 | FOR_EACH_BB_FN (bb, cfun) |
840 | { |
841 | gimple_stmt_iterator gsi; |
842 | for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi)) |
843 | { |
844 | gcall *call = dyn_cast <gcall *> (p: gsi_stmt (i: gsi)); |
845 | if (!call || gimple_call_internal_p (gs: call)) |
846 | continue; |
847 | |
848 | /* We do not clear pure/const on decls without body. */ |
849 | tree fndecl = gimple_call_fndecl (gs: call); |
850 | cgraph_node *callee; |
851 | if (fndecl |
852 | && (callee = cgraph_node::get (decl: fndecl)) |
853 | && callee->get_availability (ref: node) == AVAIL_NOT_AVAILABLE) |
854 | continue; |
855 | |
856 | /* Drop the const attribute from the call type (the pure |
857 | attribute is not available on types). */ |
858 | tree fntype = gimple_call_fntype (gs: call); |
859 | if (fntype && TYPE_READONLY (fntype)) |
860 | { |
861 | int quals = TYPE_QUALS (fntype) & ~TYPE_QUAL_CONST; |
862 | fntype = build_qualified_type (fntype, quals); |
863 | gimple_call_set_fntype (call_stmt: call, fntype); |
864 | } |
865 | |
866 | /* Update virtual operands of calls to no longer const/pure |
867 | functions. */ |
868 | update_stmt (s: call); |
869 | } |
870 | } |
871 | |
872 | /* re-merge split blocks. */ |
873 | cleanup_tree_cfg (); |
874 | update_ssa (TODO_update_ssa); |
875 | |
876 | cgraph_edge::rebuild_edges (); |
877 | |
878 | pop_cfun (); |
879 | } |
880 | |
881 | handle_missing_profiles (); |
882 | |
883 | del_node_map (); |
884 | return 0; |
885 | } |
886 | |
887 | namespace { |
888 | |
889 | const pass_data pass_data_ipa_tree_profile = |
890 | { |
891 | .type: SIMPLE_IPA_PASS, /* type */ |
892 | .name: "profile" , /* name */ |
893 | .optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */ |
894 | .tv_id: TV_IPA_PROFILE, /* tv_id */ |
895 | .properties_required: 0, /* properties_required */ |
896 | .properties_provided: 0, /* properties_provided */ |
897 | .properties_destroyed: 0, /* properties_destroyed */ |
898 | .todo_flags_start: 0, /* todo_flags_start */ |
899 | TODO_dump_symtab, /* todo_flags_finish */ |
900 | }; |
901 | |
902 | class pass_ipa_tree_profile : public simple_ipa_opt_pass |
903 | { |
904 | public: |
905 | pass_ipa_tree_profile (gcc::context *ctxt) |
906 | : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt) |
907 | {} |
908 | |
909 | /* opt_pass methods: */ |
910 | bool gate (function *) final override; |
911 | unsigned int execute (function *) final override { return tree_profiling (); } |
912 | |
913 | }; // class pass_ipa_tree_profile |
914 | |
915 | bool |
916 | pass_ipa_tree_profile::gate (function *) |
917 | { |
918 | /* When profile instrumentation, use or test coverage shall be performed. |
919 | But for AutoFDO, this there is no instrumentation, thus this pass is |
920 | disabled. */ |
921 | return (!in_lto_p && !flag_auto_profile |
922 | && (flag_branch_probabilities || flag_test_coverage |
923 | || profile_arc_flag)); |
924 | } |
925 | |
926 | } // anon namespace |
927 | |
928 | simple_ipa_opt_pass * |
929 | make_pass_ipa_tree_profile (gcc::context *ctxt) |
930 | { |
931 | return new pass_ipa_tree_profile (ctxt); |
932 | } |
933 | |
934 | #include "gt-tree-profile.h" |
935 | |