1/* Callgraph clones
2 Copyright (C) 2003-2023 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21/* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
24
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
27 clones
28
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
32
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
38
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
42
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
49 a given function.
50
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
54
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
61
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
66
67#include "config.h"
68#include "system.h"
69#include "coretypes.h"
70#include "backend.h"
71#include "target.h"
72#include "rtl.h"
73#include "tree.h"
74#include "gimple.h"
75#include "stringpool.h"
76#include "cgraph.h"
77#include "lto-streamer.h"
78#include "tree-eh.h"
79#include "tree-cfg.h"
80#include "tree-inline.h"
81#include "dumpfile.h"
82#include "gimple-pretty-print.h"
83#include "alloc-pool.h"
84#include "symbol-summary.h"
85#include "tree-vrp.h"
86#include "ipa-prop.h"
87#include "ipa-fnsummary.h"
88#include "symtab-thunks.h"
89#include "symtab-clones.h"
90
91/* Create clone of edge in the node N represented by CALL_EXPR
92 the callgraph. */
93
94cgraph_edge *
95cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid,
96 profile_count num, profile_count den,
97 bool update_original)
98{
99 cgraph_edge *new_edge;
100 profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
101 profile_count prof_count = count.apply_scale (num, den);
102
103 if (indirect_unknown_callee)
104 {
105 tree decl;
106
107 if (call_stmt && (decl = gimple_call_fndecl (gs: call_stmt))
108 /* When the call is speculative, we need to resolve it
109 via cgraph_resolve_speculation and not here. */
110 && !speculative)
111 {
112 cgraph_node *callee = cgraph_node::get (decl);
113 gcc_checking_assert (callee);
114 new_edge = n->create_edge (callee, call_stmt, count: prof_count, cloning_p: true);
115 }
116 else
117 {
118 new_edge = n->create_indirect_edge (call_stmt,
119 ecf_flags: indirect_info->ecf_flags,
120 count: prof_count, cloning_p: true);
121 *new_edge->indirect_info = *indirect_info;
122 }
123 }
124 else
125 {
126 new_edge = n->create_edge (callee, call_stmt, count: prof_count, cloning_p: true);
127 if (indirect_info)
128 {
129 new_edge->indirect_info
130 = ggc_cleared_alloc<cgraph_indirect_call_info> ();
131 *new_edge->indirect_info = *indirect_info;
132 }
133 }
134
135 new_edge->inline_failed = inline_failed;
136 new_edge->indirect_inlining_edge = indirect_inlining_edge;
137 if (!call_stmt)
138 new_edge->lto_stmt_uid = stmt_uid;
139 new_edge->speculative_id = speculative_id;
140 /* Clone flags that depend on call_stmt availability manually. */
141 new_edge->can_throw_external = can_throw_external;
142 new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p;
143 new_edge->speculative = speculative;
144 new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor;
145
146 /* Update IPA profile. Local profiles need no updating in original. */
147 if (update_original)
148 count = count.combine_with_ipa_count_within (ipa: count.ipa ()
149 - new_edge->count.ipa (),
150 ipa2: caller->count);
151 symtab->call_edge_duplication_hooks (cs1: this, cs2: new_edge);
152 return new_edge;
153}
154
155/* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
156 clone or its thunk. */
157
158static void
159set_new_clone_decl_and_node_flags (cgraph_node *new_node)
160{
161 DECL_EXTERNAL (new_node->decl) = 0;
162 TREE_PUBLIC (new_node->decl) = 0;
163 DECL_COMDAT (new_node->decl) = 0;
164 DECL_WEAK (new_node->decl) = 0;
165 DECL_VIRTUAL_P (new_node->decl) = 0;
166 DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0;
167 DECL_STATIC_DESTRUCTOR (new_node->decl) = 0;
168 DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0);
169 DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0);
170 DECL_IS_REPLACEABLE_OPERATOR (new_node->decl) = 0;
171
172 new_node->externally_visible = 0;
173 new_node->local = 1;
174 new_node->lowered = true;
175 new_node->semantic_interposition = 0;
176}
177
178/* Duplicate thunk THUNK if necessary but make it to refer to NODE.
179 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
180 Function can return NODE if no thunk is necessary, which can happen when
181 thunk is this_adjusting but we are removing this parameter. */
182
183static cgraph_node *
184duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node)
185{
186 cgraph_node *new_thunk, *thunk_of;
187 thunk_of = thunk->callees->callee->ultimate_alias_target ();
188
189 if (thunk_of->thunk)
190 node = duplicate_thunk_for_node (thunk: thunk_of, node);
191
192 if (!DECL_ARGUMENTS (thunk->decl))
193 thunk->get_untransformed_body ();
194
195 thunk_info *i = thunk_info::get (node: thunk);
196 cgraph_edge *cs;
197 for (cs = node->callers; cs; cs = cs->next_caller)
198 if (cs->caller->thunk)
199 {
200 thunk_info *i2 = thunk_info::get (node: cs->caller);
201 if (*i2 == *i)
202 return cs->caller;
203 }
204
205 tree new_decl;
206 clone_info *info = clone_info::get (node);
207 if (info && info->param_adjustments)
208 {
209 /* We do not need to duplicate this_adjusting thunks if we have removed
210 this. */
211 if (i->this_adjusting
212 && !info->param_adjustments->first_param_intact_p ())
213 return node;
214
215 new_decl = copy_node (thunk->decl);
216 ipa_param_body_adjustments body_adj (info->param_adjustments,
217 new_decl);
218 body_adj.modify_formal_parameters ();
219 }
220 else
221 {
222 new_decl = copy_node (thunk->decl);
223 for (tree *arg = &DECL_ARGUMENTS (new_decl);
224 *arg; arg = &DECL_CHAIN (*arg))
225 {
226 tree next = DECL_CHAIN (*arg);
227 *arg = copy_node (*arg);
228 DECL_CONTEXT (*arg) = new_decl;
229 DECL_CHAIN (*arg) = next;
230 }
231 }
232
233 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl));
234 gcc_checking_assert (!DECL_INITIAL (new_decl));
235 gcc_checking_assert (!DECL_RESULT (new_decl));
236 gcc_checking_assert (!DECL_RTL_SET_P (new_decl));
237
238 DECL_NAME (new_decl) = clone_function_name_numbered (decl: thunk->decl,
239 suffix: "artificial_thunk");
240 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
241
242 /* We need to force DECL_IGNORED_P because the new thunk is created after
243 early debug was run. */
244 DECL_IGNORED_P (new_decl) = 1;
245
246 new_thunk = cgraph_node::create (decl: new_decl);
247 set_new_clone_decl_and_node_flags (new_thunk);
248 new_thunk->definition = true;
249 new_thunk->can_change_signature = node->can_change_signature;
250 new_thunk->thunk = thunk->thunk;
251 new_thunk->unique_name = in_lto_p;
252 new_thunk->former_clone_of = thunk->decl;
253 if (info && info->param_adjustments)
254 clone_info::get_create (node: new_thunk)->param_adjustments
255 = info->param_adjustments;
256 new_thunk->unit_id = thunk->unit_id;
257 new_thunk->merged_comdat = thunk->merged_comdat;
258 new_thunk->merged_extern_inline = thunk->merged_extern_inline;
259
260 cgraph_edge *e = new_thunk->create_edge (callee: node, NULL, count: new_thunk->count);
261 symtab->call_edge_duplication_hooks (cs1: thunk->callees, cs2: e);
262 symtab->call_cgraph_duplication_hooks (node: thunk, node2: new_thunk);
263 return new_thunk;
264}
265
266/* If E does not lead to a thunk, simply redirect it to N. Otherwise create
267 one or more equivalent thunks for N and redirect E to the first in the
268 chain. Note that it is then necessary to call
269 n->expand_all_artificial_thunks once all callers are redirected. */
270
271void
272cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n)
273{
274 cgraph_node *orig_to = callee->ultimate_alias_target ();
275 if (orig_to->thunk)
276 n = duplicate_thunk_for_node (thunk: orig_to, node: n);
277
278 redirect_callee (n);
279}
280
281/* Call expand_thunk on all callers that are thunks and if analyze those nodes
282 that were expanded. */
283
284void
285cgraph_node::expand_all_artificial_thunks ()
286{
287 cgraph_edge *e;
288 for (e = callers; e;)
289 if (e->caller->thunk)
290 {
291 cgraph_node *thunk = e->caller;
292
293 e = e->next_caller;
294 if (expand_thunk (thunk, false, false))
295 {
296 thunk->thunk = false;
297 thunk->analyze ();
298 ipa_analyze_node (thunk);
299 inline_analyze_function (node: thunk);
300 }
301 thunk->expand_all_artificial_thunks ();
302 }
303 else
304 e = e->next_caller;
305}
306
307void
308dump_callgraph_transformation (const cgraph_node *original,
309 const cgraph_node *clone,
310 const char *suffix)
311{
312 if (symtab->ipa_clones_dump_file)
313 {
314 fprintf (stream: symtab->ipa_clones_dump_file,
315 format: "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
316 original->asm_name (), original->order,
317 DECL_SOURCE_FILE (original->decl),
318 DECL_SOURCE_LINE (original->decl),
319 DECL_SOURCE_COLUMN (original->decl), clone->asm_name (),
320 clone->order, DECL_SOURCE_FILE (clone->decl),
321 DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl),
322 suffix);
323
324 symtab->cloned_nodes.add (k: original);
325 symtab->cloned_nodes.add (k: clone);
326 }
327}
328
329/* Turn profile of N to local profile. */
330
331static void
332localize_profile (cgraph_node *n)
333{
334 n->count = n->count.guessed_local ();
335 for (cgraph_edge *e = n->callees; e; e=e->next_callee)
336 {
337 e->count = e->count.guessed_local ();
338 if (!e->inline_failed)
339 localize_profile (n: e->callee);
340 }
341 for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee)
342 e->count = e->count.guessed_local ();
343}
344
345/* Create node representing clone of N executed COUNT times. Decrease
346 the execution counts from original node too.
347 The new clone will have decl set to DECL that may or may not be the same
348 as decl of N.
349
350 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
351 function's profile to reflect the fact that part of execution is handled
352 by node.
353 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
354 the new clone. Otherwise the caller is responsible for doing so later.
355
356 If the new node is being inlined into another one, NEW_INLINED_TO should be
357 the outline function the new one is (even indirectly) inlined to. All hooks
358 will see this in node's inlined_to, when invoked. Can be NULL if the
359 node is not inlined.
360
361 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
362 will be overwritten by the new structure. Otherwise the new node will
363 share parameter manipulation information with the original node. */
364
365cgraph_node *
366cgraph_node::create_clone (tree new_decl, profile_count prof_count,
367 bool update_original,
368 vec<cgraph_edge *> redirect_callers,
369 bool call_duplication_hook,
370 cgraph_node *new_inlined_to,
371 ipa_param_adjustments *param_adjustments,
372 const char *suffix)
373{
374 cgraph_node *new_node = symtab->create_empty ();
375 cgraph_edge *e;
376 unsigned i;
377 profile_count old_count = count;
378 bool nonzero = count.ipa ().nonzero_p ();
379
380 if (new_inlined_to)
381 dump_callgraph_transformation (original: this, clone: new_inlined_to, suffix: "inlining to");
382
383 /* When inlining we scale precisely to prof_count, when cloning we can
384 preserve local profile. */
385 if (!new_inlined_to)
386 prof_count = count.combine_with_ipa_count (ipa: prof_count);
387 new_node->count = prof_count;
388 new_node->calls_declare_variant_alt = this->calls_declare_variant_alt;
389
390 /* Update IPA profile. Local profiles need no updating in original. */
391 if (update_original)
392 {
393 if (inlined_to)
394 count = count.combine_with_ipa_count_within (ipa: count.ipa ()
395 - prof_count.ipa (),
396 ipa2: inlined_to->count);
397 else
398 count = count.combine_with_ipa_count (ipa: count.ipa () - prof_count.ipa ());
399 }
400 new_node->decl = new_decl;
401 new_node->register_symbol ();
402 new_node->lto_file_data = lto_file_data;
403 new_node->analyzed = analyzed;
404 new_node->definition = definition;
405 new_node->versionable = versionable;
406 new_node->can_change_signature = can_change_signature;
407 new_node->redefined_extern_inline = redefined_extern_inline;
408 new_node->semantic_interposition = semantic_interposition;
409 new_node->tm_may_enter_irr = tm_may_enter_irr;
410 new_node->externally_visible = false;
411 new_node->no_reorder = no_reorder;
412 new_node->local = true;
413 new_node->inlined_to = new_inlined_to;
414 new_node->rtl = rtl;
415 new_node->frequency = frequency;
416 new_node->tp_first_run = tp_first_run;
417 new_node->tm_clone = tm_clone;
418 new_node->icf_merged = icf_merged;
419 new_node->thunk = thunk;
420 new_node->unit_id = unit_id;
421 new_node->merged_comdat = merged_comdat;
422 new_node->merged_extern_inline = merged_extern_inline;
423 clone_info *info = clone_info::get (node: this);
424
425 if (param_adjustments)
426 clone_info::get_create (node: new_node)->param_adjustments = param_adjustments;
427 else if (info && info->param_adjustments)
428 clone_info::get_create (node: new_node)->param_adjustments
429 = info->param_adjustments;
430 new_node->split_part = split_part;
431
432 FOR_EACH_VEC_ELT (redirect_callers, i, e)
433 {
434 /* Redirect calls to the old version node to point to its new
435 version. The only exception is when the edge was proved to
436 be unreachable during the cloning procedure. */
437 if (!e->callee
438 || !fndecl_built_in_p (node: e->callee->decl, name1: BUILT_IN_UNREACHABLE,
439 names: BUILT_IN_UNREACHABLE_TRAP))
440 e->redirect_callee_duplicating_thunks (n: new_node);
441 }
442 new_node->expand_all_artificial_thunks ();
443
444 for (e = callees;e; e=e->next_callee)
445 e->clone (n: new_node, call_stmt: e->call_stmt, stmt_uid: e->lto_stmt_uid, num: new_node->count, den: old_count,
446 update_original);
447
448 for (e = indirect_calls; e; e = e->next_callee)
449 e->clone (n: new_node, call_stmt: e->call_stmt, stmt_uid: e->lto_stmt_uid,
450 num: new_node->count, den: old_count, update_original);
451 new_node->clone_references (node: this);
452
453 new_node->next_sibling_clone = clones;
454 if (clones)
455 clones->prev_sibling_clone = new_node;
456 clones = new_node;
457 new_node->clone_of = this;
458
459 if (call_duplication_hook)
460 symtab->call_cgraph_duplication_hooks (node: this, node2: new_node);
461 /* With partial train run we do not want to assume that original's
462 count is zero whenever we redurect all executed edges to clone.
463 Simply drop profile to local one in this case. */
464 if (update_original
465 && opt_for_fn (decl, flag_profile_partial_training)
466 && nonzero
467 && count.ipa_p ()
468 && !count.ipa ().nonzero_p ()
469 && !inlined_to)
470 localize_profile (n: this);
471
472 if (!new_inlined_to)
473 dump_callgraph_transformation (original: this, clone: new_node, suffix);
474
475 return new_node;
476}
477
478static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids;
479
480/* Return a new assembler name for a clone of decl named NAME. Apart
481 from the string SUFFIX, the new name will end with a unique (for
482 each NAME) unspecified number. If clone numbering is not needed
483 then the two argument clone_function_name should be used instead.
484 Should not be called directly except for by
485 lto-partition.cc:privatize_symbol_name_1. */
486
487tree
488clone_function_name_numbered (const char *name, const char *suffix)
489{
490 /* Initialize the function->counter mapping the first time it's
491 needed. */
492 if (!clone_fn_ids)
493 clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (size: 64);
494 unsigned int &suffix_counter = clone_fn_ids->get_or_insert (
495 IDENTIFIER_POINTER (get_identifier (name)));
496 return clone_function_name (name, suffix, number: suffix_counter++);
497}
498
499/* Return a new assembler name for a clone of DECL. Apart from string
500 SUFFIX, the new name will end with a unique (for each DECL
501 assembler name) unspecified number. If clone numbering is not
502 needed then the two argument clone_function_name should be used
503 instead. */
504
505tree
506clone_function_name_numbered (tree decl, const char *suffix)
507{
508 tree name = DECL_ASSEMBLER_NAME (decl);
509 return clone_function_name_numbered (IDENTIFIER_POINTER (name),
510 suffix);
511}
512
513/* Return a new assembler name for a clone of decl named NAME. Apart
514 from the string SUFFIX, the new name will end with the specified
515 NUMBER. If clone numbering is not needed then the two argument
516 clone_function_name should be used instead. */
517
518tree
519clone_function_name (const char *name, const char *suffix,
520 unsigned long number)
521{
522 size_t len = strlen (s: name);
523 char *tmp_name, *prefix;
524
525 prefix = XALLOCAVEC (char, len + strlen (suffix) + 2);
526 memcpy (dest: prefix, src: name, n: len);
527 strcpy (dest: prefix + len + 1, src: suffix);
528 prefix[len] = symbol_table::symbol_suffix_separator ();
529 ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number);
530 return get_identifier (tmp_name);
531}
532
533/* Return a new assembler name for a clone of DECL. Apart from the
534 string SUFFIX, the new name will end with the specified NUMBER. If
535 clone numbering is not needed then the two argument
536 clone_function_name should be used instead. */
537
538tree
539clone_function_name (tree decl, const char *suffix,
540 unsigned long number)
541{
542 return clone_function_name (
543 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number);
544}
545
546/* Return a new assembler name ending with the string SUFFIX for a
547 clone of DECL. */
548
549tree
550clone_function_name (tree decl, const char *suffix)
551{
552 tree identifier = DECL_ASSEMBLER_NAME (decl);
553 /* For consistency this needs to behave the same way as
554 ASM_FORMAT_PRIVATE_NAME does, but without the final number
555 suffix. */
556 char *separator = XALLOCAVEC (char, 2);
557 separator[0] = symbol_table::symbol_suffix_separator ();
558 separator[1] = 0;
559#if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
560 const char *prefix = "__";
561#else
562 const char *prefix = "";
563#endif
564 char *result = ACONCAT ((prefix,
565 IDENTIFIER_POINTER (identifier),
566 separator,
567 suffix,
568 (char*)0));
569 return get_identifier (result);
570}
571
572
573/* Create callgraph node clone with new declaration. The actual body will be
574 copied later at compilation stage. The name of the new clone will be
575 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
576
577 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
578 bitmap interface.
579 */
580cgraph_node *
581cgraph_node::create_virtual_clone (const vec<cgraph_edge *> &redirect_callers,
582 vec<ipa_replace_map *, va_gc> *tree_map,
583 ipa_param_adjustments *param_adjustments,
584 const char * suffix, unsigned num_suffix)
585{
586 tree old_decl = decl;
587 cgraph_node *new_node = NULL;
588 tree new_decl;
589 size_t len, i;
590 ipa_replace_map *map;
591 char *name;
592
593 gcc_checking_assert (versionable);
594 /* TODO: It would be nice if we could recognize that param_adjustments do not
595 actually perform any changes, but at the moment let's require it simply
596 does not exist. */
597 gcc_assert (can_change_signature || !param_adjustments);
598
599 /* Make a new FUNCTION_DECL tree node */
600 if (!param_adjustments)
601 new_decl = copy_node (old_decl);
602 else
603 new_decl = param_adjustments->adjust_decl (orig_decl: old_decl);
604
605 /* These pointers represent function body and will be populated only when clone
606 is materialized. */
607 gcc_assert (new_decl != old_decl);
608 DECL_STRUCT_FUNCTION (new_decl) = NULL;
609 DECL_ARGUMENTS (new_decl) = NULL;
610 DECL_INITIAL (new_decl) = NULL;
611 DECL_RESULT (new_decl) = NULL;
612 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
613 sometimes storing only clone decl instead of original. */
614
615 /* Generate a new name for the new version. */
616 len = IDENTIFIER_LENGTH (DECL_NAME (old_decl));
617 name = XALLOCAVEC (char, len + strlen (suffix) + 2);
618 memcpy (dest: name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), n: len);
619 strcpy (dest: name + len + 1, src: suffix);
620 name[len] = '.';
621 DECL_NAME (new_decl) = get_identifier (name);
622 SET_DECL_ASSEMBLER_NAME (new_decl,
623 clone_function_name (old_decl, suffix, num_suffix));
624 SET_DECL_RTL (new_decl, NULL);
625
626 new_node = create_clone (new_decl, prof_count: count, update_original: false,
627 redirect_callers, call_duplication_hook: false, NULL, param_adjustments,
628 suffix);
629
630 /* Update the properties.
631 Make clone visible only within this translation unit. Make sure
632 that is not weak also.
633 ??? We cannot use COMDAT linkage because there is no
634 ABI support for this. */
635 set_new_clone_decl_and_node_flags (new_node);
636 new_node->ipcp_clone = ipcp_clone;
637 if (tree_map)
638 clone_info::get_create (node: new_node)->tree_map = tree_map;
639 if (!implicit_section)
640 new_node->set_section (*this);
641
642 /* Clones of global symbols or symbols with unique names are unique. */
643 if ((TREE_PUBLIC (old_decl)
644 && !DECL_EXTERNAL (old_decl)
645 && !DECL_WEAK (old_decl)
646 && !DECL_COMDAT (old_decl))
647 || in_lto_p)
648 new_node->unique_name = true;
649 FOR_EACH_VEC_SAFE_ELT (tree_map, i, map)
650 {
651 tree repl = map->new_tree;
652 if (map->force_load_ref)
653 {
654 gcc_assert (TREE_CODE (repl) == ADDR_EXPR);
655 repl = get_base_address (TREE_OPERAND (repl, 0));
656 }
657 new_node->maybe_create_reference (val: repl, NULL);
658 }
659
660 if (ipa_transforms_to_apply.exists ())
661 new_node->ipa_transforms_to_apply
662 = ipa_transforms_to_apply.copy ();
663
664 symtab->call_cgraph_duplication_hooks (node: this, node2: new_node);
665
666 return new_node;
667}
668
669/* callgraph node being removed from symbol table; see if its entry can be
670 replaced by other inline clone.
671 INFO is clone info to attach to the new root. */
672cgraph_node *
673cgraph_node::find_replacement (clone_info *info)
674{
675 cgraph_node *next_inline_clone, *replacement;
676
677 for (next_inline_clone = clones;
678 next_inline_clone
679 && next_inline_clone->decl != decl;
680 next_inline_clone = next_inline_clone->next_sibling_clone)
681 ;
682
683 /* If there is inline clone of the node being removed, we need
684 to put it into the position of removed node and reorganize all
685 other clones to be based on it. */
686 if (next_inline_clone)
687 {
688 cgraph_node *n;
689 cgraph_node *new_clones;
690
691 replacement = next_inline_clone;
692
693 /* Unlink inline clone from the list of clones of removed node. */
694 if (next_inline_clone->next_sibling_clone)
695 next_inline_clone->next_sibling_clone->prev_sibling_clone
696 = next_inline_clone->prev_sibling_clone;
697 if (next_inline_clone->prev_sibling_clone)
698 {
699 gcc_assert (clones != next_inline_clone);
700 next_inline_clone->prev_sibling_clone->next_sibling_clone
701 = next_inline_clone->next_sibling_clone;
702 }
703 else
704 {
705 gcc_assert (clones == next_inline_clone);
706 clones = next_inline_clone->next_sibling_clone;
707 }
708
709 new_clones = clones;
710 clones = NULL;
711
712 /* Copy clone info. */
713 if (info)
714 *clone_info::get_create (node: next_inline_clone) = *info;
715
716 /* Now place it into clone tree at same level at NODE. */
717 next_inline_clone->clone_of = clone_of;
718 next_inline_clone->prev_sibling_clone = NULL;
719 next_inline_clone->next_sibling_clone = NULL;
720 if (clone_of)
721 {
722 if (clone_of->clones)
723 clone_of->clones->prev_sibling_clone = next_inline_clone;
724 next_inline_clone->next_sibling_clone = clone_of->clones;
725 clone_of->clones = next_inline_clone;
726 }
727
728 /* Merge the clone list. */
729 if (new_clones)
730 {
731 if (!next_inline_clone->clones)
732 next_inline_clone->clones = new_clones;
733 else
734 {
735 n = next_inline_clone->clones;
736 while (n->next_sibling_clone)
737 n = n->next_sibling_clone;
738 n->next_sibling_clone = new_clones;
739 new_clones->prev_sibling_clone = n;
740 }
741 }
742
743 /* Update clone_of pointers. */
744 n = new_clones;
745 while (n)
746 {
747 n->clone_of = next_inline_clone;
748 n = n->next_sibling_clone;
749 }
750
751 /* Update order in order to be able to find a LTO section
752 with function body. */
753 replacement->order = order;
754
755 return replacement;
756 }
757 else
758 return NULL;
759}
760
761/* Like cgraph_set_call_stmt but walk the clone tree and update all
762 clones sharing the same function body.
763 When WHOLE_SPECULATIVE_EDGES is true, all three components of
764 speculative edge gets updated. Otherwise we update only direct
765 call. */
766
767void
768cgraph_node::set_call_stmt_including_clones (gimple *old_stmt,
769 gcall *new_stmt,
770 bool update_speculative)
771{
772 cgraph_node *node;
773 cgraph_edge *master_edge = get_edge (call_stmt: old_stmt);
774
775 if (master_edge)
776 cgraph_edge::set_call_stmt (e: master_edge, new_stmt, update_speculative);
777
778 node = clones;
779 if (node)
780 while (node != this)
781 {
782 cgraph_edge *edge = node->get_edge (call_stmt: old_stmt);
783 if (edge)
784 {
785 edge = cgraph_edge::set_call_stmt (e: edge, new_stmt,
786 update_speculative);
787 /* If UPDATE_SPECULATIVE is false, it means that we are turning
788 speculative call into a real code sequence. Update the
789 callgraph edges. */
790 if (edge->speculative && !update_speculative)
791 {
792 cgraph_edge *indirect = edge->speculative_call_indirect_edge ();
793
794 for (cgraph_edge *next, *direct
795 = edge->first_speculative_call_target ();
796 direct;
797 direct = next)
798 {
799 next = direct->next_speculative_call_target ();
800 direct->speculative_call_target_ref ()->speculative = false;
801 direct->speculative = false;
802 }
803 indirect->speculative = false;
804 }
805 }
806 if (node->clones)
807 node = node->clones;
808 else if (node->next_sibling_clone)
809 node = node->next_sibling_clone;
810 else
811 {
812 while (node != this && !node->next_sibling_clone)
813 node = node->clone_of;
814 if (node != this)
815 node = node->next_sibling_clone;
816 }
817 }
818}
819
820/* Like cgraph_create_edge walk the clone tree and update all clones sharing
821 same function body. If clones already have edge for OLD_STMT; only
822 update the edge same way as cgraph_set_call_stmt_including_clones does.
823
824 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
825 frequencies of the clones. */
826
827void
828cgraph_node::create_edge_including_clones (cgraph_node *callee,
829 gimple *old_stmt, gcall *stmt,
830 profile_count count,
831 cgraph_inline_failed_t reason)
832{
833 cgraph_node *node;
834
835 if (!get_edge (call_stmt: stmt))
836 {
837 cgraph_edge *edge = create_edge (callee, call_stmt: stmt, count);
838 edge->inline_failed = reason;
839 }
840
841 node = clones;
842 if (node)
843 while (node != this)
844 /* Thunk clones do not get updated while copying inline function body. */
845 if (!node->thunk)
846 {
847 cgraph_edge *edge = node->get_edge (call_stmt: old_stmt);
848
849 /* It is possible that clones already contain the edge while
850 master didn't. Either we promoted indirect call into direct
851 call in the clone or we are processing clones of unreachable
852 master where edges has been removed. */
853 if (edge)
854 edge = cgraph_edge::set_call_stmt (e: edge, new_stmt: stmt);
855 else if (! node->get_edge (call_stmt: stmt))
856 {
857 edge = node->create_edge (callee, call_stmt: stmt, count);
858 edge->inline_failed = reason;
859 }
860
861 if (node->clones)
862 node = node->clones;
863 else if (node->next_sibling_clone)
864 node = node->next_sibling_clone;
865 else
866 {
867 while (node != this && !node->next_sibling_clone)
868 node = node->clone_of;
869 if (node != this)
870 node = node->next_sibling_clone;
871 }
872 }
873}
874
875/* Remove the node from cgraph and all inline clones inlined into it.
876 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
877 removed. This allows to call the function from outer loop walking clone
878 tree. */
879
880bool
881cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node)
882{
883 cgraph_edge *e, *next;
884 bool found = false;
885
886 if (this == forbidden_node)
887 {
888 cgraph_edge::remove (edge: callers);
889 return true;
890 }
891 for (e = callees; e; e = next)
892 {
893 next = e->next_callee;
894 if (!e->inline_failed)
895 found |= e->callee->remove_symbol_and_inline_clones (forbidden_node);
896 }
897 remove ();
898 return found;
899}
900
901/* The edges representing the callers of the NEW_VERSION node were
902 fixed by cgraph_function_versioning (), now the call_expr in their
903 respective tree code should be updated to call the NEW_VERSION. */
904
905static void
906update_call_expr (cgraph_node *new_version)
907{
908 cgraph_edge *e;
909
910 gcc_assert (new_version);
911
912 /* Update the call expr on the edges to call the new version. */
913 for (e = new_version->callers; e; e = e->next_caller)
914 {
915 function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl);
916 gimple_call_set_fndecl (gs: e->call_stmt, decl: new_version->decl);
917 maybe_clean_eh_stmt_fn (inner_function, e->call_stmt);
918 }
919}
920
921
922/* Create a new cgraph node which is the new version of
923 callgraph node. REDIRECT_CALLERS holds the callers
924 edges which should be redirected to point to
925 NEW_VERSION. ALL the callees edges of the node
926 are cloned to the new version node. Return the new
927 version node.
928
929 If non-NULL BLOCK_TO_COPY determine what basic blocks
930 was copied to prevent duplications of calls that are dead
931 in the clone. */
932
933cgraph_node *
934cgraph_node::create_version_clone (tree new_decl,
935 vec<cgraph_edge *> redirect_callers,
936 bitmap bbs_to_copy,
937 const char *suffix)
938 {
939 cgraph_node *new_version;
940 cgraph_edge *e;
941 unsigned i;
942
943 new_version = cgraph_node::create (decl: new_decl);
944
945 new_version->analyzed = analyzed;
946 new_version->definition = definition;
947 new_version->local = local;
948 new_version->externally_visible = false;
949 new_version->no_reorder = no_reorder;
950 new_version->local = new_version->definition;
951 new_version->inlined_to = inlined_to;
952 new_version->rtl = rtl;
953 new_version->count = count;
954 new_version->unit_id = unit_id;
955 new_version->merged_comdat = merged_comdat;
956 new_version->merged_extern_inline = merged_extern_inline;
957
958 for (e = callees; e; e=e->next_callee)
959 if (!bbs_to_copy
960 || bitmap_bit_p (bbs_to_copy, gimple_bb (g: e->call_stmt)->index))
961 e->clone (n: new_version, call_stmt: e->call_stmt,
962 stmt_uid: e->lto_stmt_uid, num: count, den: count,
963 update_original: true);
964 for (e = indirect_calls; e; e=e->next_callee)
965 if (!bbs_to_copy
966 || bitmap_bit_p (bbs_to_copy, gimple_bb (g: e->call_stmt)->index))
967 e->clone (n: new_version, call_stmt: e->call_stmt,
968 stmt_uid: e->lto_stmt_uid, num: count, den: count,
969 update_original: true);
970 FOR_EACH_VEC_ELT (redirect_callers, i, e)
971 {
972 /* Redirect calls to the old version node to point to its new
973 version. */
974 e->redirect_callee (n: new_version);
975 }
976
977 dump_callgraph_transformation (original: this, clone: new_version, suffix);
978
979 return new_version;
980 }
981
982/* Perform function versioning.
983 Function versioning includes copying of the tree and
984 a callgraph update (creating a new cgraph node and updating
985 its callees and callers).
986
987 REDIRECT_CALLERS varray includes the edges to be redirected
988 to the new version.
989
990 TREE_MAP is a mapping of tree nodes we want to replace with
991 new ones (according to results of prior analysis).
992
993 If non-NULL ARGS_TO_SKIP determine function parameters to remove
994 from new version.
995 If SKIP_RETURN is true, the new version will return void.
996 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
997 If non_NULL NEW_ENTRY determine new entry BB of the clone.
998
999 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
1000 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
1001 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
1002 of the declaration.
1003
1004 If VERSION_DECL is set true, use clone_function_name_numbered for the
1005 function clone. Otherwise, use clone_function_name.
1006
1007 Return the new version's cgraph node. */
1008
1009cgraph_node *
1010cgraph_node::create_version_clone_with_body
1011 (vec<cgraph_edge *> redirect_callers,
1012 vec<ipa_replace_map *, va_gc> *tree_map,
1013 ipa_param_adjustments *param_adjustments,
1014 bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix,
1015 tree target_attributes, bool version_decl)
1016{
1017 tree old_decl = decl;
1018 cgraph_node *new_version_node = NULL;
1019 tree new_decl;
1020
1021 if (!tree_versionable_function_p (old_decl))
1022 return NULL;
1023
1024 /* TODO: Restore an assert that we do not change signature if
1025 can_change_signature is false. We cannot just check that
1026 param_adjustments is NULL because unfortunately ipa-split removes return
1027 values from such functions. */
1028
1029 /* Make a new FUNCTION_DECL tree node for the new version. */
1030 if (param_adjustments)
1031 new_decl = param_adjustments->adjust_decl (orig_decl: old_decl);
1032 else
1033 new_decl = copy_node (old_decl);
1034
1035 /* Generate a new name for the new version. */
1036 tree fnname = (version_decl ? clone_function_name_numbered (decl: old_decl, suffix)
1037 : clone_function_name (decl: old_decl, suffix));
1038 DECL_NAME (new_decl) = fnname;
1039 SET_DECL_ASSEMBLER_NAME (new_decl, fnname);
1040 SET_DECL_RTL (new_decl, NULL);
1041
1042 DECL_VIRTUAL_P (new_decl) = 0;
1043
1044 if (target_attributes)
1045 {
1046 DECL_ATTRIBUTES (new_decl) = target_attributes;
1047
1048 location_t saved_loc = input_location;
1049 tree v = TREE_VALUE (target_attributes);
1050 input_location = DECL_SOURCE_LOCATION (new_decl);
1051 bool r = targetm.target_option.valid_attribute_p (new_decl, NULL, v, 1);
1052 input_location = saved_loc;
1053 if (!r)
1054 return NULL;
1055 }
1056
1057 /* When the old decl was a con-/destructor make sure the clone isn't. */
1058 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
1059 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
1060 DECL_SET_IS_OPERATOR_NEW (new_decl, 0);
1061 DECL_SET_IS_OPERATOR_DELETE (new_decl, 0);
1062 DECL_IS_REPLACEABLE_OPERATOR (new_decl) = 0;
1063
1064 /* Create the new version's call-graph node.
1065 and update the edges of the new node. */
1066 new_version_node = create_version_clone (new_decl, redirect_callers,
1067 bbs_to_copy, suffix);
1068
1069 if (ipa_transforms_to_apply.exists ())
1070 new_version_node->ipa_transforms_to_apply
1071 = ipa_transforms_to_apply.copy ();
1072 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1073 tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments,
1074 false, bbs_to_copy, new_entry_block);
1075
1076 /* Update the new version's properties.
1077 Make The new version visible only within this translation unit. Make sure
1078 that is not weak also.
1079 ??? We cannot use COMDAT linkage because there is no
1080 ABI support for this. */
1081 new_version_node->make_decl_local ();
1082 DECL_VIRTUAL_P (new_version_node->decl) = 0;
1083 new_version_node->externally_visible = 0;
1084 new_version_node->local = 1;
1085 new_version_node->lowered = true;
1086 if (!implicit_section)
1087 new_version_node->set_section (*this);
1088 /* Clones of global symbols or symbols with unique names are unique. */
1089 if ((TREE_PUBLIC (old_decl)
1090 && !DECL_EXTERNAL (old_decl)
1091 && !DECL_WEAK (old_decl)
1092 && !DECL_COMDAT (old_decl))
1093 || in_lto_p)
1094 new_version_node->unique_name = true;
1095
1096 /* Update the call_expr on the edges to call the new version node. */
1097 update_call_expr (new_version: new_version_node);
1098
1099 symtab->call_cgraph_insertion_hooks (node: new_version_node);
1100 return new_version_node;
1101}
1102
1103/* Remove the node from the tree of virtual and inline clones and make it a
1104 standalone node - not a clone any more. */
1105
1106void cgraph_node::remove_from_clone_tree ()
1107{
1108 if (next_sibling_clone)
1109 next_sibling_clone->prev_sibling_clone = prev_sibling_clone;
1110 if (prev_sibling_clone)
1111 prev_sibling_clone->next_sibling_clone = next_sibling_clone;
1112 else
1113 clone_of->clones = next_sibling_clone;
1114 next_sibling_clone = NULL;
1115 prev_sibling_clone = NULL;
1116 clone_of = NULL;
1117}
1118
1119/* Given virtual clone, turn it into actual clone. */
1120
1121void
1122cgraph_node::materialize_clone ()
1123{
1124 clone_info *info = clone_info::get (node: this);
1125 clone_of->get_untransformed_body ();
1126 former_clone_of = clone_of->decl;
1127 if (clone_of->former_clone_of)
1128 former_clone_of = clone_of->former_clone_of;
1129 if (symtab->dump_file)
1130 {
1131 fprintf (stream: symtab->dump_file, format: "cloning %s to %s\n",
1132 clone_of->dump_name (),
1133 dump_name ());
1134 if (info && info->tree_map)
1135 {
1136 fprintf (stream: symtab->dump_file, format: " replace map:");
1137 for (unsigned int i = 0;
1138 i < vec_safe_length (v: info->tree_map);
1139 i++)
1140 {
1141 ipa_replace_map *replace_info;
1142 replace_info = (*info->tree_map)[i];
1143 fprintf (stream: symtab->dump_file, format: "%s %i -> ",
1144 i ? "," : "", replace_info->parm_num);
1145 print_generic_expr (symtab->dump_file,
1146 replace_info->new_tree);
1147 }
1148 fprintf (stream: symtab->dump_file, format: "\n");
1149 }
1150 if (info && info->param_adjustments)
1151 info->param_adjustments->dump (f: symtab->dump_file);
1152 }
1153 clear_stmts_in_references ();
1154 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1155 tree_function_versioning (clone_of->decl, decl,
1156 info ? info->tree_map : NULL,
1157 info ? info->param_adjustments : NULL,
1158 true, NULL, NULL);
1159 if (symtab->dump_file)
1160 {
1161 dump_function_to_file (clone_of->decl, symtab->dump_file,
1162 dump_flags);
1163 dump_function_to_file (decl, symtab->dump_file, dump_flags);
1164 }
1165
1166 cgraph_node *this_clone_of = clone_of;
1167 /* Function is no longer clone. */
1168 remove_from_clone_tree ();
1169 if (!this_clone_of->analyzed && !this_clone_of->clones)
1170 this_clone_of->release_body ();
1171}
1172
1173#include "gt-cgraphclones.h"
1174

source code of gcc/cgraphclones.cc