1/* Callgraph clones
2 Copyright (C) 2003-2025 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21/* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
24
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
27 clones
28
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
32
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
38
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
42
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
49 a given function.
50
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
54
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
61
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
66
67#include "config.h"
68#include "system.h"
69#include "coretypes.h"
70#include "backend.h"
71#include "target.h"
72#include "rtl.h"
73#include "tree.h"
74#include "gimple.h"
75#include "stringpool.h"
76#include "cgraph.h"
77#include "lto-streamer.h"
78#include "tree-eh.h"
79#include "tree-cfg.h"
80#include "tree-inline.h"
81#include "attribs.h"
82#include "dumpfile.h"
83#include "gimple-pretty-print.h"
84#include "alloc-pool.h"
85#include "symbol-summary.h"
86#include "tree-vrp.h"
87#include "sreal.h"
88#include "ipa-cp.h"
89#include "ipa-prop.h"
90#include "ipa-fnsummary.h"
91#include "symtab-thunks.h"
92#include "symtab-clones.h"
93
94/* Create clone of edge in the node N represented by CALL_EXPR
95 the callgraph. */
96
97cgraph_edge *
98cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid,
99 profile_count num, profile_count den,
100 bool update_original)
101{
102 cgraph_edge *new_edge;
103 profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
104 profile_count prof_count = count.apply_scale (num, den);
105
106 if (indirect_unknown_callee)
107 {
108 tree decl;
109
110 if (call_stmt && (decl = gimple_call_fndecl (gs: call_stmt))
111 /* When the call is speculative, we need to resolve it
112 via cgraph_resolve_speculation and not here. */
113 && !speculative)
114 {
115 cgraph_node *callee = cgraph_node::get (decl);
116 gcc_checking_assert (callee);
117 new_edge = n->create_edge (callee, call_stmt, count: prof_count, cloning_p: true);
118 }
119 else
120 {
121 new_edge = n->create_indirect_edge (call_stmt,
122 ecf_flags: indirect_info->ecf_flags,
123 count: prof_count, cloning_p: true);
124 *new_edge->indirect_info = *indirect_info;
125 }
126 }
127 else
128 {
129 new_edge = n->create_edge (callee, call_stmt, count: prof_count, cloning_p: true);
130 if (indirect_info)
131 {
132 new_edge->indirect_info
133 = ggc_cleared_alloc<cgraph_indirect_call_info> ();
134 *new_edge->indirect_info = *indirect_info;
135 }
136 }
137
138 new_edge->inline_failed = inline_failed;
139 new_edge->indirect_inlining_edge = indirect_inlining_edge;
140 if (!call_stmt)
141 new_edge->lto_stmt_uid = stmt_uid;
142 new_edge->speculative_id = speculative_id;
143 /* Clone flags that depend on call_stmt availability manually. */
144 new_edge->can_throw_external = can_throw_external;
145 new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p;
146 new_edge->speculative = speculative;
147 new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor;
148
149 /* Update IPA profile. Local profiles need no updating in original. */
150 if (update_original)
151 count = count.combine_with_ipa_count_within (ipa: count.ipa ()
152 - new_edge->count.ipa (),
153 ipa2: caller->count);
154 symtab->call_edge_duplication_hooks (cs1: this, cs2: new_edge);
155 return new_edge;
156}
157
158/* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
159 clone or its thunk. */
160
161void
162set_new_clone_decl_and_node_flags (cgraph_node *new_node)
163{
164 DECL_EXTERNAL (new_node->decl) = 0;
165 TREE_PUBLIC (new_node->decl) = 0;
166 DECL_COMDAT (new_node->decl) = 0;
167 DECL_WEAK (new_node->decl) = 0;
168 DECL_VIRTUAL_P (new_node->decl) = 0;
169 DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0;
170 DECL_STATIC_DESTRUCTOR (new_node->decl) = 0;
171 DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0);
172 DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0);
173 DECL_IS_REPLACEABLE_OPERATOR (new_node->decl) = 0;
174
175 new_node->externally_visible = 0;
176 new_node->local = 1;
177 new_node->lowered = true;
178 new_node->semantic_interposition = 0;
179}
180
181/* Duplicate thunk THUNK if necessary but make it to refer to NODE.
182 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
183 Function can return NODE if no thunk is necessary, which can happen when
184 thunk is this_adjusting but we are removing this parameter. */
185
186static cgraph_node *
187duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node)
188{
189 cgraph_node *new_thunk, *thunk_of;
190 thunk_of = thunk->callees->callee->ultimate_alias_target ();
191
192 if (thunk_of->thunk)
193 node = duplicate_thunk_for_node (thunk: thunk_of, node);
194
195 if (!DECL_ARGUMENTS (thunk->decl))
196 thunk->get_untransformed_body ();
197
198 thunk_info *i = thunk_info::get (node: thunk);
199 cgraph_edge *cs;
200 for (cs = node->callers; cs; cs = cs->next_caller)
201 if (cs->caller->thunk)
202 {
203 thunk_info *i2 = thunk_info::get (node: cs->caller);
204 if (*i2 == *i)
205 return cs->caller;
206 }
207
208 tree new_decl;
209 clone_info *info = clone_info::get (node);
210 if (info && info->param_adjustments)
211 {
212 /* We do not need to duplicate this_adjusting thunks if we have removed
213 this. */
214 if (i->this_adjusting
215 && !info->param_adjustments->first_param_intact_p ())
216 return node;
217
218 new_decl = copy_node (thunk->decl);
219 ipa_param_body_adjustments body_adj (info->param_adjustments,
220 new_decl);
221 body_adj.modify_formal_parameters ();
222 }
223 else
224 {
225 new_decl = copy_node (thunk->decl);
226 for (tree *arg = &DECL_ARGUMENTS (new_decl);
227 *arg; arg = &DECL_CHAIN (*arg))
228 {
229 tree next = DECL_CHAIN (*arg);
230 *arg = copy_node (*arg);
231 DECL_CONTEXT (*arg) = new_decl;
232 DECL_CHAIN (*arg) = next;
233 }
234 }
235
236 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl));
237 gcc_checking_assert (!DECL_INITIAL (new_decl));
238 gcc_checking_assert (!DECL_RESULT (new_decl));
239 gcc_checking_assert (!DECL_RTL_SET_P (new_decl));
240
241 DECL_NAME (new_decl) = clone_function_name_numbered (decl: thunk->decl,
242 suffix: "artificial_thunk");
243 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
244
245 /* We need to force DECL_IGNORED_P because the new thunk is created after
246 early debug was run. */
247 DECL_IGNORED_P (new_decl) = 1;
248
249 new_thunk = cgraph_node::create (decl: new_decl);
250 set_new_clone_decl_and_node_flags (new_thunk);
251 new_thunk->definition = true;
252 new_thunk->can_change_signature = node->can_change_signature;
253 new_thunk->thunk = thunk->thunk;
254 new_thunk->unique_name = in_lto_p;
255 new_thunk->former_clone_of = thunk->decl;
256 if (info && info->param_adjustments)
257 clone_info::get_create (node: new_thunk)->param_adjustments
258 = info->param_adjustments;
259 new_thunk->unit_id = thunk->unit_id;
260 new_thunk->merged_comdat = thunk->merged_comdat;
261 new_thunk->merged_extern_inline = thunk->merged_extern_inline;
262
263 cgraph_edge *e = new_thunk->create_edge (callee: node, NULL, count: new_thunk->count);
264 symtab->call_edge_duplication_hooks (cs1: thunk->callees, cs2: e);
265 symtab->call_cgraph_duplication_hooks (node: thunk, node2: new_thunk);
266 return new_thunk;
267}
268
269/* If E does not lead to a thunk, simply redirect it to N. Otherwise create
270 one or more equivalent thunks for N and redirect E to the first in the
271 chain. Note that it is then necessary to call
272 n->expand_all_artificial_thunks once all callers are redirected. */
273
274void
275cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n)
276{
277 cgraph_node *orig_to = callee->ultimate_alias_target ();
278 if (orig_to->thunk)
279 n = duplicate_thunk_for_node (thunk: orig_to, node: n);
280
281 redirect_callee (n);
282}
283
284/* Call expand_thunk on all callers that are thunks and if analyze those nodes
285 that were expanded. */
286
287void
288cgraph_node::expand_all_artificial_thunks ()
289{
290 cgraph_edge *e;
291 for (e = callers; e;)
292 if (e->caller->thunk)
293 {
294 cgraph_node *thunk = e->caller;
295
296 e = e->next_caller;
297 if (expand_thunk (thunk, false, false))
298 {
299 thunk->thunk = false;
300 thunk->analyze ();
301 ipa_analyze_node (thunk);
302 inline_analyze_function (node: thunk);
303 }
304 thunk->expand_all_artificial_thunks ();
305 }
306 else
307 e = e->next_caller;
308}
309
310/* Dump information about creation of a call graph node clone to the dump file
311 created by the -fdump-ipa-clones option. ORIGINAL is the function being
312 cloned, CLONE is the new clone. SUFFIX is a string that helps identify the
313 reason for cloning, often it is the suffix used by a particular IPA pass to
314 create unique function names. SUFFIX can be NULL and in that case the
315 dumping will not take place, which must be the case only for helper clones
316 which will never be emitted to the output. */
317
318void
319dump_callgraph_transformation (const cgraph_node *original,
320 const cgraph_node *clone,
321 const char *suffix)
322{
323 if (suffix && symtab->ipa_clones_dump_file)
324 {
325 fprintf (stream: symtab->ipa_clones_dump_file,
326 format: "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
327 original->asm_name (), original->get_uid (),
328 DECL_SOURCE_FILE (original->decl),
329 DECL_SOURCE_LINE (original->decl),
330 DECL_SOURCE_COLUMN (original->decl), clone->asm_name (),
331 clone->get_uid (), DECL_SOURCE_FILE (clone->decl),
332 DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl),
333 suffix);
334
335 symtab->cloned_nodes.add (k: original);
336 symtab->cloned_nodes.add (k: clone);
337 }
338}
339
340/* Turn profile of N to local profile. */
341
342static void
343localize_profile (cgraph_node *n)
344{
345 n->count = n->count.guessed_local ();
346 for (cgraph_edge *e = n->callees; e; e=e->next_callee)
347 {
348 e->count = e->count.guessed_local ();
349 if (!e->inline_failed)
350 localize_profile (n: e->callee);
351 }
352 for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee)
353 e->count = e->count.guessed_local ();
354}
355
356/* Create node representing clone of N executed COUNT times. Decrease
357 the execution counts from original node too.
358 The new clone will have decl set to DECL that may or may not be the same
359 as decl of N.
360
361 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
362 function's profile to reflect the fact that part of execution is handled
363 by node.
364 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
365 the new clone. Otherwise the caller is responsible for doing so later.
366
367 If the new node is being inlined into another one, NEW_INLINED_TO should be
368 the outline function the new one is (even indirectly) inlined to. All hooks
369 will see this in node's inlined_to, when invoked. Should be NULL if the
370 node is not inlined.
371
372 SUFFIX is string that is appended to the original name, it should only be
373 NULL if NEW_INLINED_TO is not NULL or if the clone being created is
374 temporary and a record about it should not be added into the ipa-clones dump
375 file.
376
377 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
378 will be overwritten by the new structure. Otherwise the new node will
379 share parameter manipulation information with the original node. */
380
381cgraph_node *
382cgraph_node::create_clone (tree new_decl, profile_count prof_count,
383 bool update_original,
384 vec<cgraph_edge *> redirect_callers,
385 bool call_duplication_hook,
386 cgraph_node *new_inlined_to,
387 ipa_param_adjustments *param_adjustments,
388 const char *suffix)
389{
390 cgraph_node *new_node = symtab->create_empty ();
391 cgraph_edge *e;
392 unsigned i;
393 profile_count old_count = count;
394 bool nonzero = count.ipa ().nonzero_p ();
395
396 if (new_inlined_to)
397 dump_callgraph_transformation (original: this, clone: new_inlined_to, suffix: "inlining to");
398
399 /* When inlining we scale precisely to prof_count, when cloning we can
400 preserve local profile. */
401 if (!new_inlined_to)
402 prof_count = count.combine_with_ipa_count (ipa: prof_count);
403 new_node->count = prof_count;
404 new_node->has_omp_variant_constructs = this->has_omp_variant_constructs;
405
406 /* Update IPA profile. Local profiles need no updating in original. */
407 if (update_original)
408 {
409 if (inlined_to)
410 count = count.combine_with_ipa_count_within (ipa: count.ipa ()
411 - prof_count.ipa (),
412 ipa2: inlined_to->count);
413 else
414 count = count.combine_with_ipa_count (ipa: count.ipa () - prof_count.ipa ());
415 }
416 new_node->decl = new_decl;
417 new_node->order = order;
418 new_node->register_symbol ();
419 new_node->lto_file_data = lto_file_data;
420 new_node->analyzed = analyzed;
421 new_node->definition = definition;
422 new_node->versionable = versionable;
423 new_node->can_change_signature = can_change_signature;
424 new_node->redefined_extern_inline = redefined_extern_inline;
425 new_node->semantic_interposition = semantic_interposition;
426 new_node->tm_may_enter_irr = tm_may_enter_irr;
427 new_node->externally_visible = false;
428 new_node->no_reorder = no_reorder;
429 new_node->local = true;
430 new_node->inlined_to = new_inlined_to;
431 new_node->rtl = rtl;
432 new_node->frequency = frequency;
433 new_node->tp_first_run = tp_first_run;
434 new_node->tm_clone = tm_clone;
435 new_node->icf_merged = icf_merged;
436 new_node->thunk = thunk;
437 new_node->unit_id = unit_id;
438 new_node->merged_comdat = merged_comdat;
439 new_node->merged_extern_inline = merged_extern_inline;
440 clone_info *info = clone_info::get (node: this);
441
442 if (param_adjustments)
443 clone_info::get_create (node: new_node)->param_adjustments = param_adjustments;
444 else if (info && info->param_adjustments)
445 clone_info::get_create (node: new_node)->param_adjustments
446 = info->param_adjustments;
447 new_node->split_part = split_part;
448
449 FOR_EACH_VEC_ELT (redirect_callers, i, e)
450 {
451 /* Redirect calls to the old version node to point to its new
452 version. The only exception is when the edge was proved to
453 be unreachable during the cloning procedure. */
454 if (!e->callee
455 || !fndecl_built_in_p (node: e->callee->decl, name1: BUILT_IN_UNREACHABLE,
456 names: BUILT_IN_UNREACHABLE_TRAP))
457 e->redirect_callee_duplicating_thunks (n: new_node);
458 }
459 new_node->expand_all_artificial_thunks ();
460
461 for (e = callees;e; e=e->next_callee)
462 e->clone (n: new_node, call_stmt: e->call_stmt, stmt_uid: e->lto_stmt_uid, num: new_node->count, den: old_count,
463 update_original);
464
465 for (e = indirect_calls; e; e = e->next_callee)
466 e->clone (n: new_node, call_stmt: e->call_stmt, stmt_uid: e->lto_stmt_uid,
467 num: new_node->count, den: old_count, update_original);
468 new_node->clone_references (node: this);
469
470 new_node->next_sibling_clone = clones;
471 if (clones)
472 clones->prev_sibling_clone = new_node;
473 clones = new_node;
474 new_node->clone_of = this;
475
476 if (call_duplication_hook)
477 symtab->call_cgraph_duplication_hooks (node: this, node2: new_node);
478 /* With partial train run we do not want to assume that original's
479 count is zero whenever we redurect all executed edges to clone.
480 Simply drop profile to local one in this case. */
481 if (update_original
482 && opt_for_fn (decl, flag_profile_partial_training)
483 && nonzero
484 && count.ipa_p ()
485 && !count.ipa ().nonzero_p ()
486 && !inlined_to)
487 localize_profile (n: this);
488
489 if (!new_inlined_to)
490 dump_callgraph_transformation (original: this, clone: new_node, suffix);
491
492 return new_node;
493}
494
495static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids;
496
497/* Return a new assembler name for a clone of decl named NAME. Apart
498 from the string SUFFIX, the new name will end with a unique (for
499 each NAME) unspecified number. If clone numbering is not needed
500 then the two argument clone_function_name should be used instead.
501 Should not be called directly except for by
502 lto-partition.cc:privatize_symbol_name_1. */
503
504tree
505clone_function_name_numbered (const char *name, const char *suffix)
506{
507 /* Initialize the function->counter mapping the first time it's
508 needed. */
509 if (!clone_fn_ids)
510 clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (size: 64);
511 unsigned int &suffix_counter = clone_fn_ids->get_or_insert (
512 IDENTIFIER_POINTER (get_identifier (name)));
513 return clone_function_name (name, suffix, number: suffix_counter++);
514}
515
516/* Return a new assembler name for a clone of DECL. Apart from string
517 SUFFIX, the new name will end with a unique (for each DECL
518 assembler name) unspecified number. If clone numbering is not
519 needed then the two argument clone_function_name should be used
520 instead. */
521
522tree
523clone_function_name_numbered (tree decl, const char *suffix)
524{
525 tree name = DECL_ASSEMBLER_NAME (decl);
526 return clone_function_name_numbered (IDENTIFIER_POINTER (name),
527 suffix);
528}
529
530/* Return a new assembler name for a clone of decl named NAME. Apart
531 from the string SUFFIX, the new name will end with the specified
532 NUMBER. If clone numbering is not needed then the two argument
533 clone_function_name should be used instead. */
534
535tree
536clone_function_name (const char *name, const char *suffix,
537 unsigned long number)
538{
539 size_t len = strlen (s: name);
540 char *tmp_name, *prefix;
541
542 prefix = XALLOCAVEC (char, len + strlen (suffix) + 2);
543 memcpy (dest: prefix, src: name, n: len);
544 strcpy (dest: prefix + len + 1, src: suffix);
545 prefix[len] = symbol_table::symbol_suffix_separator ();
546 ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number);
547 return get_identifier (tmp_name);
548}
549
550/* Return a new assembler name for a clone of DECL. Apart from the
551 string SUFFIX, the new name will end with the specified NUMBER. If
552 clone numbering is not needed then the two argument
553 clone_function_name should be used instead. */
554
555tree
556clone_function_name (tree decl, const char *suffix,
557 unsigned long number)
558{
559 return clone_function_name (
560 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number);
561}
562
563/* Return a new assembler name ending with the string SUFFIX for a
564 clone of DECL. */
565
566tree
567clone_function_name (tree decl, const char *suffix)
568{
569 tree identifier = DECL_ASSEMBLER_NAME (decl);
570 /* For consistency this needs to behave the same way as
571 ASM_FORMAT_PRIVATE_NAME does, but without the final number
572 suffix. */
573 char *separator = XALLOCAVEC (char, 2);
574 separator[0] = symbol_table::symbol_suffix_separator ();
575 separator[1] = 0;
576#if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
577 const char *prefix = "__";
578#else
579 const char *prefix = "";
580#endif
581 char *result = ACONCAT ((prefix,
582 IDENTIFIER_POINTER (identifier),
583 separator,
584 suffix,
585 (char*)0));
586 return get_identifier (result);
587}
588
589
590/* Create callgraph node clone with new declaration. The actual body will be
591 copied later at compilation stage. The name of the new clone will be
592 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
593
594 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
595 bitmap interface.
596 */
597cgraph_node *
598cgraph_node::create_virtual_clone (const vec<cgraph_edge *> &redirect_callers,
599 vec<ipa_replace_map *, va_gc> *tree_map,
600 ipa_param_adjustments *param_adjustments,
601 const char * suffix, unsigned num_suffix)
602{
603 tree old_decl = decl;
604 cgraph_node *new_node = NULL;
605 tree new_decl;
606 size_t len, i;
607 ipa_replace_map *map;
608 char *name;
609
610 gcc_checking_assert (versionable);
611 /* TODO: It would be nice if we could recognize that param_adjustments do not
612 actually perform any changes, but at the moment let's require it simply
613 does not exist. */
614 gcc_assert (can_change_signature || !param_adjustments);
615
616 /* Make a new FUNCTION_DECL tree node */
617 if (!param_adjustments)
618 new_decl = copy_node (old_decl);
619 else
620 new_decl = param_adjustments->adjust_decl (orig_decl: old_decl);
621
622 /* These pointers represent function body and will be populated only when clone
623 is materialized. */
624 gcc_assert (new_decl != old_decl);
625 DECL_STRUCT_FUNCTION (new_decl) = NULL;
626 DECL_ARGUMENTS (new_decl) = NULL;
627 DECL_INITIAL (new_decl) = NULL;
628 DECL_RESULT (new_decl) = NULL;
629 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
630 sometimes storing only clone decl instead of original. */
631
632 /* Generate a new name for the new version. */
633 len = IDENTIFIER_LENGTH (DECL_NAME (old_decl));
634 name = XALLOCAVEC (char, len + strlen (suffix) + 2);
635 memcpy (dest: name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), n: len);
636 strcpy (dest: name + len + 1, src: suffix);
637 name[len] = '.';
638 DECL_NAME (new_decl) = get_identifier (name);
639 SET_DECL_ASSEMBLER_NAME (new_decl,
640 clone_function_name (old_decl, suffix, num_suffix));
641 SET_DECL_RTL (new_decl, NULL);
642
643 new_node = create_clone (new_decl, prof_count: count, update_original: false,
644 redirect_callers, call_duplication_hook: false, NULL, param_adjustments,
645 suffix);
646
647 /* Update the properties.
648 Make clone visible only within this translation unit. Make sure
649 that is not weak also.
650 ??? We cannot use COMDAT linkage because there is no
651 ABI support for this. */
652 set_new_clone_decl_and_node_flags (new_node);
653 new_node->ipcp_clone = ipcp_clone;
654 if (tree_map)
655 clone_info::get_create (node: new_node)->tree_map = tree_map;
656 if (!implicit_section)
657 new_node->set_section (*this);
658
659 /* Clones of global symbols or symbols with unique names are unique. */
660 if ((TREE_PUBLIC (old_decl)
661 && !DECL_EXTERNAL (old_decl)
662 && !DECL_WEAK (old_decl)
663 && !DECL_COMDAT (old_decl))
664 || in_lto_p)
665 new_node->unique_name = true;
666 FOR_EACH_VEC_SAFE_ELT (tree_map, i, map)
667 {
668 tree repl = map->new_tree;
669 if (map->force_load_ref)
670 {
671 gcc_assert (TREE_CODE (repl) == ADDR_EXPR);
672 repl = get_base_address (TREE_OPERAND (repl, 0));
673 }
674 new_node->maybe_create_reference (val: repl, NULL);
675 }
676
677 if (ipa_transforms_to_apply.exists ())
678 new_node->ipa_transforms_to_apply
679 = ipa_transforms_to_apply.copy ();
680
681 symtab->call_cgraph_duplication_hooks (node: this, node2: new_node);
682
683 return new_node;
684}
685
686/* callgraph node being removed from symbol table; see if its entry can be
687 replaced by other inline clone.
688 INFO is clone info to attach to the new root. */
689cgraph_node *
690cgraph_node::find_replacement (clone_info *info)
691{
692 cgraph_node *next_inline_clone, *replacement;
693
694 for (next_inline_clone = clones;
695 next_inline_clone
696 && next_inline_clone->decl != decl;
697 next_inline_clone = next_inline_clone->next_sibling_clone)
698 ;
699
700 /* If there is inline clone of the node being removed, we need
701 to put it into the position of removed node and reorganize all
702 other clones to be based on it. */
703 if (next_inline_clone)
704 {
705 cgraph_node *n;
706 cgraph_node *new_clones;
707
708 replacement = next_inline_clone;
709
710 /* Unlink inline clone from the list of clones of removed node. */
711 if (next_inline_clone->next_sibling_clone)
712 next_inline_clone->next_sibling_clone->prev_sibling_clone
713 = next_inline_clone->prev_sibling_clone;
714 if (next_inline_clone->prev_sibling_clone)
715 {
716 gcc_assert (clones != next_inline_clone);
717 next_inline_clone->prev_sibling_clone->next_sibling_clone
718 = next_inline_clone->next_sibling_clone;
719 }
720 else
721 {
722 gcc_assert (clones == next_inline_clone);
723 clones = next_inline_clone->next_sibling_clone;
724 }
725
726 new_clones = clones;
727 clones = NULL;
728
729 /* Copy clone info. */
730 if (info)
731 *clone_info::get_create (node: next_inline_clone) = *info;
732
733 /* Now place it into clone tree at same level at NODE. */
734 next_inline_clone->clone_of = clone_of;
735 next_inline_clone->prev_sibling_clone = NULL;
736 next_inline_clone->next_sibling_clone = NULL;
737 if (clone_of)
738 {
739 if (clone_of->clones)
740 clone_of->clones->prev_sibling_clone = next_inline_clone;
741 next_inline_clone->next_sibling_clone = clone_of->clones;
742 clone_of->clones = next_inline_clone;
743 }
744
745 /* Merge the clone list. */
746 if (new_clones)
747 {
748 if (!next_inline_clone->clones)
749 next_inline_clone->clones = new_clones;
750 else
751 {
752 n = next_inline_clone->clones;
753 while (n->next_sibling_clone)
754 n = n->next_sibling_clone;
755 n->next_sibling_clone = new_clones;
756 new_clones->prev_sibling_clone = n;
757 }
758 }
759
760 /* Update clone_of pointers. */
761 n = new_clones;
762 while (n)
763 {
764 n->clone_of = next_inline_clone;
765 n = n->next_sibling_clone;
766 }
767
768 /* Update order in order to be able to find a LTO section
769 with function body. */
770 replacement->order = order;
771
772 return replacement;
773 }
774 else
775 return NULL;
776}
777
778/* Like cgraph_set_call_stmt but walk the clone tree and update all
779 clones sharing the same function body.
780 When WHOLE_SPECULATIVE_EDGES is true, all three components of
781 speculative edge gets updated. Otherwise we update only direct
782 call. */
783
784void
785cgraph_node::set_call_stmt_including_clones (gimple *old_stmt,
786 gcall *new_stmt,
787 bool update_speculative)
788{
789 cgraph_node *node;
790 cgraph_edge *master_edge = get_edge (call_stmt: old_stmt);
791
792 if (master_edge)
793 cgraph_edge::set_call_stmt (e: master_edge, new_stmt, update_speculative);
794
795 node = clones;
796 if (node)
797 while (node != this)
798 {
799 cgraph_edge *edge = node->get_edge (call_stmt: old_stmt);
800 if (edge)
801 {
802 edge = cgraph_edge::set_call_stmt (e: edge, new_stmt,
803 update_speculative);
804 /* If UPDATE_SPECULATIVE is false, it means that we are turning
805 speculative call into a real code sequence. Update the
806 callgraph edges. */
807 if (edge->speculative && !update_speculative)
808 {
809 cgraph_edge *indirect = edge->speculative_call_indirect_edge ();
810
811 for (cgraph_edge *next, *direct
812 = edge->first_speculative_call_target ();
813 direct;
814 direct = next)
815 {
816 next = direct->next_speculative_call_target ();
817 direct->speculative_call_target_ref ()->speculative = false;
818 direct->speculative = false;
819 }
820 indirect->speculative = false;
821 }
822 }
823 if (node->clones)
824 node = node->clones;
825 else if (node->next_sibling_clone)
826 node = node->next_sibling_clone;
827 else
828 {
829 while (node != this && !node->next_sibling_clone)
830 node = node->clone_of;
831 if (node != this)
832 node = node->next_sibling_clone;
833 }
834 }
835}
836
837/* Like cgraph_create_edge walk the clone tree and update all clones sharing
838 same function body. If clones already have edge for OLD_STMT; only
839 update the edge same way as cgraph_set_call_stmt_including_clones does.
840
841 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
842 frequencies of the clones. */
843
844void
845cgraph_node::create_edge_including_clones (cgraph_node *callee,
846 gimple *old_stmt, gcall *stmt,
847 profile_count count,
848 cgraph_inline_failed_t reason)
849{
850 cgraph_node *node;
851
852 if (!get_edge (call_stmt: stmt))
853 {
854 cgraph_edge *edge = create_edge (callee, call_stmt: stmt, count);
855 edge->inline_failed = reason;
856 }
857
858 node = clones;
859 if (node)
860 while (node != this)
861 /* Thunk clones do not get updated while copying inline function body. */
862 if (!node->thunk)
863 {
864 cgraph_edge *edge = node->get_edge (call_stmt: old_stmt);
865
866 /* It is possible that clones already contain the edge while
867 master didn't. Either we promoted indirect call into direct
868 call in the clone or we are processing clones of unreachable
869 master where edges has been removed. */
870 if (edge)
871 edge = cgraph_edge::set_call_stmt (e: edge, new_stmt: stmt);
872 else if (! node->get_edge (call_stmt: stmt))
873 {
874 edge = node->create_edge (callee, call_stmt: stmt, count);
875 edge->inline_failed = reason;
876 }
877
878 if (node->clones)
879 node = node->clones;
880 else if (node->next_sibling_clone)
881 node = node->next_sibling_clone;
882 else
883 {
884 while (node != this && !node->next_sibling_clone)
885 node = node->clone_of;
886 if (node != this)
887 node = node->next_sibling_clone;
888 }
889 }
890}
891
892/* Remove the node from cgraph and all inline clones inlined into it.
893 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
894 removed. This allows to call the function from outer loop walking clone
895 tree. */
896
897bool
898cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node)
899{
900 cgraph_edge *e, *next;
901 bool found = false;
902
903 if (this == forbidden_node)
904 {
905 cgraph_edge::remove (edge: callers);
906 return true;
907 }
908 for (e = callees; e; e = next)
909 {
910 next = e->next_callee;
911 if (!e->inline_failed)
912 found |= e->callee->remove_symbol_and_inline_clones (forbidden_node);
913 }
914 remove ();
915 return found;
916}
917
918/* The edges representing the callers of the NEW_VERSION node were
919 fixed by cgraph_function_versioning (), now the call_expr in their
920 respective tree code should be updated to call the NEW_VERSION. */
921
922static void
923update_call_expr (cgraph_node *new_version)
924{
925 cgraph_edge *e;
926
927 gcc_assert (new_version);
928
929 /* Update the call expr on the edges to call the new version. */
930 for (e = new_version->callers; e; e = e->next_caller)
931 {
932 function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl);
933 gimple_call_set_fndecl (gs: e->call_stmt, decl: new_version->decl);
934 maybe_clean_eh_stmt_fn (inner_function, e->call_stmt);
935 }
936}
937
938
939/* Create a new cgraph node which is the new version of
940 callgraph node. REDIRECT_CALLERS holds the callers
941 edges which should be redirected to point to
942 NEW_VERSION. ALL the callees edges of the node
943 are cloned to the new version node. Return the new
944 version node.
945
946 If non-NULL BLOCK_TO_COPY determine what basic blocks
947 was copied to prevent duplications of calls that are dead
948 in the clone. */
949
950cgraph_node *
951cgraph_node::create_version_clone (tree new_decl,
952 vec<cgraph_edge *> redirect_callers,
953 bitmap bbs_to_copy,
954 const char *suffix)
955 {
956 cgraph_node *new_version;
957 cgraph_edge *e;
958 unsigned i;
959
960 new_version = cgraph_node::create (decl: new_decl);
961
962 new_version->analyzed = analyzed;
963 new_version->definition = definition;
964 new_version->local = local;
965 new_version->externally_visible = false;
966 new_version->no_reorder = no_reorder;
967 new_version->local = new_version->definition;
968 new_version->inlined_to = inlined_to;
969 new_version->rtl = rtl;
970 new_version->count = count;
971 new_version->unit_id = unit_id;
972 new_version->merged_comdat = merged_comdat;
973 new_version->merged_extern_inline = merged_extern_inline;
974
975 for (e = callees; e; e=e->next_callee)
976 if (!bbs_to_copy
977 || bitmap_bit_p (bbs_to_copy, gimple_bb (g: e->call_stmt)->index))
978 e->clone (n: new_version, call_stmt: e->call_stmt,
979 stmt_uid: e->lto_stmt_uid, num: count, den: count,
980 update_original: true);
981 for (e = indirect_calls; e; e=e->next_callee)
982 if (!bbs_to_copy
983 || bitmap_bit_p (bbs_to_copy, gimple_bb (g: e->call_stmt)->index))
984 e->clone (n: new_version, call_stmt: e->call_stmt,
985 stmt_uid: e->lto_stmt_uid, num: count, den: count,
986 update_original: true);
987 FOR_EACH_VEC_ELT (redirect_callers, i, e)
988 {
989 /* Redirect calls to the old version node to point to its new
990 version. */
991 e->redirect_callee (n: new_version);
992 }
993
994 dump_callgraph_transformation (original: this, clone: new_version, suffix);
995
996 return new_version;
997 }
998
999/* Perform function versioning.
1000 Function versioning includes copying of the tree and
1001 a callgraph update (creating a new cgraph node and updating
1002 its callees and callers).
1003
1004 REDIRECT_CALLERS varray includes the edges to be redirected
1005 to the new version.
1006
1007 TREE_MAP is a mapping of tree nodes we want to replace with
1008 new ones (according to results of prior analysis).
1009
1010 If non-NULL PARAM_ADJUSTMENTS determine how function formal parameters
1011 should be modified in the new version and if it should return void.
1012 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
1013 If non_NULL NEW_ENTRY determine new entry BB of the clone.
1014 SUFFIX is a string that will be used to create a new name for the new
1015 function.
1016
1017 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
1018 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
1019 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
1020 of the declaration.
1021
1022 If VERSION_DECL is set true, use clone_function_name_numbered for the
1023 function clone. Otherwise, use clone_function_name.
1024
1025 Return the new version's cgraph node. */
1026
1027cgraph_node *
1028cgraph_node::create_version_clone_with_body
1029 (vec<cgraph_edge *> redirect_callers,
1030 vec<ipa_replace_map *, va_gc> *tree_map,
1031 ipa_param_adjustments *param_adjustments,
1032 bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix,
1033 tree target_attributes, bool version_decl)
1034{
1035 tree old_decl = decl;
1036 cgraph_node *new_version_node = NULL;
1037 tree new_decl;
1038
1039 if (!tree_versionable_function_p (old_decl))
1040 return NULL;
1041
1042 /* TODO: Restore an assert that we do not change signature if
1043 can_change_signature is false. We cannot just check that
1044 param_adjustments is NULL because unfortunately ipa-split removes return
1045 values from such functions. */
1046
1047 /* Make a new FUNCTION_DECL tree node for the new version. */
1048 if (param_adjustments)
1049 new_decl = param_adjustments->adjust_decl (orig_decl: old_decl);
1050 else
1051 new_decl = copy_node (old_decl);
1052
1053 /* Generate a new name for the new version. */
1054 tree fnname = (version_decl ? clone_function_name_numbered (decl: old_decl, suffix)
1055 : clone_function_name (decl: old_decl, suffix));
1056 DECL_NAME (new_decl) = fnname;
1057 SET_DECL_ASSEMBLER_NAME (new_decl, fnname);
1058 SET_DECL_RTL (new_decl, NULL);
1059
1060 DECL_VIRTUAL_P (new_decl) = 0;
1061
1062 if (target_attributes)
1063 {
1064 DECL_ATTRIBUTES (new_decl) = target_attributes;
1065
1066 location_t saved_loc = input_location;
1067 tree v = TREE_VALUE (target_attributes);
1068 input_location = DECL_SOURCE_LOCATION (new_decl);
1069 bool r;
1070 tree name_id = get_attribute_name (target_attributes);
1071 const char *name_str = IDENTIFIER_POINTER (name_id);
1072 if (strcmp (s1: name_str, s2: "target") == 0)
1073 r = targetm.target_option.valid_attribute_p (new_decl, name_id, v, 1);
1074 else if (strcmp (s1: name_str, s2: "target_version") == 0)
1075 r = targetm.target_option.valid_version_attribute_p (new_decl, name_id,
1076 v, 1);
1077 else
1078 gcc_unreachable();
1079
1080 input_location = saved_loc;
1081 if (!r)
1082 return NULL;
1083 }
1084
1085 /* When the old decl was a con-/destructor make sure the clone isn't. */
1086 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
1087 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
1088 DECL_SET_IS_OPERATOR_NEW (new_decl, 0);
1089 DECL_SET_IS_OPERATOR_DELETE (new_decl, 0);
1090 DECL_IS_REPLACEABLE_OPERATOR (new_decl) = 0;
1091
1092 /* Create the new version's call-graph node.
1093 and update the edges of the new node. */
1094 new_version_node = create_version_clone (new_decl, redirect_callers,
1095 bbs_to_copy, suffix);
1096
1097 if (ipa_transforms_to_apply.exists ())
1098 new_version_node->ipa_transforms_to_apply
1099 = ipa_transforms_to_apply.copy ();
1100 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1101 tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments,
1102 false, bbs_to_copy, new_entry_block);
1103
1104 /* Update the new version's properties.
1105 Make The new version visible only within this translation unit. Make sure
1106 that is not weak also.
1107 ??? We cannot use COMDAT linkage because there is no
1108 ABI support for this. */
1109 new_version_node->make_decl_local ();
1110 DECL_VIRTUAL_P (new_version_node->decl) = 0;
1111 new_version_node->externally_visible = 0;
1112 new_version_node->local = 1;
1113 new_version_node->lowered = true;
1114 if (!implicit_section)
1115 new_version_node->set_section (*this);
1116 /* Clones of global symbols or symbols with unique names are unique. */
1117 if ((TREE_PUBLIC (old_decl)
1118 && !DECL_EXTERNAL (old_decl)
1119 && !DECL_WEAK (old_decl)
1120 && !DECL_COMDAT (old_decl))
1121 || in_lto_p)
1122 new_version_node->unique_name = true;
1123
1124 /* Update the call_expr on the edges to call the new version node. */
1125 update_call_expr (new_version: new_version_node);
1126
1127 symtab->call_cgraph_insertion_hooks (node: new_version_node);
1128 return new_version_node;
1129}
1130
1131/* Remove the node from the tree of virtual and inline clones and make it a
1132 standalone node - not a clone any more. */
1133
1134void cgraph_node::remove_from_clone_tree ()
1135{
1136 if (next_sibling_clone)
1137 next_sibling_clone->prev_sibling_clone = prev_sibling_clone;
1138 if (prev_sibling_clone)
1139 prev_sibling_clone->next_sibling_clone = next_sibling_clone;
1140 else
1141 clone_of->clones = next_sibling_clone;
1142 next_sibling_clone = NULL;
1143 prev_sibling_clone = NULL;
1144 clone_of = NULL;
1145}
1146
1147/* Given virtual clone, turn it into actual clone. */
1148
1149void
1150cgraph_node::materialize_clone ()
1151{
1152 clone_info *info = clone_info::get (node: this);
1153 clone_of->get_untransformed_body ();
1154 former_clone_of = clone_of->decl;
1155 if (clone_of->former_clone_of)
1156 former_clone_of = clone_of->former_clone_of;
1157 if (symtab->dump_file)
1158 {
1159 fprintf (stream: symtab->dump_file, format: "cloning %s to %s\n",
1160 clone_of->dump_name (),
1161 dump_name ());
1162 if (info && info->tree_map)
1163 {
1164 fprintf (stream: symtab->dump_file, format: " replace map:");
1165 for (unsigned int i = 0;
1166 i < vec_safe_length (v: info->tree_map);
1167 i++)
1168 {
1169 ipa_replace_map *replace_info;
1170 replace_info = (*info->tree_map)[i];
1171 fprintf (stream: symtab->dump_file, format: "%s %i -> ",
1172 i ? "," : "", replace_info->parm_num);
1173 print_generic_expr (symtab->dump_file,
1174 replace_info->new_tree);
1175 }
1176 fprintf (stream: symtab->dump_file, format: "\n");
1177 }
1178 if (info && info->param_adjustments)
1179 info->param_adjustments->dump (f: symtab->dump_file);
1180 }
1181 clear_stmts_in_references ();
1182 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1183 tree_function_versioning (clone_of->decl, decl,
1184 info ? info->tree_map : NULL,
1185 info ? info->param_adjustments : NULL,
1186 true, NULL, NULL);
1187 if (symtab->dump_file)
1188 {
1189 dump_function_to_file (clone_of->decl, symtab->dump_file,
1190 dump_flags);
1191 dump_function_to_file (decl, symtab->dump_file, dump_flags);
1192 }
1193
1194 cgraph_node *this_clone_of = clone_of;
1195 /* Function is no longer clone. */
1196 remove_from_clone_tree ();
1197 if (!this_clone_of->analyzed && !this_clone_of->clones)
1198 this_clone_of->release_body ();
1199}
1200
1201#include "gt-cgraphclones.h"
1202

source code of gcc/cgraphclones.cc