1	/* Tree inlining.
2	Copyright (C) 2001-2023 Free Software Foundation, Inc.
3	Contributed by Alexandre Oliva <aoliva@redhat.com>
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify
8	it under the terms of the GNU General Public License as published by
9	the Free Software Foundation; either version 3, or (at your option)
10	any later version.
11
12	GCC is distributed in the hope that it will be useful,
13	but WITHOUT ANY WARRANTY; without even the implied warranty of
14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	GNU General Public License for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21	#include "config.h"
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "target.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "gimple.h"
29	#include "cfghooks.h"
30	#include "tree-pass.h"
31	#include "ssa.h"
32	#include "cgraph.h"
33	#include "tree-pretty-print.h"
34	#include "diagnostic-core.h"
35	#include "gimple-predict.h"
36	#include "fold-const.h"
37	#include "stor-layout.h"
38	#include "calls.h"
39	#include "tree-inline.h"
40	#include "langhooks.h"
41	#include "cfganal.h"
42	#include "tree-iterator.h"
43	#include "intl.h"
44	#include "gimple-iterator.h"
45	#include "gimple-fold.h"
46	#include "tree-eh.h"
47	#include "gimplify.h"
48	#include "gimplify-me.h"
49	#include "gimple-walk.h"
50	#include "tree-cfg.h"
51	#include "tree-into-ssa.h"
52	#include "tree-dfa.h"
53	#include "tree-ssa.h"
54	#include "except.h"
55	#include "debug.h"
56	#include "value-prof.h"
57	#include "cfgloop.h"
58	#include "builtins.h"
59	#include "stringpool.h"
60	#include "attribs.h"
61	#include "sreal.h"
62	#include "tree-cfgcleanup.h"
63	#include "tree-ssa-live.h"
64	#include "alloc-pool.h"
65	#include "symbol-summary.h"
66	#include "symtab-thunks.h"
67	#include "symtab-clones.h"
68
69	/* I'm not real happy about this, but we need to handle gimple and
70	non-gimple trees. */
71
72	/* Inlining, Cloning, Versioning, Parallelization
73
74	Inlining: a function body is duplicated, but the PARM_DECLs are
75	remapped into VAR_DECLs, and non-void RETURN_EXPRs become
76	MODIFY_EXPRs that store to a dedicated returned-value variable.
77	The duplicated eh_region info of the copy will later be appended
78	to the info for the caller; the eh_region info in copied throwing
79	statements and RESX statements are adjusted accordingly.
80
81	Cloning: (only in C++) We have one body for a con/de/structor, and
82	multiple function decls, each with a unique parameter list.
83	Duplicate the body, using the given splay tree; some parameters
84	will become constants (like 0 or 1).
85
86	Versioning: a function body is duplicated and the result is a new
87	function rather than into blocks of an existing function as with
88	inlining. Some parameters will become constants.
89
90	Parallelization: a region of a function is duplicated resulting in
91	a new function. Variables may be replaced with complex expressions
92	to enable shared variable semantics.
93
94	All of these will simultaneously lookup any callgraph edges. If
95	we're going to inline the duplicated function body, and the given
96	function has some cloned callgraph nodes (one for each place this
97	function will be inlined) those callgraph edges will be duplicated.
98	If we're cloning the body, those callgraph edges will be
99	updated to point into the new body. (Note that the original
100	callgraph node and edge list will not be altered.)
101
102	See the CALL_EXPR handling case in copy_tree_body_r (). */
103
104	/* To Do:
105
106	o In order to make inlining-on-trees work, we pessimized
107	function-local static constants. In particular, they are now
108	always output, even when not addressed. Fix this by treating
109	function-local static constants just like global static
110	constants; the back-end already knows not to output them if they
111	are not needed.
112
113	o Provide heuristics to clamp inlining of recursive template
114	calls? */
115
116
117	/* Weights that estimate_num_insns uses to estimate the size of the
118	produced code. */
119
120	eni_weights eni_size_weights;
121
122	/* Weights that estimate_num_insns uses to estimate the time necessary
123	to execute the produced code. */
124
125	eni_weights eni_time_weights;
126
127	/* Prototypes. */
128
129	static tree declare_return_variable (copy_body_data *, tree, tree,
130	basic_block);
131	static void remap_block (tree *, copy_body_data *);
132	static void copy_bind_expr (tree *, int *, copy_body_data *);
133	static void declare_inline_vars (tree, tree);
134	static void remap_save_expr (tree *, hash_map<tree, tree> *, int *);
135	static void prepend_lexical_block (tree current_block, tree new_block);
136	static tree copy_result_decl_to_var (tree, copy_body_data *);
137	static tree copy_decl_maybe_to_var (tree, copy_body_data *);
138	static gimple_seq remap_gimple_stmt (gimple *, copy_body_data *);
139	static void insert_init_stmt (copy_body_data *, basic_block, gimple *);
140
141	/* Insert a tree->tree mapping for ID. Despite the name suggests
142	that the trees should be variables, it is used for more than that. */
143
144	void
145	insert_decl_map (copy_body_data *id, tree key, tree value)
146	{
147	id->decl_map->put (k: key, v: value);
148
149	/* Always insert an identity map as well. If we see this same new
150	node again, we won't want to duplicate it a second time. */
151	if (key != value && value)
152	id->decl_map->put (k: value, v: value);
153	}
154
155	/* If nonzero, we're remapping the contents of inlined debug
156	statements. If negative, an error has occurred, such as a
157	reference to a variable that isn't available in the inlined
158	context. */
159	static int processing_debug_stmt = 0;
160
161	/* Construct new SSA name for old NAME. ID is the inline context. */
162
163	static tree
164	remap_ssa_name (tree name, copy_body_data *id)
165	{
166	tree new_tree, var;
167	tree *n;
168
169	gcc_assert (TREE_CODE (name) == SSA_NAME);
170
171	n = id->decl_map->get (k: name);
172	if (n)
173	{
174	/* When we perform edge redirection as part of CFG copy, IPA-SRA can
175	remove an unused LHS from a call statement. Such LHS can however
176	still appear in debug statements, but their value is lost in this
177	function and we do not want to map them. */
178	if (id->killed_new_ssa_names
179	&& id->killed_new_ssa_names->contains (k: *n))
180	{
181	gcc_assert (processing_debug_stmt);
182	processing_debug_stmt = -1;
183	return name;
184	}
185
186	return unshare_expr (*n);
187	}
188
189	if (processing_debug_stmt)
190	{
191	if (SSA_NAME_IS_DEFAULT_DEF (name)
192	&& TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
193	&& id->entry_bb == NULL
194	&& single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
195	{
196	gimple *def_temp;
197	gimple_stmt_iterator gsi;
198	tree val = SSA_NAME_VAR (name);
199
200	n = id->decl_map->get (k: val);
201	if (n != NULL)
202	val = *n;
203	if (TREE_CODE (val) != PARM_DECL
204	&& !(VAR_P (val) && DECL_ABSTRACT_ORIGIN (val)))
205	{
206	processing_debug_stmt = -1;
207	return name;
208	}
209	n = id->decl_map->get (k: val);
210	if (n && TREE_CODE (*n) == DEBUG_EXPR_DECL)
211	return *n;
212	tree vexpr = build_debug_expr_decl (TREE_TYPE (name));
213	/* FIXME: Is setting the mode really necessary? */
214	SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (name)));
215	def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
216	gsi = gsi_after_labels (bb: single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
217	gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
218	insert_decl_map (id, key: val, value: vexpr);
219	return vexpr;
220	}
221
222	processing_debug_stmt = -1;
223	return name;
224	}
225
226	/* Remap anonymous SSA names or SSA names of anonymous decls. */
227	var = SSA_NAME_VAR (name);
228	if (!var
229	|| (!SSA_NAME_IS_DEFAULT_DEF (name)
230	&& VAR_P (var)
231	&& !VAR_DECL_IS_VIRTUAL_OPERAND (var)
232	&& DECL_ARTIFICIAL (var)
233	&& DECL_IGNORED_P (var)
234	&& !DECL_NAME (var)))
235	{
236	struct ptr_info_def *pi;
237	new_tree = make_ssa_name (var: remap_type (TREE_TYPE (name), id));
238	if (!var && SSA_NAME_IDENTIFIER (name))
239	SET_SSA_NAME_VAR_OR_IDENTIFIER (new_tree, SSA_NAME_IDENTIFIER (name));
240	insert_decl_map (id, key: name, value: new_tree);
241	SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
242	= SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
243	/* At least IPA points-to info can be directly transferred. */
244	if (id->src_cfun->gimple_df
245	&& id->src_cfun->gimple_df->ipa_pta
246	&& POINTER_TYPE_P (TREE_TYPE (name))
247	&& (pi = SSA_NAME_PTR_INFO (name))
248	&& !pi->pt.anything)
249	{
250	struct ptr_info_def *new_pi = get_ptr_info (new_tree);
251	new_pi->pt = pi->pt;
252	}
253	/* So can range-info. */
254	if (!POINTER_TYPE_P (TREE_TYPE (name))
255	&& SSA_NAME_RANGE_INFO (name))
256	duplicate_ssa_name_range_info (dest: new_tree, src: name);
257	return new_tree;
258	}
259
260	/* Do not set DEF_STMT yet as statement is not copied yet. We do that
261	in copy_bb. */
262	new_tree = remap_decl (decl: var, id);
263
264	/* We might've substituted constant or another SSA_NAME for
265	the variable.
266
267	Replace the SSA name representing RESULT_DECL by variable during
268	inlining: this saves us from need to introduce PHI node in a case
269	return value is just partly initialized. */
270	if ((VAR_P (new_tree) || TREE_CODE (new_tree) == PARM_DECL)
271	&& (!SSA_NAME_VAR (name)
272	|| TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL
273	|| !id->transform_return_to_modify))
274	{
275	struct ptr_info_def *pi;
276	new_tree = make_ssa_name (var: new_tree);
277	insert_decl_map (id, key: name, value: new_tree);
278	SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
279	= SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
280	/* At least IPA points-to info can be directly transferred. */
281	if (id->src_cfun->gimple_df
282	&& id->src_cfun->gimple_df->ipa_pta
283	&& POINTER_TYPE_P (TREE_TYPE (name))
284	&& (pi = SSA_NAME_PTR_INFO (name))
285	&& !pi->pt.anything)
286	{
287	struct ptr_info_def *new_pi = get_ptr_info (new_tree);
288	new_pi->pt = pi->pt;
289	}
290	/* So can range-info. */
291	if (!POINTER_TYPE_P (TREE_TYPE (name))
292	&& SSA_NAME_RANGE_INFO (name))
293	duplicate_ssa_name_range_info (dest: new_tree, src: name);
294	if (SSA_NAME_IS_DEFAULT_DEF (name))
295	{
296	/* By inlining function having uninitialized variable, we might
297	extend the lifetime (variable might get reused). This cause
298	ICE in the case we end up extending lifetime of SSA name across
299	abnormal edge, but also increase register pressure.
300
301	We simply initialize all uninitialized vars by 0 except
302	for case we are inlining to very first BB. We can avoid
303	this for all BBs that are not inside strongly connected
304	regions of the CFG, but this is expensive to test. */
305	if (id->entry_bb
306	&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name)
307	&& (!SSA_NAME_VAR (name)
308	|| TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL)
309	&& (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun),
310	0)->dest
311	|| EDGE_COUNT (id->entry_bb->preds) != 1))
312	{
313	gimple_stmt_iterator gsi = gsi_last_bb (bb: id->entry_bb);
314	gimple *init_stmt;
315	tree zero = build_zero_cst (TREE_TYPE (new_tree));
316
317	init_stmt = gimple_build_assign (new_tree, zero);
318	gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT);
319	SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0;
320	}
321	else
322	{
323	SSA_NAME_DEF_STMT (new_tree) = gimple_build_nop ();
324	set_ssa_default_def (cfun, SSA_NAME_VAR (new_tree), new_tree);
325	}
326	}
327	}
328	else
329	insert_decl_map (id, key: name, value: new_tree);
330	return new_tree;
331	}
332
333	/* Remap DECL during the copying of the BLOCK tree for the function. */
334
335	tree
336	remap_decl (tree decl, copy_body_data *id)
337	{
338	tree *n;
339
340	/* We only remap local variables in the current function. */
341
342	/* See if we have remapped this declaration. */
343
344	n = id->decl_map->get (k: decl);
345
346	if (!n && processing_debug_stmt)
347	{
348	processing_debug_stmt = -1;
349	return decl;
350	}
351
352	/* When remapping a type within copy_gimple_seq_and_replace_locals, all
353	necessary DECLs have already been remapped and we do not want to duplicate
354	a decl coming from outside of the sequence we are copying. */
355	if (!n
356	&& id->prevent_decl_creation_for_types
357	&& id->remapping_type_depth > 0
358	&& (VAR_P (decl) || TREE_CODE (decl) == PARM_DECL))
359	return decl;
360
361	/* If we didn't already have an equivalent for this declaration, create one
362	now. */
363	if (!n)
364	{
365	/* Make a copy of the variable or label. */
366	tree t = id->copy_decl (decl, id);
367
368	/* Remember it, so that if we encounter this local entity again
369	we can reuse this copy. Do this early because remap_type may
370	need this decl for TYPE_STUB_DECL. */
371	insert_decl_map (id, key: decl, value: t);
372
373	if (!DECL_P (t))
374	return t;
375
376	/* Remap types, if necessary. */
377	TREE_TYPE (t) = remap_type (TREE_TYPE (t), id);
378	if (TREE_CODE (t) == TYPE_DECL)
379	{
380	DECL_ORIGINAL_TYPE (t) = remap_type (DECL_ORIGINAL_TYPE (t), id);
381
382	/* Preserve the invariant that DECL_ORIGINAL_TYPE != TREE_TYPE,
383	which is enforced in gen_typedef_die when DECL_ABSTRACT_ORIGIN
384	is not set on the TYPE_DECL, for example in LTO mode. */
385	if (DECL_ORIGINAL_TYPE (t) == TREE_TYPE (t))
386	{
387	tree x = build_variant_type_copy (TREE_TYPE (t));
388	TYPE_STUB_DECL (x) = TYPE_STUB_DECL (TREE_TYPE (t));
389	TYPE_NAME (x) = TYPE_NAME (TREE_TYPE (t));
390	DECL_ORIGINAL_TYPE (t) = x;
391	}
392	}
393
394	/* Remap sizes as necessary. */
395	walk_tree (&DECL_SIZE (t), copy_tree_body_r, id, NULL);
396	walk_tree (&DECL_SIZE_UNIT (t), copy_tree_body_r, id, NULL);
397
398	/* If fields, do likewise for offset and qualifier. */
399	if (TREE_CODE (t) == FIELD_DECL)
400	{
401	walk_tree (&DECL_FIELD_OFFSET (t), copy_tree_body_r, id, NULL);
402	if (TREE_CODE (DECL_CONTEXT (t)) == QUAL_UNION_TYPE)
403	walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL);
404	}
405
406	return t;
407	}
408
409	if (id->do_not_unshare)
410	return *n;
411	else
412	return unshare_expr (*n);
413	}
414
415	static tree
416	remap_type_1 (tree type, copy_body_data *id)
417	{
418	tree new_tree, t;
419
420	/* We do need a copy. build and register it now. If this is a pointer or
421	reference type, remap the designated type and make a new pointer or
422	reference type. */
423	if (TREE_CODE (type) == POINTER_TYPE)
424	{
425	new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id),
426	TYPE_MODE (type),
427	TYPE_REF_CAN_ALIAS_ALL (type));
428	if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
429	new_tree = build_type_attribute_qual_variant (new_tree,
430	TYPE_ATTRIBUTES (type),
431	TYPE_QUALS (type));
432	insert_decl_map (id, key: type, value: new_tree);
433	return new_tree;
434	}
435	else if (TREE_CODE (type) == REFERENCE_TYPE)
436	{
437	new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id),
438	TYPE_MODE (type),
439	TYPE_REF_CAN_ALIAS_ALL (type));
440	if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
441	new_tree = build_type_attribute_qual_variant (new_tree,
442	TYPE_ATTRIBUTES (type),
443	TYPE_QUALS (type));
444	insert_decl_map (id, key: type, value: new_tree);
445	return new_tree;
446	}
447	else
448	new_tree = copy_node (type);
449
450	insert_decl_map (id, key: type, value: new_tree);
451
452	/* This is a new type, not a copy of an old type. Need to reassociate
453	variants. We can handle everything except the main variant lazily. */
454	t = TYPE_MAIN_VARIANT (type);
455	if (type != t)
456	{
457	t = remap_type (type: t, id);
458	TYPE_MAIN_VARIANT (new_tree) = t;
459	TYPE_NEXT_VARIANT (new_tree) = TYPE_NEXT_VARIANT (t);
460	TYPE_NEXT_VARIANT (t) = new_tree;
461	}
462	else
463	{
464	TYPE_MAIN_VARIANT (new_tree) = new_tree;
465	TYPE_NEXT_VARIANT (new_tree) = NULL;
466	}
467
468	if (TYPE_STUB_DECL (type))
469	TYPE_STUB_DECL (new_tree) = remap_decl (TYPE_STUB_DECL (type), id);
470
471	/* Lazily create pointer and reference types. */
472	TYPE_POINTER_TO (new_tree) = NULL;
473	TYPE_REFERENCE_TO (new_tree) = NULL;
474
475	/* Copy all types that may contain references to local variables; be sure to
476	preserve sharing in between type and its main variant when possible. */
477	switch (TREE_CODE (new_tree))
478	{
479	case INTEGER_TYPE:
480	case REAL_TYPE:
481	case FIXED_POINT_TYPE:
482	case ENUMERAL_TYPE:
483	case BOOLEAN_TYPE:
484	if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
485	{
486	gcc_checking_assert (TYPE_MIN_VALUE (type) == TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (type)));
487	gcc_checking_assert (TYPE_MAX_VALUE (type) == TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (type)));
488
489	TYPE_MIN_VALUE (new_tree) = TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (new_tree));
490	TYPE_MAX_VALUE (new_tree) = TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (new_tree));
491	}
492	else
493	{
494	t = TYPE_MIN_VALUE (new_tree);
495	if (t && TREE_CODE (t) != INTEGER_CST)
496	walk_tree (&TYPE_MIN_VALUE (new_tree), copy_tree_body_r, id, NULL);
497
498	t = TYPE_MAX_VALUE (new_tree);
499	if (t && TREE_CODE (t) != INTEGER_CST)
500	walk_tree (&TYPE_MAX_VALUE (new_tree), copy_tree_body_r, id, NULL);
501	}
502	return new_tree;
503
504	case FUNCTION_TYPE:
505	if (TYPE_MAIN_VARIANT (new_tree) != new_tree
506	&& TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
507	TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
508	else
509	TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
510	if (TYPE_MAIN_VARIANT (new_tree) != new_tree
511	&& TYPE_ARG_TYPES (type) == TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (type)))
512	TYPE_ARG_TYPES (new_tree) = TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (new_tree));
513	else
514	walk_tree (&TYPE_ARG_TYPES (new_tree), copy_tree_body_r, id, NULL);
515	return new_tree;
516
517	case ARRAY_TYPE:
518	if (TYPE_MAIN_VARIANT (new_tree) != new_tree
519	&& TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
520	TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
521	else
522	TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
523
524	if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
525	{
526	gcc_checking_assert (TYPE_DOMAIN (type)
527	== TYPE_DOMAIN (TYPE_MAIN_VARIANT (type)));
528	TYPE_DOMAIN (new_tree) = TYPE_DOMAIN (TYPE_MAIN_VARIANT (new_tree));
529	}
530	else
531	{
532	TYPE_DOMAIN (new_tree) = remap_type (TYPE_DOMAIN (new_tree), id);
533	/* For array bounds where we have decided not to copy over the bounds
534	variable which isn't used in OpenMP/OpenACC region, change them to
535	an uninitialized VAR_DECL temporary. */
536	if (id->adjust_array_error_bounds
537	&& TYPE_DOMAIN (new_tree)
538	&& TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) == error_mark_node
539	&& TYPE_MAX_VALUE (TYPE_DOMAIN (type)) != error_mark_node)
540	{
541	tree v = create_tmp_var (TREE_TYPE (TYPE_DOMAIN (new_tree)));
542	DECL_ATTRIBUTES (v)
543	= tree_cons (get_identifier ("omp dummy var"), NULL_TREE,
544	DECL_ATTRIBUTES (v));
545	TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) = v;
546	}
547	}
548	break;
549
550	case RECORD_TYPE:
551	case UNION_TYPE:
552	case QUAL_UNION_TYPE:
553	if (TYPE_MAIN_VARIANT (type) != type
554	&& TYPE_FIELDS (type) == TYPE_FIELDS (TYPE_MAIN_VARIANT (type)))
555	TYPE_FIELDS (new_tree) = TYPE_FIELDS (TYPE_MAIN_VARIANT (new_tree));
556	else
557	{
558	tree f, nf = NULL;
559
560	for (f = TYPE_FIELDS (new_tree); f ; f = DECL_CHAIN (f))
561	{
562	t = remap_decl (decl: f, id);
563	DECL_CONTEXT (t) = new_tree;
564	DECL_CHAIN (t) = nf;
565	nf = t;
566	}
567	TYPE_FIELDS (new_tree) = nreverse (nf);
568	}
569	break;
570
571	case OFFSET_TYPE:
572	default:
573	/* Shouldn't have been thought variable sized. */
574	gcc_unreachable ();
575	}
576
577	/* All variants of type share the same size, so use the already remaped data. */
578	if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
579	{
580	tree s = TYPE_SIZE (type);
581	tree mvs = TYPE_SIZE (TYPE_MAIN_VARIANT (type));
582	tree su = TYPE_SIZE_UNIT (type);
583	tree mvsu = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (type));
584	gcc_checking_assert ((TREE_CODE (s) == PLACEHOLDER_EXPR
585	&& (TREE_CODE (mvs) == PLACEHOLDER_EXPR))
586	|| s == mvs);
587	gcc_checking_assert ((TREE_CODE (su) == PLACEHOLDER_EXPR
588	&& (TREE_CODE (mvsu) == PLACEHOLDER_EXPR))
589	|| su == mvsu);
590	TYPE_SIZE (new_tree) = TYPE_SIZE (TYPE_MAIN_VARIANT (new_tree));
591	TYPE_SIZE_UNIT (new_tree) = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (new_tree));
592	}
593	else
594	{
595	walk_tree (&TYPE_SIZE (new_tree), copy_tree_body_r, id, NULL);
596	walk_tree (&TYPE_SIZE_UNIT (new_tree), copy_tree_body_r, id, NULL);
597	}
598
599	return new_tree;
600	}
601
602	/* Helper function for remap_type_2, called through walk_tree. */
603
604	static tree
605	remap_type_3 (tree *tp, int *walk_subtrees, void *data)
606	{
607	copy_body_data *id = (copy_body_data *) data;
608
609	if (TYPE_P (*tp))
610	*walk_subtrees = 0;
611
612	else if (DECL_P (*tp) && remap_decl (decl: *tp, id) != *tp)
613	return *tp;
614
615	return NULL_TREE;
616	}
617
618	/* Return true if TYPE needs to be remapped because remap_decl on any
619	needed embedded decl returns something other than that decl. */
620
621	static bool
622	remap_type_2 (tree type, copy_body_data *id)
623	{
624	tree t;
625
626	#define RETURN_TRUE_IF_VAR(T) \
627	do \
628	{ \
629	tree _t = (T); \
630	if (_t) \
631	{ \
632	if (DECL_P (_t) && remap_decl (_t, id) != _t) \
633	return true; \
634	if (!TYPE_SIZES_GIMPLIFIED (type) \
635	&& walk_tree (&_t, remap_type_3, id, NULL)) \
636	return true; \
637	} \
638	} \
639	while (0)
640
641	switch (TREE_CODE (type))
642	{
643	case POINTER_TYPE:
644	case REFERENCE_TYPE:
645	case FUNCTION_TYPE:
646	case METHOD_TYPE:
647	return remap_type_2 (TREE_TYPE (type), id);
648
649	case INTEGER_TYPE:
650	case REAL_TYPE:
651	case FIXED_POINT_TYPE:
652	case ENUMERAL_TYPE:
653	case BOOLEAN_TYPE:
654	RETURN_TRUE_IF_VAR (TYPE_MIN_VALUE (type));
655	RETURN_TRUE_IF_VAR (TYPE_MAX_VALUE (type));
656	return false;
657
658	case ARRAY_TYPE:
659	if (remap_type_2 (TREE_TYPE (type), id)
660	|| (TYPE_DOMAIN (type) && remap_type_2 (TYPE_DOMAIN (type), id)))
661	return true;
662	break;
663
664	case RECORD_TYPE:
665	case UNION_TYPE:
666	case QUAL_UNION_TYPE:
667	for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
668	if (TREE_CODE (t) == FIELD_DECL)
669	{
670	RETURN_TRUE_IF_VAR (DECL_FIELD_OFFSET (t));
671	RETURN_TRUE_IF_VAR (DECL_SIZE (t));
672	RETURN_TRUE_IF_VAR (DECL_SIZE_UNIT (t));
673	if (TREE_CODE (type) == QUAL_UNION_TYPE)
674	RETURN_TRUE_IF_VAR (DECL_QUALIFIER (t));
675	}
676	break;
677
678	default:
679	return false;
680	}
681
682	RETURN_TRUE_IF_VAR (TYPE_SIZE (type));
683	RETURN_TRUE_IF_VAR (TYPE_SIZE_UNIT (type));
684	return false;
685	#undef RETURN_TRUE_IF_VAR
686	}
687
688	tree
689	remap_type (tree type, copy_body_data *id)
690	{
691	tree *node;
692	tree tmp;
693
694	if (type == NULL)
695	return type;
696
697	/* See if we have remapped this type. */
698	node = id->decl_map->get (k: type);
699	if (node)
700	return *node;
701
702	/* The type only needs remapping if it's variably modified. */
703	if (! variably_modified_type_p (type, id->src_fn)
704	/* Don't remap if copy_decl method doesn't always return a new
705	decl and for all embedded decls returns the passed in decl. */
706	|| (id->dont_remap_vla_if_no_change && !remap_type_2 (type, id)))
707	{
708	insert_decl_map (id, key: type, value: type);
709	return type;
710	}
711
712	id->remapping_type_depth++;
713	tmp = remap_type_1 (type, id);
714	id->remapping_type_depth--;
715
716	return tmp;
717	}
718
719	/* Decide if DECL can be put into BLOCK_NONLOCAL_VARs. */
720
721	static bool
722	can_be_nonlocal (tree decl, copy_body_data *id)
723	{
724	/* We cannot duplicate function decls. */
725	if (TREE_CODE (decl) == FUNCTION_DECL)
726	return true;
727
728	/* Local static vars must be non-local or we get multiple declaration
729	problems. */
730	if (VAR_P (decl) && !auto_var_in_fn_p (decl, id->src_fn))
731	return true;
732
733	return false;
734	}
735
736	static tree
737	remap_decls (tree decls, vec<tree, va_gc> **nonlocalized_list,
738	copy_body_data *id)
739	{
740	tree old_var;
741	tree new_decls = NULL_TREE;
742
743	/* Remap its variables. */
744	for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
745	{
746	tree new_var;
747
748	if (can_be_nonlocal (decl: old_var, id))
749	{
750	/* We need to add this variable to the local decls as otherwise
751	nothing else will do so. */
752	if (VAR_P (old_var) && ! DECL_EXTERNAL (old_var) && cfun)
753	add_local_decl (cfun, d: old_var);
754	if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
755	&& !DECL_IGNORED_P (old_var)
756	&& nonlocalized_list)
757	vec_safe_push (v&: *nonlocalized_list, obj: old_var);
758	continue;
759	}
760
761	/* Remap the variable. */
762	new_var = remap_decl (decl: old_var, id);
763
764	/* If we didn't remap this variable, we can't mess with its
765	TREE_CHAIN. If we remapped this variable to the return slot, it's
766	already declared somewhere else, so don't declare it here. */
767
768	if (new_var == id->retvar)
769	;
770	else if (!new_var)
771	{
772	if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
773	&& !DECL_IGNORED_P (old_var)
774	&& nonlocalized_list)
775	vec_safe_push (v&: *nonlocalized_list, obj: old_var);
776	}
777	else
778	{
779	gcc_assert (DECL_P (new_var));
780	DECL_CHAIN (new_var) = new_decls;
781	new_decls = new_var;
782
783	/* Also copy value-expressions. */
784	if (VAR_P (new_var) && DECL_HAS_VALUE_EXPR_P (new_var))
785	{
786	tree tem = DECL_VALUE_EXPR (new_var);
787	bool old_regimplify = id->regimplify;
788	id->remapping_type_depth++;
789	walk_tree (&tem, copy_tree_body_r, id, NULL);
790	id->remapping_type_depth--;
791	id->regimplify = old_regimplify;
792	SET_DECL_VALUE_EXPR (new_var, tem);
793	}
794	}
795	}
796
797	return nreverse (new_decls);
798	}
799
800	/* Copy the BLOCK to contain remapped versions of the variables
801	therein. And hook the new block into the block-tree. */
802
803	static void
804	remap_block (tree *block, copy_body_data *id)
805	{
806	tree old_block;
807	tree new_block;
808
809	/* Make the new block. */
810	old_block = *block;
811	new_block = make_node (BLOCK);
812	TREE_USED (new_block) = TREE_USED (old_block);
813	BLOCK_ABSTRACT_ORIGIN (new_block) = BLOCK_ORIGIN (old_block);
814	BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block);
815	BLOCK_NONLOCALIZED_VARS (new_block)
816	= vec_safe_copy (BLOCK_NONLOCALIZED_VARS (old_block));
817	*block = new_block;
818
819	/* Remap its variables. */
820	BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block),
821	nonlocalized_list: &BLOCK_NONLOCALIZED_VARS (new_block),
822	id);
823
824	/* Remember the remapped block. */
825	insert_decl_map (id, key: old_block, value: new_block);
826	}
827
828	/* Copy the whole block tree and root it in id->block. */
829
830	static tree
831	remap_blocks (tree block, copy_body_data *id)
832	{
833	tree t;
834	tree new_tree = block;
835
836	if (!block)
837	return NULL;
838
839	remap_block (block: &new_tree, id);
840	gcc_assert (new_tree != block);
841	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
842	prepend_lexical_block (current_block: new_tree, new_block: remap_blocks (block: t, id));
843	/* Blocks are in arbitrary order, but make things slightly prettier and do
844	not swap order when producing a copy. */
845	BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree));
846	return new_tree;
847	}
848
849	/* Remap the block tree rooted at BLOCK to nothing. */
850
851	static void
852	remap_blocks_to_null (tree block, copy_body_data *id)
853	{
854	tree t;
855	insert_decl_map (id, key: block, NULL_TREE);
856	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
857	remap_blocks_to_null (block: t, id);
858	}
859
860	/* Remap the location info pointed to by LOCUS. */
861
862	static location_t
863	remap_location (location_t locus, copy_body_data *id)
864	{
865	if (LOCATION_BLOCK (locus))
866	{
867	tree *n = id->decl_map->get (LOCATION_BLOCK (locus));
868	gcc_assert (n);
869	if (*n)
870	return set_block (loc: locus, block: *n);
871	}
872
873	locus = LOCATION_LOCUS (locus);
874
875	if (locus != UNKNOWN_LOCATION && id->block)
876	return set_block (loc: locus, block: id->block);
877
878	return locus;
879	}
880
881	static void
882	copy_statement_list (tree *tp)
883	{
884	tree_stmt_iterator oi, ni;
885	tree new_tree;
886
887	new_tree = alloc_stmt_list ();
888	ni = tsi_start (t: new_tree);
889	oi = tsi_start (t: *tp);
890	TREE_TYPE (new_tree) = TREE_TYPE (*tp);
891	*tp = new_tree;
892
893	for (; !tsi_end_p (i: oi); tsi_next (i: &oi))
894	{
895	tree stmt = tsi_stmt (i: oi);
896	if (TREE_CODE (stmt) == STATEMENT_LIST)
897	/* This copy is not redundant; tsi_link_after will smash this
898	STATEMENT_LIST into the end of the one we're building, and we
899	don't want to do that with the original. */
900	copy_statement_list (tp: &stmt);
901	tsi_link_after (&ni, stmt, TSI_CONTINUE_LINKING);
902	}
903	}
904
905	static void
906	copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id)
907	{
908	tree block = BIND_EXPR_BLOCK (*tp);
909	/* Copy (and replace) the statement. */
910	copy_tree_r (tp, walk_subtrees, NULL);
911	if (block)
912	{
913	remap_block (block: &block, id);
914	BIND_EXPR_BLOCK (*tp) = block;
915	}
916
917	if (BIND_EXPR_VARS (*tp))
918	/* This will remap a lot of the same decls again, but this should be
919	harmless. */
920	BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id);
921	}
922
923
924	/* Create a new gimple_seq by remapping all the statements in BODY
925	using the inlining information in ID. */
926
927	static gimple_seq
928	remap_gimple_seq (gimple_seq body, copy_body_data *id)
929	{
930	gimple_stmt_iterator si;
931	gimple_seq new_body = NULL;
932
933	for (si = gsi_start (seq&: body); !gsi_end_p (i: si); gsi_next (i: &si))
934	{
935	gimple_seq new_stmts = remap_gimple_stmt (gsi_stmt (i: si), id);
936	gimple_seq_add_seq (&new_body, new_stmts);
937	}
938
939	return new_body;
940	}
941
942
943	/* Copy a GIMPLE_BIND statement STMT, remapping all the symbols in its
944	block using the mapping information in ID. */
945
946	static gimple *
947	copy_gimple_bind (gbind *stmt, copy_body_data *id)
948	{
949	gimple *new_bind;
950	tree new_block, new_vars;
951	gimple_seq body, new_body;
952
953	/* Copy the statement. Note that we purposely don't use copy_stmt
954	here because we need to remap statements as we copy. */
955	body = gimple_bind_body (gs: stmt);
956	new_body = remap_gimple_seq (body, id);
957
958	new_block = gimple_bind_block (bind_stmt: stmt);
959	if (new_block)
960	remap_block (block: &new_block, id);
961
962	/* This will remap a lot of the same decls again, but this should be
963	harmless. */
964	new_vars = gimple_bind_vars (bind_stmt: stmt);
965	if (new_vars)
966	new_vars = remap_decls (decls: new_vars, NULL, id);
967
968	new_bind = gimple_build_bind (new_vars, new_body, new_block);
969
970	return new_bind;
971	}
972
973	/* Return true if DECL is a parameter or a SSA_NAME for a parameter. */
974
975	static bool
976	is_parm (tree decl)
977	{
978	if (TREE_CODE (decl) == SSA_NAME)
979	{
980	decl = SSA_NAME_VAR (decl);
981	if (!decl)
982	return false;
983	}
984
985	return (TREE_CODE (decl) == PARM_DECL);
986	}
987
988	/* Remap the dependence CLIQUE from the source to the destination function
989	as specified in ID. */
990
991	static unsigned short
992	remap_dependence_clique (copy_body_data *id, unsigned short clique)
993	{
994	if (clique == 0 || processing_debug_stmt)
995	return 0;
996	if (!id->dependence_map)
997	id->dependence_map = new hash_map<dependence_hash, unsigned short>;
998	bool existed;
999	unsigned short &newc = id->dependence_map->get_or_insert (k: clique, existed: &existed);
1000	if (!existed)
1001	{
1002	/* Clique 1 is reserved for local ones set by PTA. */
1003	if (cfun->last_clique == 0)
1004	cfun->last_clique = 1;
1005	newc = ++cfun->last_clique;
1006	}
1007	return newc;
1008	}
1009
1010	/* Remap the GIMPLE operand pointed to by *TP. DATA is really a
1011	'struct walk_stmt_info *'. DATA->INFO is a 'copy_body_data *'.
1012	WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
1013	recursing into the children nodes of *TP. */
1014
1015	static tree
1016	remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
1017	{
1018	struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
1019	copy_body_data *id = (copy_body_data *) wi_p->info;
1020	tree fn = id->src_fn;
1021
1022	/* For recursive invocations this is no longer the LHS itself. */
1023	bool is_lhs = wi_p->is_lhs;
1024	wi_p->is_lhs = false;
1025
1026	if (TREE_CODE (*tp) == SSA_NAME)
1027	{
1028	*tp = remap_ssa_name (name: *tp, id);
1029	*walk_subtrees = 0;
1030	if (is_lhs)
1031	SSA_NAME_DEF_STMT (*tp) = wi_p->stmt;
1032	return NULL;
1033	}
1034	else if (auto_var_in_fn_p (*tp, fn))
1035	{
1036	/* Local variables and labels need to be replaced by equivalent
1037	variables. We don't want to copy static variables; there's
1038	only one of those, no matter how many times we inline the
1039	containing function. Similarly for globals from an outer
1040	function. */
1041	tree new_decl;
1042
1043	/* Remap the declaration. */
1044	new_decl = remap_decl (decl: *tp, id);
1045	gcc_assert (new_decl);
1046	/* Replace this variable with the copy. */
1047	STRIP_TYPE_NOPS (new_decl);
1048	/* ??? The C++ frontend uses void * pointer zero to initialize
1049	any other type. This confuses the middle-end type verification.
1050	As cloned bodies do not go through gimplification again the fixup
1051	there doesn't trigger. */
1052	if (TREE_CODE (new_decl) == INTEGER_CST
1053	&& !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
1054	new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
1055	*tp = new_decl;
1056	*walk_subtrees = 0;
1057	}
1058	else if (TREE_CODE (*tp) == STATEMENT_LIST)
1059	gcc_unreachable ();
1060	else if (TREE_CODE (*tp) == SAVE_EXPR)
1061	gcc_unreachable ();
1062	else if (TREE_CODE (*tp) == LABEL_DECL
1063	&& (!DECL_CONTEXT (*tp)
1064	|| decl_function_context (*tp) == id->src_fn))
1065	/* These may need to be remapped for EH handling. */
1066	*tp = remap_decl (decl: *tp, id);
1067	else if (TREE_CODE (*tp) == FIELD_DECL)
1068	{
1069	/* If the enclosing record type is variably_modified_type_p, the field
1070	has already been remapped. Otherwise, it need not be. */
1071	tree *n = id->decl_map->get (k: *tp);
1072	if (n)
1073	*tp = *n;
1074	*walk_subtrees = 0;
1075	}
1076	else if (TYPE_P (*tp))
1077	/* Types may need remapping as well. */
1078	*tp = remap_type (type: *tp, id);
1079	else if (CONSTANT_CLASS_P (*tp))
1080	{
1081	/* If this is a constant, we have to copy the node iff the type
1082	will be remapped. copy_tree_r will not copy a constant. */
1083	tree new_type = remap_type (TREE_TYPE (*tp), id);
1084
1085	if (new_type == TREE_TYPE (*tp))
1086	*walk_subtrees = 0;
1087
1088	else if (TREE_CODE (*tp) == INTEGER_CST)
1089	*tp = wide_int_to_tree (type: new_type, cst: wi::to_wide (t: *tp));
1090	else
1091	{
1092	*tp = copy_node (*tp);
1093	TREE_TYPE (*tp) = new_type;
1094	}
1095	}
1096	else
1097	{
1098	/* Otherwise, just copy the node. Note that copy_tree_r already
1099	knows not to copy VAR_DECLs, etc., so this is safe. */
1100
1101	if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
1102	{
1103	/* We need to re-canonicalize MEM_REFs from inline substitutions
1104	that can happen when a pointer argument is an ADDR_EXPR.
1105	Recurse here manually to allow that. */
1106	tree ptr = TREE_OPERAND (*tp, 0);
1107	tree type = remap_type (TREE_TYPE (*tp), id);
1108	tree old = *tp;
1109	walk_tree (&ptr, remap_gimple_op_r, data, NULL);
1110	*tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1111	TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1112	TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1113	copy_warning (*tp, old);
1114	if (MR_DEPENDENCE_CLIQUE (old) != 0)
1115	{
1116	MR_DEPENDENCE_CLIQUE (*tp)
1117	= remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1118	MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1119	}
1120	/* We cannot propagate the TREE_THIS_NOTRAP flag if we have
1121	remapped a parameter as the property might be valid only
1122	for the parameter itself. */
1123	if (TREE_THIS_NOTRAP (old)
1124	&& (!is_parm (TREE_OPERAND (old, 0))
1125	|| (!id->transform_parameter && is_parm (decl: ptr))))
1126	TREE_THIS_NOTRAP (*tp) = 1;
1127	REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
1128	*walk_subtrees = 0;
1129	return NULL;
1130	}
1131
1132	/* Here is the "usual case". Copy this tree node, and then
1133	tweak some special cases. */
1134	copy_tree_r (tp, walk_subtrees, NULL);
1135
1136	if (TREE_CODE (*tp) != OMP_CLAUSE)
1137	TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1138
1139	if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1140	{
1141	/* The copied TARGET_EXPR has never been expanded, even if the
1142	original node was expanded already. */
1143	TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1144	TREE_OPERAND (*tp, 3) = NULL_TREE;
1145	}
1146	else if (TREE_CODE (*tp) == ADDR_EXPR)
1147	{
1148	/* Variable substitution need not be simple. In particular,
1149	the MEM_REF substitution above. Make sure that
1150	TREE_CONSTANT and friends are up-to-date. */
1151	int invariant = is_gimple_min_invariant (*tp);
1152	walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL);
1153	recompute_tree_invariant_for_addr_expr (*tp);
1154
1155	/* If this used to be invariant, but is not any longer,
1156	then regimplification is probably needed. */
1157	if (invariant && !is_gimple_min_invariant (*tp))
1158	id->regimplify = true;
1159
1160	*walk_subtrees = 0;
1161	}
1162	}
1163
1164	/* Update the TREE_BLOCK for the cloned expr. */
1165	if (EXPR_P (*tp))
1166	{
1167	tree new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1168	tree old_block = TREE_BLOCK (*tp);
1169	if (old_block)
1170	{
1171	tree *n;
1172	n = id->decl_map->get (TREE_BLOCK (*tp));
1173	if (n)
1174	new_block = *n;
1175	}
1176	TREE_SET_BLOCK (*tp, new_block);
1177	}
1178
1179	/* Keep iterating. */
1180	return NULL_TREE;
1181	}
1182
1183
1184	/* Called from copy_body_id via walk_tree. DATA is really a
1185	`copy_body_data *'. */
1186
1187	tree
1188	copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
1189	{
1190	copy_body_data *id = (copy_body_data *) data;
1191	tree fn = id->src_fn;
1192	tree new_block;
1193
1194	/* Begin by recognizing trees that we'll completely rewrite for the
1195	inlining context. Our output for these trees is completely
1196	different from out input (e.g. RETURN_EXPR is deleted, and morphs
1197	into an edge). Further down, we'll handle trees that get
1198	duplicated and/or tweaked. */
1199
1200	/* When requested, RETURN_EXPRs should be transformed to just the
1201	contained MODIFY_EXPR. The branch semantics of the return will
1202	be handled elsewhere by manipulating the CFG rather than a statement. */
1203	if (TREE_CODE (*tp) == RETURN_EXPR && id->transform_return_to_modify)
1204	{
1205	tree assignment = TREE_OPERAND (*tp, 0);
1206
1207	/* If we're returning something, just turn that into an
1208	assignment into the equivalent of the original RESULT_DECL.
1209	If the "assignment" is just the result decl, the result
1210	decl has already been set (e.g. a recent "foo (&result_decl,
1211	...)"); just toss the entire RETURN_EXPR. */
1212	if (assignment && TREE_CODE (assignment) == MODIFY_EXPR)
1213	{
1214	/* Replace the RETURN_EXPR with (a copy of) the
1215	MODIFY_EXPR hanging underneath. */
1216	*tp = copy_node (assignment);
1217	}
1218	else /* Else the RETURN_EXPR returns no value. */
1219	{
1220	*tp = NULL;
1221	return (tree) (void *)1;
1222	}
1223	}
1224	else if (TREE_CODE (*tp) == SSA_NAME)
1225	{
1226	*tp = remap_ssa_name (name: *tp, id);
1227	*walk_subtrees = 0;
1228	return NULL;
1229	}
1230
1231	/* Local variables and labels need to be replaced by equivalent
1232	variables. We don't want to copy static variables; there's only
1233	one of those, no matter how many times we inline the containing
1234	function. Similarly for globals from an outer function. */
1235	else if (auto_var_in_fn_p (*tp, fn))
1236	{
1237	tree new_decl;
1238
1239	/* Remap the declaration. */
1240	new_decl = remap_decl (decl: *tp, id);
1241	gcc_assert (new_decl);
1242	/* Replace this variable with the copy. */
1243	STRIP_TYPE_NOPS (new_decl);
1244	*tp = new_decl;
1245	*walk_subtrees = 0;
1246	}
1247	else if (TREE_CODE (*tp) == STATEMENT_LIST)
1248	copy_statement_list (tp);
1249	else if (TREE_CODE (*tp) == SAVE_EXPR
1250	|| TREE_CODE (*tp) == TARGET_EXPR)
1251	remap_save_expr (tp, id->decl_map, walk_subtrees);
1252	else if (TREE_CODE (*tp) == LABEL_DECL
1253	&& (! DECL_CONTEXT (*tp)
1254	|| decl_function_context (*tp) == id->src_fn))
1255	/* These may need to be remapped for EH handling. */
1256	*tp = remap_decl (decl: *tp, id);
1257	else if (TREE_CODE (*tp) == BIND_EXPR)
1258	copy_bind_expr (tp, walk_subtrees, id);
1259	/* Types may need remapping as well. */
1260	else if (TYPE_P (*tp))
1261	*tp = remap_type (type: *tp, id);
1262
1263	/* If this is a constant, we have to copy the node iff the type will be
1264	remapped. copy_tree_r will not copy a constant. */
1265	else if (CONSTANT_CLASS_P (*tp))
1266	{
1267	tree new_type = remap_type (TREE_TYPE (*tp), id);
1268
1269	if (new_type == TREE_TYPE (*tp))
1270	*walk_subtrees = 0;
1271
1272	else if (TREE_CODE (*tp) == INTEGER_CST)
1273	*tp = wide_int_to_tree (type: new_type, cst: wi::to_wide (t: *tp));
1274	else
1275	{
1276	*tp = copy_node (*tp);
1277	TREE_TYPE (*tp) = new_type;
1278	}
1279	}
1280
1281	/* Otherwise, just copy the node. Note that copy_tree_r already
1282	knows not to copy VAR_DECLs, etc., so this is safe. */
1283	else
1284	{
1285	/* Here we handle trees that are not completely rewritten.
1286	First we detect some inlining-induced bogosities for
1287	discarding. */
1288	if (TREE_CODE (*tp) == MODIFY_EXPR
1289	&& TREE_OPERAND (*tp, 0) == TREE_OPERAND (*tp, 1)
1290	&& (auto_var_in_fn_p (TREE_OPERAND (*tp, 0), fn)))
1291	{
1292	/* Some assignments VAR = VAR; don't generate any rtl code
1293	and thus don't count as variable modification. Avoid
1294	keeping bogosities like 0 = 0. */
1295	tree decl = TREE_OPERAND (*tp, 0), value;
1296	tree *n;
1297
1298	n = id->decl_map->get (k: decl);
1299	if (n)
1300	{
1301	value = *n;
1302	STRIP_TYPE_NOPS (value);
1303	if (TREE_CONSTANT (value) || TREE_READONLY (value))
1304	{
1305	*tp = build_empty_stmt (EXPR_LOCATION (*tp));
1306	return copy_tree_body_r (tp, walk_subtrees, data);
1307	}
1308	}
1309	}
1310	else if (INDIRECT_REF_P (*tp))
1311	{
1312	/* Get rid of *& from inline substitutions that can happen when a
1313	pointer argument is an ADDR_EXPR. */
1314	tree decl = TREE_OPERAND (*tp, 0);
1315	tree *n = id->decl_map->get (k: decl);
1316	if (n)
1317	{
1318	/* If we happen to get an ADDR_EXPR in n->value, strip
1319	it manually here as we'll eventually get ADDR_EXPRs
1320	which lie about their types pointed to. In this case
1321	build_fold_indirect_ref wouldn't strip the INDIRECT_REF,
1322	but we absolutely rely on that. As fold_indirect_ref
1323	does other useful transformations, try that first, though. */
1324	tree type = TREE_TYPE (*tp);
1325	tree ptr = id->do_not_unshare ? *n : unshare_expr (*n);
1326	tree old = *tp;
1327	*tp = id->do_not_fold ? NULL : gimple_fold_indirect_ref (ptr);
1328	if (! *tp)
1329	{
1330	type = remap_type (type, id);
1331	if (TREE_CODE (ptr) == ADDR_EXPR && !id->do_not_fold)
1332	{
1333	*tp
1334	= fold_indirect_ref_1 (EXPR_LOCATION (ptr), type, ptr);
1335	/* ??? We should either assert here or build
1336	a VIEW_CONVERT_EXPR instead of blindly leaking
1337	incompatible types to our IL. */
1338	if (! *tp)
1339	*tp = TREE_OPERAND (ptr, 0);
1340	}
1341	else
1342	{
1343	*tp = build1 (INDIRECT_REF, type, ptr);
1344	TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1345	TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1346	TREE_READONLY (*tp) = TREE_READONLY (old);
1347	/* We cannot propagate the TREE_THIS_NOTRAP flag if we
1348	have remapped a parameter as the property might be
1349	valid only for the parameter itself. */
1350	if (TREE_THIS_NOTRAP (old)
1351	&& (!is_parm (TREE_OPERAND (old, 0))
1352	|| (!id->transform_parameter && is_parm (decl: ptr))))
1353	TREE_THIS_NOTRAP (*tp) = 1;
1354	}
1355	}
1356	*walk_subtrees = 0;
1357	return NULL;
1358	}
1359	}
1360	else if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
1361	{
1362	/* We need to re-canonicalize MEM_REFs from inline substitutions
1363	that can happen when a pointer argument is an ADDR_EXPR.
1364	Recurse here manually to allow that. */
1365	tree ptr = TREE_OPERAND (*tp, 0);
1366	tree type = remap_type (TREE_TYPE (*tp), id);
1367	tree old = *tp;
1368	walk_tree (&ptr, copy_tree_body_r, data, NULL);
1369	*tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1370	TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1371	TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1372	copy_warning (*tp, old);
1373	if (MR_DEPENDENCE_CLIQUE (old) != 0)
1374	{
1375	MR_DEPENDENCE_CLIQUE (*tp)
1376	= remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1377	MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1378	}
1379	/* We cannot propagate the TREE_THIS_NOTRAP flag if we have
1380	remapped a parameter as the property might be valid only
1381	for the parameter itself. */
1382	if (TREE_THIS_NOTRAP (old)
1383	&& (!is_parm (TREE_OPERAND (old, 0))
1384	|| (!id->transform_parameter && is_parm (decl: ptr))))
1385	TREE_THIS_NOTRAP (*tp) = 1;
1386	REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
1387	*walk_subtrees = 0;
1388	return NULL;
1389	}
1390
1391	/* Here is the "usual case". Copy this tree node, and then
1392	tweak some special cases. */
1393	copy_tree_r (tp, walk_subtrees, NULL);
1394
1395	/* If EXPR has block defined, map it to newly constructed block.
1396	When inlining we want EXPRs without block appear in the block
1397	of function call if we are not remapping a type. */
1398	if (EXPR_P (*tp))
1399	{
1400	new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1401	if (TREE_BLOCK (*tp))
1402	{
1403	tree *n;
1404	n = id->decl_map->get (TREE_BLOCK (*tp));
1405	if (n)
1406	new_block = *n;
1407	}
1408	TREE_SET_BLOCK (*tp, new_block);
1409	}
1410
1411	if (TREE_CODE (*tp) != OMP_CLAUSE)
1412	TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1413
1414	/* The copied TARGET_EXPR has never been expanded, even if the
1415	original node was expanded already. */
1416	if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1417	{
1418	TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1419	TREE_OPERAND (*tp, 3) = NULL_TREE;
1420	}
1421
1422	/* Variable substitution need not be simple. In particular, the
1423	INDIRECT_REF substitution above. Make sure that TREE_CONSTANT
1424	and friends are up-to-date. */
1425	else if (TREE_CODE (*tp) == ADDR_EXPR)
1426	{
1427	int invariant = is_gimple_min_invariant (*tp);
1428	walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
1429
1430	/* Handle the case where we substituted an INDIRECT_REF
1431	into the operand of the ADDR_EXPR. */
1432	if (INDIRECT_REF_P (TREE_OPERAND (*tp, 0))
1433	&& !id->do_not_fold)
1434	{
1435	tree t = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
1436	if (TREE_TYPE (t) != TREE_TYPE (*tp))
1437	t = fold_convert (remap_type (TREE_TYPE (*tp), id), t);
1438	*tp = t;
1439	}
1440	else
1441	recompute_tree_invariant_for_addr_expr (*tp);
1442
1443	/* If this used to be invariant, but is not any longer,
1444	then regimplification is probably needed. */
1445	if (invariant && !is_gimple_min_invariant (*tp))
1446	id->regimplify = true;
1447
1448	*walk_subtrees = 0;
1449	}
1450	else if (TREE_CODE (*tp) == OMP_CLAUSE
1451	&& (OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_AFFINITY
1452	|| OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_DEPEND))
1453	{
1454	tree t = OMP_CLAUSE_DECL (*tp);
1455	if (t
1456	&& TREE_CODE (t) == TREE_LIST
1457	&& TREE_PURPOSE (t)
1458	&& TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
1459	{
1460	*walk_subtrees = 0;
1461	OMP_CLAUSE_DECL (*tp) = copy_node (t);
1462	t = OMP_CLAUSE_DECL (*tp);
1463	TREE_PURPOSE (t) = copy_node (TREE_PURPOSE (t));
1464	for (int i = 0; i <= 4; i++)
1465	walk_tree (&TREE_VEC_ELT (TREE_PURPOSE (t), i),
1466	copy_tree_body_r, id, NULL);
1467	if (TREE_VEC_ELT (TREE_PURPOSE (t), 5))
1468	remap_block (block: &TREE_VEC_ELT (TREE_PURPOSE (t), 5), id);
1469	walk_tree (&TREE_VALUE (t), copy_tree_body_r, id, NULL);
1470	}
1471	}
1472	}
1473
1474	/* Keep iterating. */
1475	return NULL_TREE;
1476	}
1477
1478	/* Helper for remap_gimple_stmt. Given an EH region number for the
1479	source function, map that to the duplicate EH region number in
1480	the destination function. */
1481
1482	static int
1483	remap_eh_region_nr (int old_nr, copy_body_data *id)
1484	{
1485	eh_region old_r, new_r;
1486
1487	old_r = get_eh_region_from_number_fn (id->src_cfun, old_nr);
1488	new_r = static_cast<eh_region> (*id->eh_map->get (k: old_r));
1489
1490	return new_r->index;
1491	}
1492
1493	/* Similar, but operate on INTEGER_CSTs. */
1494
1495	static tree
1496	remap_eh_region_tree_nr (tree old_t_nr, copy_body_data *id)
1497	{
1498	int old_nr, new_nr;
1499
1500	old_nr = tree_to_shwi (old_t_nr);
1501	new_nr = remap_eh_region_nr (old_nr, id);
1502
1503	return build_int_cst (integer_type_node, new_nr);
1504	}
1505
1506	/* Helper for copy_bb. Remap statement STMT using the inlining
1507	information in ID. Return the new statement copy. */
1508
1509	static gimple_seq
1510	remap_gimple_stmt (gimple *stmt, copy_body_data *id)
1511	{
1512	gimple *copy = NULL;
1513	struct walk_stmt_info wi;
1514	bool skip_first = false;
1515	gimple_seq stmts = NULL;
1516
1517	if (is_gimple_debug (gs: stmt)
1518	&& (gimple_debug_nonbind_marker_p (s: stmt)
1519	? !DECL_STRUCT_FUNCTION (id->dst_fn)->debug_nonbind_markers
1520	: !opt_for_fn (id->dst_fn, flag_var_tracking_assignments)))
1521	return NULL;
1522
1523	if (!is_gimple_debug (gs: stmt)
1524	&& id->param_body_adjs
1525	&& id->param_body_adjs->m_dead_stmts.contains (k: stmt))
1526	{
1527	tree *dval = id->param_body_adjs->m_dead_stmt_debug_equiv.get (k: stmt);
1528	if (!dval)
1529	return NULL;
1530
1531	gcc_assert (is_gimple_assign (stmt));
1532	tree lhs = gimple_assign_lhs (gs: stmt);
1533	tree *dvar = id->param_body_adjs->m_dead_ssa_debug_equiv.get (k: lhs);
1534	gdebug *bind = gimple_build_debug_bind (*dvar, *dval, stmt);
1535	if (id->reset_location)
1536	gimple_set_location (g: bind, location: input_location);
1537	id->debug_stmts.safe_push (obj: bind);
1538	gimple_seq_add_stmt_without_update (&stmts, bind);
1539	return stmts;
1540	}
1541
1542	/* Begin by recognizing trees that we'll completely rewrite for the
1543	inlining context. Our output for these trees is completely
1544	different from our input (e.g. RETURN_EXPR is deleted and morphs
1545	into an edge). Further down, we'll handle trees that get
1546	duplicated and/or tweaked. */
1547
1548	/* When requested, GIMPLE_RETURN should be transformed to just the
1549	contained GIMPLE_ASSIGN. The branch semantics of the return will
1550	be handled elsewhere by manipulating the CFG rather than the
1551	statement. */
1552	if (gimple_code (g: stmt) == GIMPLE_RETURN && id->transform_return_to_modify)
1553	{
1554	tree retval = gimple_return_retval (gs: as_a <greturn *> (p: stmt));
1555
1556	/* If we're returning something, just turn that into an
1557	assignment to the equivalent of the original RESULT_DECL.
1558	If RETVAL is just the result decl, the result decl has
1559	already been set (e.g. a recent "foo (&result_decl, ...)");
1560	just toss the entire GIMPLE_RETURN. Likewise for when the
1561	call doesn't want the return value. */
1562	if (retval
1563	&& (TREE_CODE (retval) != RESULT_DECL
1564	&& (!id->call_stmt
1565	|| gimple_call_lhs (gs: id->call_stmt) != NULL_TREE)
1566	&& (TREE_CODE (retval) != SSA_NAME
1567	|| ! SSA_NAME_VAR (retval)
1568	|| TREE_CODE (SSA_NAME_VAR (retval)) != RESULT_DECL)))
1569	{
1570	copy = gimple_build_assign (id->do_not_unshare
1571	? id->retvar : unshare_expr (id->retvar),
1572	retval);
1573	/* id->retvar is already substituted. Skip it on later remapping. */
1574	skip_first = true;
1575	}
1576	else
1577	return NULL;
1578	}
1579	else if (gimple_has_substatements (g: stmt))
1580	{
1581	gimple_seq s1, s2;
1582
1583	/* When cloning bodies from the C++ front end, we will be handed bodies
1584	in High GIMPLE form. Handle here all the High GIMPLE statements that
1585	have embedded statements. */
1586	switch (gimple_code (g: stmt))
1587	{
1588	case GIMPLE_BIND:
1589	copy = copy_gimple_bind (stmt: as_a <gbind *> (p: stmt), id);
1590	break;
1591
1592	case GIMPLE_CATCH:
1593	{
1594	gcatch *catch_stmt = as_a <gcatch *> (p: stmt);
1595	s1 = remap_gimple_seq (body: gimple_catch_handler (catch_stmt), id);
1596	copy = gimple_build_catch (gimple_catch_types (catch_stmt), s1);
1597	}
1598	break;
1599
1600	case GIMPLE_EH_FILTER:
1601	s1 = remap_gimple_seq (body: gimple_eh_filter_failure (gs: stmt), id);
1602	copy = gimple_build_eh_filter (gimple_eh_filter_types (gs: stmt), s1);
1603	break;
1604
1605	case GIMPLE_TRY:
1606	s1 = remap_gimple_seq (body: gimple_try_eval (gs: stmt), id);
1607	s2 = remap_gimple_seq (body: gimple_try_cleanup (gs: stmt), id);
1608	copy = gimple_build_try (s1, s2, gimple_try_kind (gs: stmt));
1609	break;
1610
1611	case GIMPLE_WITH_CLEANUP_EXPR:
1612	s1 = remap_gimple_seq (body: gimple_wce_cleanup (gs: stmt), id);
1613	copy = gimple_build_wce (s1);
1614	break;
1615
1616	case GIMPLE_OMP_PARALLEL:
1617	{
1618	gomp_parallel *omp_par_stmt = as_a <gomp_parallel *> (p: stmt);
1619	s1 = remap_gimple_seq (body: gimple_omp_body (gs: omp_par_stmt), id);
1620	copy = gimple_build_omp_parallel
1621	(s1,
1622	gimple_omp_parallel_clauses (gs: omp_par_stmt),
1623	gimple_omp_parallel_child_fn (omp_parallel_stmt: omp_par_stmt),
1624	gimple_omp_parallel_data_arg (omp_parallel_stmt: omp_par_stmt));
1625	}
1626	break;
1627
1628	case GIMPLE_OMP_TASK:
1629	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1630	copy = gimple_build_omp_task
1631	(s1,
1632	gimple_omp_task_clauses (gs: stmt),
1633	gimple_omp_task_child_fn (gs: stmt),
1634	gimple_omp_task_data_arg (gs: stmt),
1635	gimple_omp_task_copy_fn (gs: stmt),
1636	gimple_omp_task_arg_size (gs: stmt),
1637	gimple_omp_task_arg_align (gs: stmt));
1638	break;
1639
1640	case GIMPLE_OMP_FOR:
1641	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1642	s2 = remap_gimple_seq (body: gimple_omp_for_pre_body (gs: stmt), id);
1643	copy = gimple_build_omp_for (s1, gimple_omp_for_kind (g: stmt),
1644	gimple_omp_for_clauses (gs: stmt),
1645	gimple_omp_for_collapse (gs: stmt), s2);
1646	{
1647	size_t i;
1648	for (i = 0; i < gimple_omp_for_collapse (gs: stmt); i++)
1649	{
1650	gimple_omp_for_set_index (gs: copy, i,
1651	index: gimple_omp_for_index (gs: stmt, i));
1652	gimple_omp_for_set_initial (gs: copy, i,
1653	initial: gimple_omp_for_initial (gs: stmt, i));
1654	gimple_omp_for_set_final (gs: copy, i,
1655	final: gimple_omp_for_final (gs: stmt, i));
1656	gimple_omp_for_set_incr (gs: copy, i,
1657	incr: gimple_omp_for_incr (gs: stmt, i));
1658	gimple_omp_for_set_cond (gs: copy, i,
1659	cond: gimple_omp_for_cond (gs: stmt, i));
1660	}
1661	}
1662	break;
1663
1664	case GIMPLE_OMP_MASTER:
1665	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1666	copy = gimple_build_omp_master (s1);
1667	break;
1668
1669	case GIMPLE_OMP_MASKED:
1670	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1671	copy = gimple_build_omp_masked
1672	(s1, gimple_omp_masked_clauses (gs: stmt));
1673	break;
1674
1675	case GIMPLE_OMP_SCOPE:
1676	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1677	copy = gimple_build_omp_scope
1678	(s1, gimple_omp_scope_clauses (gs: stmt));
1679	break;
1680
1681	case GIMPLE_OMP_TASKGROUP:
1682	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1683	copy = gimple_build_omp_taskgroup
1684	(s1, gimple_omp_taskgroup_clauses (gs: stmt));
1685	break;
1686
1687	case GIMPLE_OMP_ORDERED:
1688	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1689	copy = gimple_build_omp_ordered
1690	(s1,
1691	gimple_omp_ordered_clauses (ord_stmt: as_a <gomp_ordered *> (p: stmt)));
1692	break;
1693
1694	case GIMPLE_OMP_SCAN:
1695	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1696	copy = gimple_build_omp_scan
1697	(s1, gimple_omp_scan_clauses (scan_stmt: as_a <gomp_scan *> (p: stmt)));
1698	break;
1699
1700	case GIMPLE_OMP_SECTION:
1701	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1702	copy = gimple_build_omp_section (s1);
1703	break;
1704
1705	case GIMPLE_OMP_SECTIONS:
1706	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1707	copy = gimple_build_omp_sections
1708	(s1, gimple_omp_sections_clauses (gs: stmt));
1709	break;
1710
1711	case GIMPLE_OMP_STRUCTURED_BLOCK:
1712	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1713	copy = gimple_build_omp_structured_block (s1);
1714	break;
1715
1716	case GIMPLE_OMP_SINGLE:
1717	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1718	copy = gimple_build_omp_single
1719	(s1, gimple_omp_single_clauses (gs: stmt));
1720	break;
1721
1722	case GIMPLE_OMP_TARGET:
1723	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1724	copy = gimple_build_omp_target
1725	(s1, gimple_omp_target_kind (g: stmt),
1726	gimple_omp_target_clauses (gs: stmt));
1727	break;
1728
1729	case GIMPLE_OMP_TEAMS:
1730	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1731	copy = gimple_build_omp_teams
1732	(s1, gimple_omp_teams_clauses (gs: stmt));
1733	break;
1734
1735	case GIMPLE_OMP_CRITICAL:
1736	s1 = remap_gimple_seq (body: gimple_omp_body (gs: stmt), id);
1737	copy = gimple_build_omp_critical (s1,
1738	gimple_omp_critical_name
1739	(crit_stmt: as_a <gomp_critical *> (p: stmt)),
1740	gimple_omp_critical_clauses
1741	(crit_stmt: as_a <gomp_critical *> (p: stmt)));
1742	break;
1743
1744	case GIMPLE_ASSUME:
1745	s1 = remap_gimple_seq (body: gimple_assume_body (gs: stmt), id);
1746	copy = gimple_build_assume (gimple_assume_guard (gs: stmt), s1);
1747	break;
1748
1749	case GIMPLE_TRANSACTION:
1750	{
1751	gtransaction *old_trans_stmt = as_a <gtransaction *> (p: stmt);
1752	gtransaction *new_trans_stmt;
1753	s1 = remap_gimple_seq (body: gimple_transaction_body (transaction_stmt: old_trans_stmt),
1754	id);
1755	copy = new_trans_stmt = gimple_build_transaction (s1);
1756	gimple_transaction_set_subcode (transaction_stmt: new_trans_stmt,
1757	subcode: gimple_transaction_subcode (transaction_stmt: old_trans_stmt));
1758	gimple_transaction_set_label_norm (transaction_stmt: new_trans_stmt,
1759	label: gimple_transaction_label_norm (transaction_stmt: old_trans_stmt));
1760	gimple_transaction_set_label_uninst (transaction_stmt: new_trans_stmt,
1761	label: gimple_transaction_label_uninst (transaction_stmt: old_trans_stmt));
1762	gimple_transaction_set_label_over (transaction_stmt: new_trans_stmt,
1763	label: gimple_transaction_label_over (transaction_stmt: old_trans_stmt));
1764	}
1765	break;
1766
1767	default:
1768	gcc_unreachable ();
1769	}
1770	}
1771	else
1772	{
1773	if (gimple_assign_single_p (gs: stmt)
1774	&& gimple_assign_lhs (gs: stmt) == gimple_assign_rhs1 (gs: stmt)
1775	&& auto_var_in_fn_p (gimple_assign_lhs (gs: stmt), id->src_fn))
1776	{
1777	/* Here we handle statements that are not completely rewritten.
1778	First we detect some inlining-induced bogosities for
1779	discarding. */
1780
1781	/* Some assignments VAR = VAR; don't generate any rtl code
1782	and thus don't count as variable modification. Avoid
1783	keeping bogosities like 0 = 0. */
1784	tree decl = gimple_assign_lhs (gs: stmt), value;
1785	tree *n;
1786
1787	n = id->decl_map->get (k: decl);
1788	if (n)
1789	{
1790	value = *n;
1791	STRIP_TYPE_NOPS (value);
1792	if (TREE_CONSTANT (value) || TREE_READONLY (value))
1793	return NULL;
1794	}
1795	}
1796
1797	/* For *ptr_N ={v} {CLOBBER}, if ptr_N is SSA_NAME defined
1798	in a block that we aren't copying during tree_function_versioning,
1799	just drop the clobber stmt. */
1800	if (id->blocks_to_copy && gimple_clobber_p (s: stmt))
1801	{
1802	tree lhs = gimple_assign_lhs (gs: stmt);
1803	if (TREE_CODE (lhs) == MEM_REF
1804	&& TREE_CODE (TREE_OPERAND (lhs, 0)) == SSA_NAME)
1805	{
1806	gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (lhs, 0));
1807	if (gimple_bb (g: def_stmt)
1808	&& !bitmap_bit_p (id->blocks_to_copy,
1809	gimple_bb (g: def_stmt)->index))
1810	return NULL;
1811	}
1812	}
1813
1814	/* We do not allow CLOBBERs of handled components. In case
1815	returned value is stored via such handled component, remove
1816	the clobber so stmt verifier is happy. */
1817	if (gimple_clobber_p (s: stmt)
1818	&& TREE_CODE (gimple_assign_lhs (stmt)) == RESULT_DECL)
1819	{
1820	tree remapped = remap_decl (decl: gimple_assign_lhs (gs: stmt), id);
1821	if (!DECL_P (remapped)
1822	&& TREE_CODE (remapped) != MEM_REF)
1823	return NULL;
1824	}
1825
1826	if (gimple_debug_bind_p (s: stmt))
1827	{
1828	tree var = gimple_debug_bind_get_var (dbg: stmt);
1829	tree value = gimple_debug_bind_get_value (dbg: stmt);
1830	if (id->param_body_adjs
1831	&& id->param_body_adjs->m_dead_stmts.contains (k: stmt))
1832	{
1833	value = unshare_expr_without_location (value);
1834	id->param_body_adjs->remap_with_debug_expressions (t: &value);
1835	}
1836
1837	gdebug *copy = gimple_build_debug_bind (var, value, stmt);
1838	if (id->reset_location)
1839	gimple_set_location (g: copy, location: input_location);
1840	id->debug_stmts.safe_push (obj: copy);
1841	gimple_seq_add_stmt_without_update (&stmts, copy);
1842	return stmts;
1843	}
1844	if (gimple_debug_source_bind_p (s: stmt))
1845	{
1846	gdebug *copy = gimple_build_debug_source_bind
1847	(gimple_debug_source_bind_get_var (dbg: stmt),
1848	gimple_debug_source_bind_get_value (dbg: stmt),
1849	stmt);
1850	if (id->reset_location)
1851	gimple_set_location (g: copy, location: input_location);
1852	id->debug_stmts.safe_push (obj: copy);
1853	gimple_seq_add_stmt_without_update (&stmts, copy);
1854	return stmts;
1855	}
1856	if (gimple_debug_nonbind_marker_p (s: stmt))
1857	{
1858	/* If the inlined function has too many debug markers,
1859	don't copy them. */
1860	if (id->src_cfun->debug_marker_count
1861	> param_max_debug_marker_count
1862	|| id->reset_location)
1863	return stmts;
1864
1865	gdebug *copy = as_a <gdebug *> (p: gimple_copy (stmt));
1866	id->debug_stmts.safe_push (obj: copy);
1867	gimple_seq_add_stmt_without_update (&stmts, copy);
1868	return stmts;
1869	}
1870
1871	/* Create a new deep copy of the statement. */
1872	copy = gimple_copy (stmt);
1873
1874	/* Clear flags that need revisiting. */
1875	if (gcall *call_stmt = dyn_cast <gcall *> (p: copy))
1876	{
1877	if (gimple_call_tail_p (s: call_stmt))
1878	gimple_call_set_tail (s: call_stmt, tail_p: false);
1879	if (gimple_call_from_thunk_p (s: call_stmt))
1880	gimple_call_set_from_thunk (s: call_stmt, from_thunk_p: false);
1881	if (gimple_call_internal_p (gs: call_stmt))
1882	switch (gimple_call_internal_fn (gs: call_stmt))
1883	{
1884	case IFN_GOMP_SIMD_LANE:
1885	case IFN_GOMP_SIMD_VF:
1886	case IFN_GOMP_SIMD_LAST_LANE:
1887	case IFN_GOMP_SIMD_ORDERED_START:
1888	case IFN_GOMP_SIMD_ORDERED_END:
1889	DECL_STRUCT_FUNCTION (id->dst_fn)->has_simduid_loops = true;
1890	break;
1891	default:
1892	break;
1893	}
1894	}
1895
1896	/* Remap the region numbers for __builtin_eh_{pointer,filter},
1897	RESX and EH_DISPATCH. */
1898	if (id->eh_map)
1899	switch (gimple_code (g: copy))
1900	{
1901	case GIMPLE_CALL:
1902	{
1903	tree r, fndecl = gimple_call_fndecl (gs: copy);
1904	if (fndecl && fndecl_built_in_p (node: fndecl, klass: BUILT_IN_NORMAL))
1905	switch (DECL_FUNCTION_CODE (decl: fndecl))
1906	{
1907	case BUILT_IN_EH_COPY_VALUES:
1908	r = gimple_call_arg (gs: copy, index: 1);
1909	r = remap_eh_region_tree_nr (old_t_nr: r, id);
1910	gimple_call_set_arg (gs: copy, index: 1, arg: r);
1911	/* FALLTHRU */
1912
1913	case BUILT_IN_EH_POINTER:
1914	case BUILT_IN_EH_FILTER:
1915	r = gimple_call_arg (gs: copy, index: 0);
1916	r = remap_eh_region_tree_nr (old_t_nr: r, id);
1917	gimple_call_set_arg (gs: copy, index: 0, arg: r);
1918	break;
1919
1920	default:
1921	break;
1922	}
1923
1924	/* Reset alias info if we didn't apply measures to
1925	keep it valid over inlining by setting DECL_PT_UID. */
1926	if (!id->src_cfun->gimple_df
1927	|| !id->src_cfun->gimple_df->ipa_pta)
1928	gimple_call_reset_alias_info (as_a <gcall *> (p: copy));
1929	}
1930	break;
1931
1932	case GIMPLE_RESX:
1933	{
1934	gresx *resx_stmt = as_a <gresx *> (p: copy);
1935	int r = gimple_resx_region (resx_stmt);
1936	r = remap_eh_region_nr (old_nr: r, id);
1937	gimple_resx_set_region (resx_stmt, region: r);
1938	}
1939	break;
1940
1941	case GIMPLE_EH_DISPATCH:
1942	{
1943	geh_dispatch *eh_dispatch = as_a <geh_dispatch *> (p: copy);
1944	int r = gimple_eh_dispatch_region (eh_dispatch_stmt: eh_dispatch);
1945	r = remap_eh_region_nr (old_nr: r, id);
1946	gimple_eh_dispatch_set_region (eh_dispatch_stmt: eh_dispatch, region: r);
1947	}
1948	break;
1949
1950	default:
1951	break;
1952	}
1953	}
1954
1955	/* If STMT has a block defined, map it to the newly constructed block. */
1956	if (tree block = gimple_block (g: copy))
1957	{
1958	tree *n;
1959	n = id->decl_map->get (k: block);
1960	gcc_assert (n);
1961	gimple_set_block (g: copy, block: *n);
1962	}
1963	if (id->param_body_adjs)
1964	{
1965	gimple_seq extra_stmts = NULL;
1966	id->param_body_adjs->modify_gimple_stmt (stmt: &copy, extra_stmts: &extra_stmts, orig_stmt: stmt);
1967	if (!gimple_seq_empty_p (s: extra_stmts))
1968	{
1969	memset (s: &wi, c: 0, n: sizeof (wi));
1970	wi.info = id;
1971	for (gimple_stmt_iterator egsi = gsi_start (seq&: extra_stmts);
1972	!gsi_end_p (i: egsi);
1973	gsi_next (i: &egsi))
1974	walk_gimple_op (gsi_stmt (i: egsi), remap_gimple_op_r, &wi);
1975	gimple_seq_add_seq_without_update (&stmts, extra_stmts);
1976	}
1977	}
1978
1979	if (id->reset_location)
1980	gimple_set_location (g: copy, location: input_location);
1981
1982	/* Debug statements ought to be rebuilt and not copied. */
1983	gcc_checking_assert (!is_gimple_debug (copy));
1984
1985	/* Remap all the operands in COPY. */
1986	memset (s: &wi, c: 0, n: sizeof (wi));
1987	wi.info = id;
1988	if (skip_first)
1989	walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL);
1990	else
1991	walk_gimple_op (copy, remap_gimple_op_r, &wi);
1992
1993	/* Clear the copied virtual operands. We are not remapping them here
1994	but are going to recreate them from scratch. */
1995	if (gimple_has_mem_ops (g: copy))
1996	{
1997	gimple_set_vdef (g: copy, NULL_TREE);
1998	gimple_set_vuse (g: copy, NULL_TREE);
1999	}
2000
2001	if (cfun->can_throw_non_call_exceptions)
2002	{
2003	/* When inlining a function which does not have non-call exceptions
2004	enabled into a function that has (which only happens with
2005	always-inline) we have to fixup stmts that cannot throw. */
2006	if (gcond *cond = dyn_cast <gcond *> (p: copy))
2007	if (gimple_could_trap_p (cond))
2008	{
2009	gassign *cmp
2010	= gimple_build_assign (make_ssa_name (boolean_type_node),
2011	gimple_cond_code (gs: cond),
2012	gimple_cond_lhs (gs: cond),
2013	gimple_cond_rhs (gs: cond));
2014	gimple_seq_add_stmt_without_update (&stmts, cmp);
2015	gimple_cond_set_code (gs: cond, code: NE_EXPR);
2016	gimple_cond_set_lhs (gs: cond, lhs: gimple_assign_lhs (gs: cmp));
2017	gimple_cond_set_rhs (gs: cond, boolean_false_node);
2018	}
2019	}
2020
2021	gimple_seq_add_stmt_without_update (&stmts, copy);
2022	return stmts;
2023	}
2024
2025
2026	/* Copy basic block, scale profile accordingly. Edges will be taken care of
2027	later */
2028
2029	static basic_block
2030	copy_bb (copy_body_data *id, basic_block bb,
2031	profile_count num, profile_count den)
2032	{
2033	gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
2034	basic_block copy_basic_block;
2035	tree decl;
2036	basic_block prev;
2037
2038	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
2039
2040	/* Search for previous copied basic block. */
2041	prev = bb->prev_bb;
2042	while (!prev->aux)
2043	prev = prev->prev_bb;
2044
2045	/* create_basic_block() will append every new block to
2046	basic_block_info automatically. */
2047	copy_basic_block = create_basic_block (NULL, (basic_block) prev->aux);
2048	copy_basic_block->count = bb->count.apply_scale (num, den);
2049
2050	copy_gsi = gsi_start_bb (bb: copy_basic_block);
2051
2052	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
2053	{
2054	gimple_seq stmts;
2055	gimple *stmt = gsi_stmt (i: gsi);
2056	gimple *orig_stmt = stmt;
2057	gimple_stmt_iterator stmts_gsi;
2058	bool stmt_added = false;
2059
2060	id->regimplify = false;
2061	stmts = remap_gimple_stmt (stmt, id);
2062
2063	if (gimple_seq_empty_p (s: stmts))
2064	continue;
2065
2066	seq_gsi = copy_gsi;
2067
2068	for (stmts_gsi = gsi_start (seq&: stmts);
2069	!gsi_end_p (i: stmts_gsi); )
2070	{
2071	stmt = gsi_stmt (i: stmts_gsi);
2072
2073	/* Advance iterator now before stmt is moved to seq_gsi. */
2074	gsi_next (i: &stmts_gsi);
2075
2076	if (gimple_nop_p (g: stmt))
2077	continue;
2078
2079	gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun,
2080	orig_stmt);
2081
2082	gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT);
2083
2084	if (id->regimplify)
2085	gimple_regimplify_operands (stmt, &seq_gsi);
2086
2087	stmt_added = true;
2088	}
2089
2090	if (!stmt_added)
2091	continue;
2092
2093	/* If copy_basic_block has been empty at the start of this iteration,
2094	call gsi_start_bb again to get at the newly added statements. */
2095	if (gsi_end_p (i: copy_gsi))
2096	copy_gsi = gsi_start_bb (bb: copy_basic_block);
2097	else
2098	gsi_next (i: &copy_gsi);
2099
2100	/* Process the new statement. The call to gimple_regimplify_operands
2101	possibly turned the statement into multiple statements, we
2102	need to process all of them. */
2103	do
2104	{
2105	tree fn;
2106	gcall *call_stmt;
2107
2108	stmt = gsi_stmt (i: copy_gsi);
2109	call_stmt = dyn_cast <gcall *> (p: stmt);
2110	if (call_stmt
2111	&& gimple_call_va_arg_pack_p (s: call_stmt)
2112	&& id->call_stmt
2113	&& ! gimple_call_va_arg_pack_p (s: id->call_stmt))
2114	{
2115	/* __builtin_va_arg_pack () should be replaced by
2116	all arguments corresponding to ... in the caller. */
2117	tree p;
2118	gcall *new_call;
2119	vec<tree> argarray;
2120	size_t nargs_caller = gimple_call_num_args (gs: id->call_stmt);
2121	size_t nargs = nargs_caller;
2122
2123	for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
2124	{
2125	/* Avoid crashing on invalid IL that doesn't have a
2126	varargs function or that passes not enough arguments. */
2127	if (nargs == 0)
2128	break;
2129	nargs--;
2130	}
2131
2132	/* Create the new array of arguments. */
2133	size_t nargs_callee = gimple_call_num_args (gs: call_stmt);
2134	size_t n = nargs + nargs_callee;
2135	argarray.create (nelems: n);
2136	argarray.safe_grow_cleared (len: n, exact: true);
2137
2138	/* Copy all the arguments before '...' */
2139	if (nargs_callee)
2140	memcpy (dest: argarray.address (),
2141	src: gimple_call_arg_ptr (gs: call_stmt, index: 0),
2142	n: nargs_callee * sizeof (tree));
2143
2144	/* Append the arguments passed in '...' */
2145	if (nargs)
2146	memcpy (dest: argarray.address () + nargs_callee,
2147	src: gimple_call_arg_ptr (gs: id->call_stmt, index: 0)
2148	+ (nargs_caller - nargs), n: nargs * sizeof (tree));
2149
2150	new_call = gimple_build_call_vec (gimple_call_fn (gs: call_stmt),
2151	argarray);
2152
2153	argarray.release ();
2154
2155	/* Copy all GIMPLE_CALL flags, location and block, except
2156	GF_CALL_VA_ARG_PACK. */
2157	gimple_call_copy_flags (dest_call: new_call, orig_call: call_stmt);
2158	gimple_call_set_va_arg_pack (s: new_call, pass_arg_pack_p: false);
2159	gimple_call_set_fntype (call_stmt: new_call, fntype: gimple_call_fntype (gs: call_stmt));
2160	/* location includes block. */
2161	gimple_set_location (g: new_call, location: gimple_location (g: stmt));
2162	gimple_call_set_lhs (gs: new_call, lhs: gimple_call_lhs (gs: call_stmt));
2163
2164	gsi_replace (&copy_gsi, new_call, false);
2165	stmt = new_call;
2166	}
2167	else if (call_stmt
2168	&& id->call_stmt
2169	&& (decl = gimple_call_fndecl (gs: stmt))
2170	&& fndecl_built_in_p (node: decl, name1: BUILT_IN_VA_ARG_PACK_LEN))
2171	{
2172	/* __builtin_va_arg_pack_len () should be replaced by
2173	the number of anonymous arguments. */
2174	size_t nargs = gimple_call_num_args (gs: id->call_stmt);
2175	tree count, p;
2176	gimple *new_stmt;
2177
2178	for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
2179	nargs--;
2180
2181	if (!gimple_call_lhs (gs: stmt))
2182	{
2183	/* Drop unused calls. */
2184	gsi_remove (&copy_gsi, false);
2185	continue;
2186	}
2187	else if (!gimple_call_va_arg_pack_p (s: id->call_stmt))
2188	{
2189	count = build_int_cst (integer_type_node, nargs);
2190	new_stmt = gimple_build_assign (gimple_call_lhs (gs: stmt), count);
2191	gsi_replace (&copy_gsi, new_stmt, false);
2192	stmt = new_stmt;
2193	}
2194	else if (nargs != 0)
2195	{
2196	tree newlhs = create_tmp_reg_or_ssa_name (integer_type_node);
2197	count = build_int_cst (integer_type_node, nargs);
2198	new_stmt = gimple_build_assign (gimple_call_lhs (gs: stmt),
2199	PLUS_EXPR, newlhs, count);
2200	gimple_call_set_lhs (gs: stmt, lhs: newlhs);
2201	gsi_insert_after (&copy_gsi, new_stmt, GSI_NEW_STMT);
2202	}
2203	}
2204	else if (call_stmt
2205	&& id->call_stmt
2206	&& gimple_call_internal_p (gs: stmt)
2207	&& gimple_call_internal_fn (gs: stmt) == IFN_TSAN_FUNC_EXIT)
2208	{
2209	/* Drop TSAN_FUNC_EXIT () internal calls during inlining. */
2210	gsi_remove (&copy_gsi, false);
2211	continue;
2212	}
2213
2214	/* Statements produced by inlining can be unfolded, especially
2215	when we constant propagated some operands. We can't fold
2216	them right now for two reasons:
2217	1) folding require SSA_NAME_DEF_STMTs to be correct
2218	2) we can't change function calls to builtins.
2219	So we just mark statement for later folding. We mark
2220	all new statements, instead just statements that has changed
2221	by some nontrivial substitution so even statements made
2222	foldable indirectly are updated. If this turns out to be
2223	expensive, copy_body can be told to watch for nontrivial
2224	changes. */
2225	if (id->statements_to_fold)
2226	id->statements_to_fold->add (k: stmt);
2227
2228	/* We're duplicating a CALL_EXPR. Find any corresponding
2229	callgraph edges and update or duplicate them. */
2230	if (gcall *call_stmt = dyn_cast <gcall *> (p: stmt))
2231	{
2232	struct cgraph_edge *edge;
2233
2234	switch (id->transform_call_graph_edges)
2235	{
2236	case CB_CGE_DUPLICATE:
2237	edge = id->src_node->get_edge (call_stmt: orig_stmt);
2238	if (edge)
2239	{
2240	struct cgraph_edge *old_edge = edge;
2241
2242	/* A speculative call is consist of multiple
2243	edges - indirect edge and one or more direct edges
2244	Duplicate the whole thing and distribute frequencies
2245	accordingly. */
2246	if (edge->speculative)
2247	{
2248	int n = 0;
2249	profile_count direct_cnt
2250	= profile_count::zero ();
2251
2252	/* First figure out the distribution of counts
2253	so we can re-scale BB profile accordingly. */
2254	for (cgraph_edge *e = old_edge; e;
2255	e = e->next_speculative_call_target ())
2256	direct_cnt = direct_cnt + e->count;
2257
2258	cgraph_edge *indirect
2259	= old_edge->speculative_call_indirect_edge ();
2260	profile_count indir_cnt = indirect->count;
2261
2262	/* Next iterate all direct edges, clone it and its
2263	corresponding reference and update profile. */
2264	for (cgraph_edge *e = old_edge;
2265	e;
2266	e = e->next_speculative_call_target ())
2267	{
2268	profile_count cnt = e->count;
2269
2270	id->dst_node->clone_reference
2271	(ref: e->speculative_call_target_ref (), stmt);
2272	edge = e->clone (n: id->dst_node, call_stmt,
2273	stmt_uid: gimple_uid (g: stmt), num, den,
2274	update_original: true);
2275	profile_probability prob
2276	= cnt.probability_in (overall: direct_cnt
2277	+ indir_cnt);
2278	edge->count
2279	= copy_basic_block->count.apply_probability
2280	(prob);
2281	n++;
2282	}
2283	gcc_checking_assert
2284	(indirect->num_speculative_call_targets_p ()
2285	== n);
2286
2287	/* Duplicate the indirect edge after all direct edges
2288	cloned. */
2289	indirect = indirect->clone (n: id->dst_node, call_stmt,
2290	stmt_uid: gimple_uid (g: stmt),
2291	num, den,
2292	update_original: true);
2293
2294	profile_probability prob
2295	= indir_cnt.probability_in (overall: direct_cnt
2296	+ indir_cnt);
2297	indirect->count
2298	= copy_basic_block->count.apply_probability (prob);
2299	}
2300	else
2301	{
2302	edge = edge->clone (n: id->dst_node, call_stmt,
2303	stmt_uid: gimple_uid (g: stmt),
2304	num, den,
2305	update_original: true);
2306	edge->count = copy_basic_block->count;
2307	}
2308	}
2309	break;
2310
2311	case CB_CGE_MOVE_CLONES:
2312	id->dst_node->set_call_stmt_including_clones (old_stmt: orig_stmt,
2313	new_stmt: call_stmt);
2314	edge = id->dst_node->get_edge (call_stmt: stmt);
2315	break;
2316
2317	case CB_CGE_MOVE:
2318	edge = id->dst_node->get_edge (call_stmt: orig_stmt);
2319	if (edge)
2320	edge = cgraph_edge::set_call_stmt (e: edge, new_stmt: call_stmt);
2321	break;
2322
2323	default:
2324	gcc_unreachable ();
2325	}
2326
2327	/* Constant propagation on argument done during inlining
2328	may create new direct call. Produce an edge for it. */
2329	if ((!edge
2330	|| (edge->indirect_inlining_edge
2331	&& id->transform_call_graph_edges == CB_CGE_MOVE_CLONES))
2332	&& id->dst_node->definition
2333	&& (fn = gimple_call_fndecl (gs: stmt)) != NULL)
2334	{
2335	struct cgraph_node *dest = cgraph_node::get_create (fn);
2336
2337	/* We have missing edge in the callgraph. This can happen
2338	when previous inlining turned an indirect call into a
2339	direct call by constant propagating arguments or we are
2340	producing dead clone (for further cloning). In all
2341	other cases we hit a bug (incorrect node sharing is the
2342	most common reason for missing edges). */
2343	gcc_assert (!dest->definition
2344	|| dest->address_taken
2345	|| !id->src_node->definition
2346	|| !id->dst_node->definition);
2347	if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
2348	id->dst_node->create_edge_including_clones
2349	(callee: dest, old_stmt: orig_stmt, stmt: call_stmt, count: bb->count,
2350	reason: CIF_ORIGINALLY_INDIRECT_CALL);
2351	else
2352	id->dst_node->create_edge (callee: dest, call_stmt,
2353	count: bb->count)->inline_failed
2354	= CIF_ORIGINALLY_INDIRECT_CALL;
2355	if (dump_file)
2356	{
2357	fprintf (stream: dump_file, format: "Created new direct edge to %s\n",
2358	dest->dump_name ());
2359	}
2360	}
2361
2362	notice_special_calls (as_a <gcall *> (p: stmt));
2363	}
2364
2365	maybe_duplicate_eh_stmt_fn (cfun, stmt, id->src_cfun, orig_stmt,
2366	id->eh_map, id->eh_lp_nr);
2367
2368	gsi_next (i: &copy_gsi);
2369	}
2370	while (!gsi_end_p (i: copy_gsi));
2371
2372	copy_gsi = gsi_last_bb (bb: copy_basic_block);
2373	}
2374
2375	return copy_basic_block;
2376	}
2377
2378	/* Inserting Single Entry Multiple Exit region in SSA form into code in SSA
2379	form is quite easy, since dominator relationship for old basic blocks does
2380	not change.
2381
2382	There is however exception where inlining might change dominator relation
2383	across EH edges from basic block within inlined functions destinating
2384	to landing pads in function we inline into.
2385
2386	The function fills in PHI_RESULTs of such PHI nodes if they refer
2387	to gimple regs. Otherwise, the function mark PHI_RESULT of such
2388	PHI nodes for renaming. For non-gimple regs, renaming is safe: the
2389	EH edges are abnormal and SSA_NAME_OCCURS_IN_ABNORMAL_PHI must be
2390	set, and this means that there will be no overlapping live ranges
2391	for the underlying symbol.
2392
2393	This might change in future if we allow redirecting of EH edges and
2394	we might want to change way build CFG pre-inlining to include
2395	all the possible edges then. */
2396	static void
2397	update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
2398	bool can_throw, bool nonlocal_goto)
2399	{
2400	edge e;
2401	edge_iterator ei;
2402
2403	FOR_EACH_EDGE (e, ei, bb->succs)
2404	if (!e->dest->aux
2405	|| ((basic_block)e->dest->aux)->index == ENTRY_BLOCK)
2406	{
2407	gphi *phi;
2408	gphi_iterator si;
2409
2410	if (!nonlocal_goto)
2411	gcc_assert (e->flags & EDGE_EH);
2412
2413	if (!can_throw)
2414	gcc_assert (!(e->flags & EDGE_EH));
2415
2416	for (si = gsi_start_phis (e->dest); !gsi_end_p (i: si); gsi_next (i: &si))
2417	{
2418	edge re;
2419
2420	phi = si.phi ();
2421
2422	/* For abnormal goto/call edges the receiver can be the
2423	ENTRY_BLOCK. Do not assert this cannot happen. */
2424
2425	gcc_assert ((e->flags & EDGE_EH)
2426	|| SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)));
2427
2428	re = find_edge (ret_bb, e->dest);
2429	gcc_checking_assert (re);
2430	gcc_assert ((re->flags & (EDGE_EH | EDGE_ABNORMAL))
2431	== (e->flags & (EDGE_EH | EDGE_ABNORMAL)));
2432
2433	SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
2434	USE_FROM_PTR (PHI_ARG_DEF_PTR_FROM_EDGE (phi, re)));
2435	}
2436	}
2437	}
2438
2439	/* Insert clobbers for automatic variables of inlined ID->src_fn
2440	function at the start of basic block ID->eh_landing_pad_dest. */
2441
2442	static void
2443	add_clobbers_to_eh_landing_pad (copy_body_data *id)
2444	{
2445	tree var;
2446	basic_block bb = id->eh_landing_pad_dest;
2447	live_vars_map *vars = NULL;
2448	unsigned int cnt = 0;
2449	unsigned int i;
2450	FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
2451	if (VAR_P (var)
2452	&& !DECL_HARD_REGISTER (var)
2453	&& !TREE_THIS_VOLATILE (var)
2454	&& !DECL_HAS_VALUE_EXPR_P (var)
2455	&& !is_gimple_reg (var)
2456	&& auto_var_in_fn_p (var, id->src_fn)
2457	&& !lookup_attribute (attr_name: "omp simd array", DECL_ATTRIBUTES (var)))
2458	{
2459	tree *t = id->decl_map->get (k: var);
2460	if (!t)
2461	continue;
2462	tree new_var = *t;
2463	if (VAR_P (new_var)
2464	&& !DECL_HARD_REGISTER (new_var)
2465	&& !TREE_THIS_VOLATILE (new_var)
2466	&& !DECL_HAS_VALUE_EXPR_P (new_var)
2467	&& !is_gimple_reg (new_var)
2468	&& auto_var_in_fn_p (new_var, id->dst_fn))
2469	{
2470	if (vars == NULL)
2471	vars = new live_vars_map;
2472	vars->put (DECL_UID (var), v: cnt++);
2473	}
2474	}
2475	if (vars == NULL)
2476	return;
2477
2478	vec<bitmap_head> live = compute_live_vars (id->src_cfun, vars);
2479	FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
2480	if (VAR_P (var))
2481	{
2482	edge e;
2483	edge_iterator ei;
2484	bool needed = false;
2485	unsigned int *v = vars->get (DECL_UID (var));
2486	if (v == NULL)
2487	continue;
2488	FOR_EACH_EDGE (e, ei, bb->preds)
2489	if ((e->flags & EDGE_EH) != 0
2490	&& e->src->index >= id->add_clobbers_to_eh_landing_pads)
2491	{
2492	basic_block src_bb = (basic_block) e->src->aux;
2493
2494	if (bitmap_bit_p (&live[src_bb->index], *v))
2495	{
2496	needed = true;
2497	break;
2498	}
2499	}
2500	if (needed)
2501	{
2502	tree new_var = *id->decl_map->get (k: var);
2503	gimple_stmt_iterator gsi = gsi_after_labels (bb);
2504	tree clobber = build_clobber (TREE_TYPE (new_var));
2505	gimple *clobber_stmt = gimple_build_assign (new_var, clobber);
2506	gsi_insert_before (&gsi, clobber_stmt, GSI_NEW_STMT);
2507	}
2508	}
2509	destroy_live_vars (live);
2510	delete vars;
2511	}
2512
2513	/* Copy edges from BB into its copy constructed earlier, scale profile
2514	accordingly. Edges will be taken care of later. Assume aux
2515	pointers to point to the copies of each BB. Return true if any
2516	debug stmts are left after a statement that must end the basic block. */
2517
2518	static bool
2519	copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
2520	basic_block ret_bb, basic_block abnormal_goto_dest,
2521	copy_body_data *id)
2522	{
2523	basic_block new_bb = (basic_block) bb->aux;
2524	edge_iterator ei;
2525	edge old_edge;
2526	gimple_stmt_iterator si;
2527	bool need_debug_cleanup = false;
2528
2529	/* Use the indices from the original blocks to create edges for the
2530	new ones. */
2531	FOR_EACH_EDGE (old_edge, ei, bb->succs)
2532	if (!(old_edge->flags & EDGE_EH))
2533	{
2534	edge new_edge;
2535	int flags = old_edge->flags;
2536	location_t locus = old_edge->goto_locus;
2537
2538	/* Return edges do get a FALLTHRU flag when they get inlined. */
2539	if (old_edge->dest->index == EXIT_BLOCK
2540	&& !(flags & (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE|EDGE_FAKE))
2541	&& old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
2542	flags |= EDGE_FALLTHRU;
2543
2544	new_edge
2545	= make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
2546	new_edge->probability = old_edge->probability;
2547	if (!id->reset_location)
2548	new_edge->goto_locus = remap_location (locus, id);
2549	}
2550
2551	if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK)
2552	return false;
2553
2554	/* When doing function splitting, we must decrease count of the return block
2555	which was previously reachable by block we did not copy. */
2556	if (single_succ_p (bb) && single_succ_edge (bb)->dest->index == EXIT_BLOCK)
2557	FOR_EACH_EDGE (old_edge, ei, bb->preds)
2558	if (old_edge->src->index != ENTRY_BLOCK
2559	&& !old_edge->src->aux)
2560	new_bb->count -= old_edge->count ().apply_scale (num, den);
2561
2562	/* Walk stmts from end to start so that splitting will adjust the BB
2563	pointer for each stmt at most once, even when we split the block
2564	multiple times. */
2565	bool seen_nondebug = false;
2566	for (si = gsi_last_bb (bb: new_bb); !gsi_end_p (i: si);)
2567	{
2568	bool can_throw, nonlocal_goto;
2569	gimple *copy_stmt = gsi_stmt (i: si);
2570
2571	/* Do this before the possible split_block. */
2572	gsi_prev (i: &si);
2573
2574	/* If this tree could throw an exception, there are two
2575	cases where we need to add abnormal edge(s): the
2576	tree wasn't in a region and there is a "current
2577	region" in the caller; or the original tree had
2578	EH edges. In both cases split the block after the tree,
2579	and add abnormal edge(s) as needed; we need both
2580	those from the callee and the caller.
2581	We check whether the copy can throw, because the const
2582	propagation can change an INDIRECT_REF which throws
2583	into a COMPONENT_REF which doesn't. If the copy
2584	can throw, the original could also throw. */
2585	can_throw = stmt_can_throw_internal (cfun, copy_stmt);
2586	nonlocal_goto
2587	= (stmt_can_make_abnormal_goto (copy_stmt)
2588	&& !computed_goto_p (copy_stmt));
2589
2590	if (can_throw || nonlocal_goto)
2591	{
2592	/* If there's only debug insns after copy_stmt don't split
2593	the block but instead mark the block for cleanup. */
2594	if (!seen_nondebug)
2595	need_debug_cleanup = true;
2596	else
2597	{
2598	/* Note that bb's predecessor edges aren't necessarily
2599	right at this point; split_block doesn't care. */
2600	edge e = split_block (new_bb, copy_stmt);
2601	e->dest->aux = new_bb->aux;
2602	seen_nondebug = false;
2603	}
2604	}
2605
2606	if (!is_gimple_debug (gs: copy_stmt))
2607	seen_nondebug = true;
2608
2609	bool update_probs = false;
2610
2611	if (gimple_code (g: copy_stmt) == GIMPLE_EH_DISPATCH)
2612	{
2613	make_eh_dispatch_edges (as_a <geh_dispatch *> (p: copy_stmt));
2614	update_probs = true;
2615	}
2616	else if (can_throw)
2617	{
2618	make_eh_edge (copy_stmt);
2619	update_probs = true;
2620	}
2621
2622	/* EH edges may not match old edges. Copy as much as possible. */
2623	if (update_probs)
2624	{
2625	edge e;
2626	edge_iterator ei;
2627	basic_block copy_stmt_bb = gimple_bb (g: copy_stmt);
2628
2629	FOR_EACH_EDGE (old_edge, ei, bb->succs)
2630	if ((old_edge->flags & EDGE_EH)
2631	&& (e = find_edge (copy_stmt_bb,
2632	(basic_block) old_edge->dest->aux))
2633	&& (e->flags & EDGE_EH))
2634	e->probability = old_edge->probability;
2635
2636	FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
2637	if (e->flags & EDGE_EH)
2638	{
2639	if (!e->probability.initialized_p ())
2640	e->probability = profile_probability::never ();
2641	if (e->dest->index < id->add_clobbers_to_eh_landing_pads)
2642	{
2643	if (id->eh_landing_pad_dest == NULL)
2644	id->eh_landing_pad_dest = e->dest;
2645	else
2646	gcc_assert (id->eh_landing_pad_dest == e->dest);
2647	}
2648	}
2649	}
2650
2651
2652	/* If the call we inline cannot make abnormal goto do not add
2653	additional abnormal edges but only retain those already present
2654	in the original function body. */
2655	if (abnormal_goto_dest == NULL)
2656	nonlocal_goto = false;
2657	if (nonlocal_goto)
2658	{
2659	basic_block copy_stmt_bb = gimple_bb (g: copy_stmt);
2660
2661	if (get_abnormal_succ_dispatcher (copy_stmt_bb))
2662	nonlocal_goto = false;
2663	/* ABNORMAL_DISPATCHER (1) is for longjmp/setjmp or nonlocal gotos
2664	in OpenMP regions which aren't allowed to be left abnormally.
2665	So, no need to add abnormal edge in that case. */
2666	else if (is_gimple_call (gs: copy_stmt)
2667	&& gimple_call_internal_p (gs: copy_stmt)
2668	&& (gimple_call_internal_fn (gs: copy_stmt)
2669	== IFN_ABNORMAL_DISPATCHER)
2670	&& gimple_call_arg (gs: copy_stmt, index: 0) == boolean_true_node)
2671	nonlocal_goto = false;
2672	else
2673	make_single_succ_edge (copy_stmt_bb, abnormal_goto_dest,
2674	EDGE_ABNORMAL);
2675	}
2676
2677	if ((can_throw || nonlocal_goto)
2678	&& gimple_in_ssa_p (cfun))
2679	update_ssa_across_abnormal_edges (bb: gimple_bb (g: copy_stmt), ret_bb,
2680	can_throw, nonlocal_goto);
2681	}
2682	return need_debug_cleanup;
2683	}
2684
2685	/* Copy the PHIs. All blocks and edges are copied, some blocks
2686	was possibly split and new outgoing EH edges inserted.
2687	BB points to the block of original function and AUX pointers links
2688	the original and newly copied blocks. */
2689
2690	static void
2691	copy_phis_for_bb (basic_block bb, copy_body_data *id)
2692	{
2693	basic_block const new_bb = (basic_block) bb->aux;
2694	edge_iterator ei;
2695	gphi *phi;
2696	gphi_iterator si;
2697	edge new_edge;
2698	bool inserted = false;
2699
2700	for (si = gsi_start_phis (bb); !gsi_end_p (i: si); gsi_next (i: &si))
2701	{
2702	tree res, new_res;
2703	gphi *new_phi;
2704
2705	phi = si.phi ();
2706	res = PHI_RESULT (phi);
2707	new_res = res;
2708	if (!virtual_operand_p (op: res)
2709	&& (!id->param_body_adjs
2710	|| !id->param_body_adjs->m_dead_stmts.contains (k: phi)))
2711	{
2712	walk_tree (&new_res, copy_tree_body_r, id, NULL);
2713	if (EDGE_COUNT (new_bb->preds) == 0)
2714	{
2715	/* Technically we'd want a SSA_DEFAULT_DEF here... */
2716	SSA_NAME_DEF_STMT (new_res) = gimple_build_nop ();
2717	}
2718	else
2719	{
2720	new_phi = create_phi_node (new_res, new_bb);
2721	FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2722	{
2723	edge old_edge = find_edge ((basic_block) new_edge->src->aux,
2724	bb);
2725	tree arg;
2726	tree new_arg;
2727	edge_iterator ei2;
2728	location_t locus;
2729
2730	/* When doing partial cloning, we allow PHIs on the entry
2731	block as long as all the arguments are the same.
2732	Find any input edge to see argument to copy. */
2733	if (!old_edge)
2734	FOR_EACH_EDGE (old_edge, ei2, bb->preds)
2735	if (!old_edge->src->aux)
2736	break;
2737
2738	arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge);
2739	new_arg = arg;
2740	walk_tree (&new_arg, copy_tree_body_r, id, NULL);
2741	gcc_assert (new_arg);
2742	/* With return slot optimization we can end up with
2743	non-gimple (foo *)&this->m, fix that here. */
2744	if (TREE_CODE (new_arg) != SSA_NAME
2745	&& TREE_CODE (new_arg) != FUNCTION_DECL
2746	&& !is_gimple_val (new_arg))
2747	{
2748	gimple_seq stmts = NULL;
2749	new_arg = force_gimple_operand (new_arg, &stmts, true,
2750	NULL);
2751	gsi_insert_seq_on_edge (new_edge, stmts);
2752	inserted = true;
2753	}
2754	locus = gimple_phi_arg_location_from_edge (phi, e: old_edge);
2755	if (id->reset_location)
2756	locus = input_location;
2757	else
2758	locus = remap_location (locus, id);
2759	add_phi_arg (new_phi, new_arg, new_edge, locus);
2760	}
2761	}
2762	}
2763	}
2764
2765	/* Commit the delayed edge insertions. */
2766	if (inserted)
2767	FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2768	gsi_commit_one_edge_insert (new_edge, NULL);
2769	}
2770
2771
2772	/* Wrapper for remap_decl so it can be used as a callback. */
2773
2774	static tree
2775	remap_decl_1 (tree decl, void *data)
2776	{
2777	return remap_decl (decl, id: (copy_body_data *) data);
2778	}
2779
2780	/* Build struct function and associated datastructures for the new clone
2781	NEW_FNDECL to be build. CALLEE_FNDECL is the original. Function changes
2782	the cfun to the function of new_fndecl (and current_function_decl too). */
2783
2784	static void
2785	initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
2786	{
2787	struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
2788
2789	/* Register specific tree functions. */
2790	gimple_register_cfg_hooks ();
2791
2792	/* Get clean struct function. */
2793	push_struct_function (fndecl: new_fndecl, true);
2794	targetm.target_option.relayout_function (new_fndecl);
2795
2796	/* We will rebuild these, so just sanity check that they are empty. */
2797	gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL);
2798	gcc_assert (cfun->local_decls == NULL);
2799	gcc_assert (cfun->cfg == NULL);
2800	gcc_assert (cfun->decl == new_fndecl);
2801
2802	/* Copy items we preserve during cloning. */
2803	cfun->static_chain_decl = src_cfun->static_chain_decl;
2804	cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area;
2805	cfun->function_end_locus = src_cfun->function_end_locus;
2806	cfun->curr_properties = src_cfun->curr_properties;
2807	cfun->last_verified = src_cfun->last_verified;
2808	cfun->va_list_gpr_size = src_cfun->va_list_gpr_size;
2809	cfun->va_list_fpr_size = src_cfun->va_list_fpr_size;
2810	cfun->has_nonlocal_label = src_cfun->has_nonlocal_label;
2811	cfun->calls_eh_return = src_cfun->calls_eh_return;
2812	cfun->stdarg = src_cfun->stdarg;
2813	cfun->after_inlining = src_cfun->after_inlining;
2814	cfun->can_throw_non_call_exceptions
2815	= src_cfun->can_throw_non_call_exceptions;
2816	cfun->can_delete_dead_exceptions = src_cfun->can_delete_dead_exceptions;
2817	cfun->returns_struct = src_cfun->returns_struct;
2818	cfun->returns_pcc_struct = src_cfun->returns_pcc_struct;
2819
2820	init_empty_tree_cfg ();
2821
2822	profile_status_for_fn (cfun) = profile_status_for_fn (src_cfun);
2823	cfun->cfg->full_profile = src_cfun->cfg->full_profile;
2824
2825	profile_count num = count;
2826	profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
2827	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
2828
2829	ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
2830	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num: count,
2831	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
2832	EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
2833	EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num: count,
2834	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
2835	if (src_cfun->eh)
2836	init_eh_for_function ();
2837
2838	if (src_cfun->gimple_df)
2839	{
2840	init_tree_ssa (cfun);
2841	cfun->gimple_df->in_ssa_p = src_cfun->gimple_df->in_ssa_p;
2842	if (cfun->gimple_df->in_ssa_p)
2843	init_ssa_operands (cfun);
2844	}
2845	}
2846
2847	/* Helper function for copy_cfg_body. Move debug stmts from the end
2848	of NEW_BB to the beginning of successor basic blocks when needed. If the
2849	successor has multiple predecessors, reset them, otherwise keep
2850	their value. */
2851
2852	static void
2853	maybe_move_debug_stmts_to_successors (copy_body_data *id, basic_block new_bb)
2854	{
2855	edge e;
2856	edge_iterator ei;
2857	gimple_stmt_iterator si = gsi_last_nondebug_bb (bb: new_bb);
2858
2859	if (gsi_end_p (i: si)
2860	|| gsi_one_before_end_p (i: si)
2861	|| !(stmt_can_throw_internal (cfun, gsi_stmt (i: si))
2862	|| stmt_can_make_abnormal_goto (gsi_stmt (i: si))))
2863	return;
2864
2865	FOR_EACH_EDGE (e, ei, new_bb->succs)
2866	{
2867	gimple_stmt_iterator ssi = gsi_last_bb (bb: new_bb);
2868	gimple_stmt_iterator dsi = gsi_after_labels (bb: e->dest);
2869	while (is_gimple_debug (gs: gsi_stmt (i: ssi)))
2870	{
2871	gimple *stmt = gsi_stmt (i: ssi);
2872	gdebug *new_stmt;
2873	tree var;
2874	tree value;
2875
2876	/* For the last edge move the debug stmts instead of copying
2877	them. */
2878	if (ei_one_before_end_p (i: ei))
2879	{
2880	si = ssi;
2881	gsi_prev (i: &ssi);
2882	if (!single_pred_p (bb: e->dest) && gimple_debug_bind_p (s: stmt))
2883	{
2884	gimple_debug_bind_reset_value (dbg: stmt);
2885	gimple_set_location (g: stmt, UNKNOWN_LOCATION);
2886	}
2887	gsi_remove (&si, false);
2888	gsi_insert_before (&dsi, stmt, GSI_NEW_STMT);
2889	continue;
2890	}
2891
2892	if (gimple_debug_bind_p (s: stmt))
2893	{
2894	var = gimple_debug_bind_get_var (dbg: stmt);
2895	if (single_pred_p (bb: e->dest))
2896	{
2897	value = gimple_debug_bind_get_value (dbg: stmt);
2898	value = unshare_expr (value);
2899	new_stmt = gimple_build_debug_bind (var, value, stmt);
2900	}
2901	else
2902	new_stmt = gimple_build_debug_bind (var, NULL_TREE, NULL);
2903	}
2904	else if (gimple_debug_source_bind_p (s: stmt))
2905	{
2906	var = gimple_debug_source_bind_get_var (dbg: stmt);
2907	value = gimple_debug_source_bind_get_value (dbg: stmt);
2908	new_stmt = gimple_build_debug_source_bind (var, value, stmt);
2909	}
2910	else if (gimple_debug_nonbind_marker_p (s: stmt))
2911	new_stmt = as_a <gdebug *> (p: gimple_copy (stmt));
2912	else
2913	gcc_unreachable ();
2914	gsi_insert_before (&dsi, new_stmt, GSI_NEW_STMT);
2915	id->debug_stmts.safe_push (obj: new_stmt);
2916	gsi_prev (i: &ssi);
2917	}
2918	}
2919	}
2920
2921	/* Make a copy of the sub-loops of SRC_PARENT and place them
2922	as siblings of DEST_PARENT. */
2923
2924	static void
2925	copy_loops (copy_body_data *id,
2926	class loop *dest_parent, class loop *src_parent)
2927	{
2928	class loop *src_loop = src_parent->inner;
2929	while (src_loop)
2930	{
2931	if (!id->blocks_to_copy
2932	|| bitmap_bit_p (id->blocks_to_copy, src_loop->header->index))
2933	{
2934	class loop *dest_loop = alloc_loop ();
2935
2936	/* Assign the new loop its header and latch and associate
2937	those with the new loop. */
2938	dest_loop->header = (basic_block)src_loop->header->aux;
2939	dest_loop->header->loop_father = dest_loop;
2940	if (src_loop->latch != NULL)
2941	{
2942	dest_loop->latch = (basic_block)src_loop->latch->aux;
2943	dest_loop->latch->loop_father = dest_loop;
2944	}
2945
2946	/* Copy loop meta-data. */
2947	copy_loop_info (loop: src_loop, target: dest_loop);
2948	if (dest_loop->unroll)
2949	cfun->has_unroll = true;
2950	if (dest_loop->force_vectorize)
2951	cfun->has_force_vectorize_loops = true;
2952	if (id->src_cfun->last_clique != 0)
2953	dest_loop->owned_clique
2954	= remap_dependence_clique (id,
2955	clique: src_loop->owned_clique
2956	? src_loop->owned_clique : 1);
2957
2958	/* Finally place it into the loop array and the loop tree. */
2959	place_new_loop (cfun, dest_loop);
2960	flow_loop_tree_node_add (dest_parent, dest_loop);
2961
2962	if (src_loop->simduid)
2963	{
2964	dest_loop->simduid = remap_decl (decl: src_loop->simduid, id);
2965	cfun->has_simduid_loops = true;
2966	}
2967
2968	/* Recurse. */
2969	copy_loops (id, dest_parent: dest_loop, src_parent: src_loop);
2970	}
2971	src_loop = src_loop->next;
2972	}
2973	}
2974
2975	/* Call redirect_call_stmt_to_callee on all calls in BB. */
2976
2977	void
2978	redirect_all_calls (copy_body_data * id, basic_block bb)
2979	{
2980	gimple_stmt_iterator si;
2981	gimple *last = last_nondebug_stmt (bb);
2982	for (si = gsi_start_bb (bb); !gsi_end_p (i: si); gsi_next (i: &si))
2983	{
2984	gimple *stmt = gsi_stmt (i: si);
2985	if (is_gimple_call (gs: stmt))
2986	{
2987	tree old_lhs = gimple_call_lhs (gs: stmt);
2988	struct cgraph_edge *edge = id->dst_node->get_edge (call_stmt: stmt);
2989	if (edge)
2990	{
2991	gimple *new_stmt
2992	= cgraph_edge::redirect_call_stmt_to_callee (e: edge);
2993	/* If IPA-SRA transformation, run as part of edge redirection,
2994	removed the LHS because it is unused, save it to
2995	killed_new_ssa_names so that we can prune it from debug
2996	statements. */
2997	if (old_lhs
2998	&& TREE_CODE (old_lhs) == SSA_NAME
2999	&& !gimple_call_lhs (gs: new_stmt))
3000	{
3001	if (!id->killed_new_ssa_names)
3002	id->killed_new_ssa_names = new hash_set<tree> (16);
3003	id->killed_new_ssa_names->add (k: old_lhs);
3004	}
3005
3006	if (stmt == last && id->call_stmt && maybe_clean_eh_stmt (stmt))
3007	gimple_purge_dead_eh_edges (bb);
3008	}
3009	}
3010	}
3011	}
3012
3013	/* Make a copy of the body of FN so that it can be inserted inline in
3014	another function. Walks FN via CFG, returns new fndecl. */
3015
3016	static tree
3017	copy_cfg_body (copy_body_data * id,
3018	basic_block entry_block_map, basic_block exit_block_map,
3019	basic_block new_entry)
3020	{
3021	tree callee_fndecl = id->src_fn;
3022	/* Original cfun for the callee, doesn't change. */
3023	struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
3024	struct function *cfun_to_copy;
3025	basic_block bb;
3026	tree new_fndecl = NULL;
3027	bool need_debug_cleanup = false;
3028	int last;
3029	profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
3030	profile_count num = entry_block_map->count;
3031
3032	cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
3033
3034	/* Register specific tree functions. */
3035	gimple_register_cfg_hooks ();
3036
3037	/* If we are inlining just region of the function, make sure to connect
3038	new entry to ENTRY_BLOCK_PTR_FOR_FN (cfun). Since new entry can be
3039	part of loop, we must compute frequency and probability of
3040	ENTRY_BLOCK_PTR_FOR_FN (cfun) based on the frequencies and
3041	probabilities of edges incoming from nonduplicated region. */
3042	if (new_entry)
3043	{
3044	edge e;
3045	edge_iterator ei;
3046	den = profile_count::zero ();
3047
3048	FOR_EACH_EDGE (e, ei, new_entry->preds)
3049	if (!e->src->aux)
3050	den += e->count ();
3051	ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = den;
3052	}
3053
3054	profile_count::adjust_for_ipa_scaling (num: &num, den: &den);
3055
3056	/* Must have a CFG here at this point. */
3057	gcc_assert (ENTRY_BLOCK_PTR_FOR_FN
3058	(DECL_STRUCT_FUNCTION (callee_fndecl)));
3059
3060
3061	ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = entry_block_map;
3062	EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = exit_block_map;
3063	entry_block_map->aux = ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy);
3064	exit_block_map->aux = EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy);
3065
3066	/* Duplicate any exception-handling regions. */
3067	if (cfun->eh)
3068	id->eh_map = duplicate_eh_regions (cfun_to_copy, NULL, id->eh_lp_nr,
3069	remap_decl_1, id);
3070
3071	/* Use aux pointers to map the original blocks to copy. */
3072	FOR_EACH_BB_FN (bb, cfun_to_copy)
3073	if (!id->blocks_to_copy || bitmap_bit_p (id->blocks_to_copy, bb->index))
3074	{
3075	basic_block new_bb = copy_bb (id, bb, num, den);
3076	bb->aux = new_bb;
3077	new_bb->aux = bb;
3078	new_bb->loop_father = entry_block_map->loop_father;
3079	}
3080
3081	last = last_basic_block_for_fn (cfun);
3082
3083	/* Now that we've duplicated the blocks, duplicate their edges. */
3084	basic_block abnormal_goto_dest = NULL;
3085	if (id->call_stmt
3086	&& stmt_can_make_abnormal_goto (id->call_stmt))
3087	{
3088	gimple_stmt_iterator gsi = gsi_for_stmt (id->call_stmt);
3089
3090	bb = gimple_bb (g: id->call_stmt);
3091	gsi_next (i: &gsi);
3092	if (gsi_end_p (i: gsi))
3093	abnormal_goto_dest = get_abnormal_succ_dispatcher (bb);
3094	}
3095	FOR_ALL_BB_FN (bb, cfun_to_copy)
3096	if (!id->blocks_to_copy
3097	|| (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
3098	need_debug_cleanup |= copy_edges_for_bb (bb, num, den, ret_bb: exit_block_map,
3099	abnormal_goto_dest, id);
3100
3101	if (id->eh_landing_pad_dest)
3102	{
3103	add_clobbers_to_eh_landing_pad (id);
3104	id->eh_landing_pad_dest = NULL;
3105	}
3106
3107	if (new_entry)
3108	{
3109	edge e = make_edge (entry_block_map, (basic_block)new_entry->aux,
3110	EDGE_FALLTHRU);
3111	e->probability = profile_probability::always ();
3112	}
3113
3114	/* Duplicate the loop tree, if available and wanted. */
3115	if (loops_for_fn (fn: src_cfun) != NULL
3116	&& current_loops != NULL)
3117	{
3118	copy_loops (id, dest_parent: entry_block_map->loop_father,
3119	src_parent: get_loop (fn: src_cfun, num: 0));
3120	/* Defer to cfgcleanup to update loop-father fields of basic-blocks. */
3121	loops_state_set (flags: LOOPS_NEED_FIXUP);
3122	}
3123
3124	/* If the loop tree in the source function needed fixup, mark the
3125	destination loop tree for fixup, too. */
3126	if (loops_for_fn (fn: src_cfun)->state & LOOPS_NEED_FIXUP)
3127	loops_state_set (flags: LOOPS_NEED_FIXUP);
3128
3129	if (gimple_in_ssa_p (cfun))
3130	FOR_ALL_BB_FN (bb, cfun_to_copy)
3131	if (!id->blocks_to_copy
3132	|| (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
3133	copy_phis_for_bb (bb, id);
3134
3135	FOR_ALL_BB_FN (bb, cfun_to_copy)
3136	if (bb->aux)
3137	{
3138	if (need_debug_cleanup
3139	&& bb->index != ENTRY_BLOCK
3140	&& bb->index != EXIT_BLOCK)
3141	maybe_move_debug_stmts_to_successors (id, new_bb: (basic_block) bb->aux);
3142	/* Update call edge destinations. This cannot be done before loop
3143	info is updated, because we may split basic blocks. */
3144	if (id->transform_call_graph_edges == CB_CGE_DUPLICATE
3145	&& bb->index != ENTRY_BLOCK
3146	&& bb->index != EXIT_BLOCK)
3147	redirect_all_calls (id, bb: (basic_block)bb->aux);
3148	((basic_block)bb->aux)->aux = NULL;
3149	bb->aux = NULL;
3150	}
3151
3152	/* Zero out AUX fields of newly created block during EH edge
3153	insertion. */
3154	for (; last < last_basic_block_for_fn (cfun); last++)
3155	{
3156	if (need_debug_cleanup)
3157	maybe_move_debug_stmts_to_successors (id,
3158	BASIC_BLOCK_FOR_FN (cfun, last));
3159	BASIC_BLOCK_FOR_FN (cfun, last)->aux = NULL;
3160	/* Update call edge destinations. This cannot be done before loop
3161	info is updated, because we may split basic blocks. */
3162	if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
3163	redirect_all_calls (id, BASIC_BLOCK_FOR_FN (cfun, last));
3164	}
3165	entry_block_map->aux = NULL;
3166	exit_block_map->aux = NULL;
3167
3168	if (id->eh_map)
3169	{
3170	delete id->eh_map;
3171	id->eh_map = NULL;
3172	}
3173	if (id->dependence_map)
3174	{
3175	delete id->dependence_map;
3176	id->dependence_map = NULL;
3177	}
3178
3179	return new_fndecl;
3180	}
3181
3182	/* Copy the debug STMT using ID. We deal with these statements in a
3183	special way: if any variable in their VALUE expression wasn't
3184	remapped yet, we won't remap it, because that would get decl uids
3185	out of sync, causing codegen differences between -g and -g0. If
3186	this arises, we drop the VALUE expression altogether. */
3187
3188	static void
3189	copy_debug_stmt (gdebug *stmt, copy_body_data *id)
3190	{
3191	tree t, *n;
3192	struct walk_stmt_info wi;
3193
3194	if (tree block = gimple_block (g: stmt))
3195	{
3196	n = id->decl_map->get (k: block);
3197	gimple_set_block (g: stmt, block: n ? *n : id->block);
3198	}
3199
3200	if (gimple_debug_nonbind_marker_p (s: stmt))
3201	{
3202	if (id->call_stmt && !gimple_block (g: stmt))
3203	{
3204	gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3205	gsi_remove (&gsi, true);
3206	}
3207	return;
3208	}
3209
3210	/* Remap all the operands in COPY. */
3211	memset (s: &wi, c: 0, n: sizeof (wi));
3212	wi.info = id;
3213
3214	processing_debug_stmt = 1;
3215
3216	if (gimple_debug_source_bind_p (s: stmt))
3217	t = gimple_debug_source_bind_get_var (dbg: stmt);
3218	else if (gimple_debug_bind_p (s: stmt))
3219	t = gimple_debug_bind_get_var (dbg: stmt);
3220	else
3221	gcc_unreachable ();
3222
3223	if (TREE_CODE (t) == PARM_DECL
3224	&& id->debug_map
3225	&& (n = id->debug_map->get (k: t)))
3226	{
3227	gcc_assert (VAR_P (*n));
3228	t = *n;
3229	}
3230	else if (VAR_P (t) && !is_global_var (t) && !id->decl_map->get (k: t))
3231	/* T is a non-localized variable. */;
3232	else
3233	walk_tree (&t, remap_gimple_op_r, &wi, NULL);
3234
3235	if (gimple_debug_bind_p (s: stmt))
3236	{
3237	gimple_debug_bind_set_var (dbg: stmt, var: t);
3238
3239	if (gimple_debug_bind_has_value_p (dbg: stmt))
3240	walk_tree (gimple_debug_bind_get_value_ptr (stmt),
3241	remap_gimple_op_r, &wi, NULL);
3242
3243	/* Punt if any decl couldn't be remapped. */
3244	if (processing_debug_stmt < 0)
3245	gimple_debug_bind_reset_value (dbg: stmt);
3246	}
3247	else if (gimple_debug_source_bind_p (s: stmt))
3248	{
3249	gimple_debug_source_bind_set_var (dbg: stmt, var: t);
3250	/* When inlining and source bind refers to one of the optimized
3251	away parameters, change the source bind into normal debug bind
3252	referring to the corresponding DEBUG_EXPR_DECL that should have
3253	been bound before the call stmt. */
3254	t = gimple_debug_source_bind_get_value (dbg: stmt);
3255	if (t != NULL_TREE
3256	&& TREE_CODE (t) == PARM_DECL
3257	&& id->call_stmt)
3258	{
3259	vec<tree, va_gc> **debug_args = decl_debug_args_lookup (id->src_fn);
3260	unsigned int i;
3261	if (debug_args != NULL)
3262	{
3263	for (i = 0; i < vec_safe_length (v: *debug_args); i += 2)
3264	if ((**debug_args)[i] == DECL_ORIGIN (t)
3265	&& TREE_CODE ((**debug_args)[i + 1]) == DEBUG_EXPR_DECL)
3266	{
3267	t = (**debug_args)[i + 1];
3268	stmt->subcode = GIMPLE_DEBUG_BIND;
3269	gimple_debug_bind_set_value (dbg: stmt, value: t);
3270	break;
3271	}
3272	}
3273	}
3274	if (gimple_debug_source_bind_p (s: stmt))
3275	walk_tree (gimple_debug_source_bind_get_value_ptr (stmt),
3276	remap_gimple_op_r, &wi, NULL);
3277	}
3278
3279	processing_debug_stmt = 0;
3280
3281	update_stmt (s: stmt);
3282	}
3283
3284	/* Process deferred debug stmts. In order to give values better odds
3285	of being successfully remapped, we delay the processing of debug
3286	stmts until all other stmts that might require remapping are
3287	processed. */
3288
3289	static void
3290	copy_debug_stmts (copy_body_data *id)
3291	{
3292	if (!id->debug_stmts.exists ())
3293	return;
3294
3295	for (gdebug *stmt : id->debug_stmts)
3296	copy_debug_stmt (stmt, id);
3297
3298	id->debug_stmts.release ();
3299	}
3300
3301	/* Make a copy of the body of SRC_FN so that it can be inserted inline in
3302	another function. */
3303
3304	static tree
3305	copy_tree_body (copy_body_data *id)
3306	{
3307	tree fndecl = id->src_fn;
3308	tree body = DECL_SAVED_TREE (fndecl);
3309
3310	walk_tree (&body, copy_tree_body_r, id, NULL);
3311
3312	return body;
3313	}
3314
3315	/* Make a copy of the body of FN so that it can be inserted inline in
3316	another function. */
3317
3318	static tree
3319	copy_body (copy_body_data *id,
3320	basic_block entry_block_map, basic_block exit_block_map,
3321	basic_block new_entry)
3322	{
3323	tree fndecl = id->src_fn;
3324	tree body;
3325
3326	/* If this body has a CFG, walk CFG and copy. */
3327	gcc_assert (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (fndecl)));
3328	body = copy_cfg_body (id, entry_block_map, exit_block_map,
3329	new_entry);
3330	copy_debug_stmts (id);
3331	delete id->killed_new_ssa_names;
3332	id->killed_new_ssa_names = NULL;
3333
3334	return body;
3335	}
3336
3337	/* Return true if VALUE is an ADDR_EXPR of an automatic variable
3338	defined in function FN, or of a data member thereof. */
3339
3340	static bool
3341	self_inlining_addr_expr (tree value, tree fn)
3342	{
3343	tree var;
3344
3345	if (TREE_CODE (value) != ADDR_EXPR)
3346	return false;
3347
3348	var = get_base_address (TREE_OPERAND (value, 0));
3349
3350	return var && auto_var_in_fn_p (var, fn);
3351	}
3352
3353	/* Append to BB a debug annotation that binds VAR to VALUE, inheriting
3354	lexical block and line number information from base_stmt, if given,
3355	or from the last stmt of the block otherwise. */
3356
3357	static gimple *
3358	insert_init_debug_bind (copy_body_data *id,
3359	basic_block bb, tree var, tree value,
3360	gimple *base_stmt)
3361	{
3362	gimple *note;
3363	gimple_stmt_iterator gsi;
3364	tree tracked_var;
3365
3366	if (!gimple_in_ssa_p (fun: id->src_cfun))
3367	return NULL;
3368
3369	if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
3370	return NULL;
3371
3372	tracked_var = target_for_debug_bind (var);
3373	if (!tracked_var)
3374	return NULL;
3375
3376	if (bb)
3377	{
3378	gsi = gsi_last_bb (bb);
3379	if (!base_stmt && !gsi_end_p (i: gsi))
3380	base_stmt = gsi_stmt (i: gsi);
3381	}
3382
3383	note = gimple_build_debug_bind (tracked_var,
3384	value == error_mark_node
3385	? NULL_TREE : unshare_expr (value),
3386	base_stmt);
3387
3388	if (bb)
3389	{
3390	if (!gsi_end_p (i: gsi))
3391	gsi_insert_after (&gsi, note, GSI_SAME_STMT);
3392	else
3393	gsi_insert_before (&gsi, note, GSI_SAME_STMT);
3394	}
3395
3396	return note;
3397	}
3398
3399	static void
3400	insert_init_stmt (copy_body_data *id, basic_block bb, gimple *init_stmt)
3401	{
3402	/* If VAR represents a zero-sized variable, it's possible that the
3403	assignment statement may result in no gimple statements. */
3404	if (init_stmt)
3405	{
3406	gimple_stmt_iterator si = gsi_last_bb (bb);
3407
3408	/* We can end up with init statements that store to a non-register
3409	from a rhs with a conversion. Handle that here by forcing the
3410	rhs into a temporary. gimple_regimplify_operands is not
3411	prepared to do this for us. */
3412	if (!is_gimple_debug (gs: init_stmt)
3413	&& !is_gimple_reg (gimple_assign_lhs (gs: init_stmt))
3414	&& is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt)))
3415	&& gimple_assign_rhs_class (gs: init_stmt) == GIMPLE_UNARY_RHS)
3416	{
3417	tree rhs = build1 (gimple_assign_rhs_code (gs: init_stmt),
3418	TREE_TYPE (gimple_assign_lhs (init_stmt)),
3419	gimple_assign_rhs1 (gs: init_stmt));
3420	rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false,
3421	GSI_NEW_STMT);
3422	gimple_assign_set_rhs_code (s: init_stmt, TREE_CODE (rhs));
3423	gimple_assign_set_rhs1 (gs: init_stmt, rhs);
3424	}
3425	gsi_insert_after (&si, init_stmt, GSI_NEW_STMT);
3426	if (!is_gimple_debug (gs: init_stmt))
3427	{
3428	gimple_regimplify_operands (init_stmt, &si);
3429
3430	tree def = gimple_assign_lhs (gs: init_stmt);
3431	insert_init_debug_bind (id, bb, var: def, value: def, base_stmt: init_stmt);
3432	}
3433	}
3434	}
3435
3436	/* Deal with mismatched formal/actual parameters, in a rather brute-force way
3437	if need be (which should only be necessary for invalid programs). Attempt
3438	to convert VAL to TYPE and return the result if it is possible, just return
3439	a zero constant of the given type if it fails. */
3440
3441	tree
3442	force_value_to_type (tree type, tree value)
3443	{
3444	/* If we can match up types by promotion/demotion do so. */
3445	if (fold_convertible_p (type, value))
3446	return fold_convert (type, value);
3447
3448	/* ??? For valid programs we should not end up here.
3449	Still if we end up with truly mismatched types here, fall back
3450	to using a VIEW_CONVERT_EXPR or a literal zero to not leak invalid
3451	GIMPLE to the following passes. */
3452	if (TREE_CODE (value) == WITH_SIZE_EXPR)
3453	return error_mark_node;
3454	else if (!is_gimple_reg_type (TREE_TYPE (value))
3455	|| TYPE_SIZE (type) == TYPE_SIZE (TREE_TYPE (value)))
3456	return fold_build1 (VIEW_CONVERT_EXPR, type, value);
3457	else
3458	return build_zero_cst (type);
3459	}
3460
3461	/* Initialize parameter P with VALUE. If needed, produce init statement
3462	at the end of BB. When BB is NULL, we return init statement to be
3463	output later. */
3464	static gimple *
3465	setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn,
3466	basic_block bb, tree *vars)
3467	{
3468	gimple *init_stmt = NULL;
3469	tree var;
3470	tree def = (gimple_in_ssa_p (cfun)
3471	? ssa_default_def (id->src_cfun, p) : NULL);
3472
3473	/* Make an equivalent VAR_DECL. Note that we must NOT remap the type
3474	here since the type of this decl must be visible to the calling
3475	function. */
3476	var = copy_decl_to_var (p, id);
3477
3478	/* Declare this new variable. */
3479	DECL_CHAIN (var) = *vars;
3480	*vars = var;
3481
3482	/* Make gimplifier happy about this variable. */
3483	DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3484
3485	/* If the parameter is never assigned to, has no SSA_NAMEs created,
3486	we would not need to create a new variable here at all, if it
3487	weren't for debug info. Still, we can just use the argument
3488	value. */
3489	if (TREE_READONLY (p)
3490	&& !TREE_ADDRESSABLE (p)
3491	&& value
3492	&& !TREE_SIDE_EFFECTS (value)
3493	&& !def)
3494	{
3495	/* We may produce non-gimple trees by adding NOPs or introduce invalid
3496	sharing when the value is not constant or DECL. And we need to make
3497	sure that it cannot be modified from another path in the callee. */
3498	if (((is_gimple_min_invariant (value)
3499	/* When the parameter is used in a context that forces it to
3500	not be a GIMPLE register avoid substituting something that
3501	is not a decl there. */
3502	&& ! DECL_NOT_GIMPLE_REG_P (p))
3503	|| (DECL_P (value) && TREE_READONLY (value))
3504	|| (auto_var_in_fn_p (value, id->dst_fn)
3505	&& !TREE_ADDRESSABLE (value)))
3506	&& useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value))
3507	/* We have to be very careful about ADDR_EXPR. Make sure
3508	the base variable isn't a local variable of the inlined
3509	function, e.g., when doing recursive inlining, direct or
3510	mutually-recursive or whatever, which is why we don't
3511	just test whether fn == current_function_decl. */
3512	&& ! self_inlining_addr_expr (value, fn))
3513	{
3514	insert_decl_map (id, key: p, value);
3515	if (!id->debug_map)
3516	id->debug_map = new hash_map<tree, tree>;
3517	id->debug_map->put (k: p, v: var);
3518	return insert_init_debug_bind (id, bb, var, value, NULL);
3519	}
3520	}
3521
3522	/* Register the VAR_DECL as the equivalent for the PARM_DECL;
3523	that way, when the PARM_DECL is encountered, it will be
3524	automatically replaced by the VAR_DECL. */
3525	insert_decl_map (id, key: p, value: var);
3526
3527	/* Even if P was TREE_READONLY, the new VAR should not be. In the original
3528	code, we would have constructed a temporary, and then the function body
3529	would have never changed the value of P. However, now, we will be
3530	constructing VAR directly. Therefore, it must not be TREE_READONLY. */
3531	TREE_READONLY (var) = 0;
3532
3533	tree rhs = value;
3534	if (value
3535	&& value != error_mark_node
3536	&& !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value)))
3537	rhs = force_value_to_type (TREE_TYPE (p), value);
3538
3539	/* If there is no setup required and we are in SSA, take the easy route
3540	replacing all SSA names representing the function parameter by the
3541	SSA name passed to function.
3542
3543	We need to construct map for the variable anyway as it might be used
3544	in different SSA names when parameter is set in function.
3545
3546	Do replacement at -O0 for const arguments replaced by constant.
3547	This is important for builtin_constant_p and other construct requiring
3548	constant argument to be visible in inlined function body. */
3549	if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p)
3550	&& (optimize
3551	|| (TREE_READONLY (p)
3552	&& is_gimple_min_invariant (rhs)))
3553	&& (TREE_CODE (rhs) == SSA_NAME
3554	|| is_gimple_min_invariant (rhs))
3555	&& !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
3556	{
3557	insert_decl_map (id, key: def, value: rhs);
3558	return insert_init_debug_bind (id, bb, var, value: rhs, NULL);
3559	}
3560
3561	/* If the value of argument is never used, don't care about initializing
3562	it. */
3563	if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p))
3564	{
3565	gcc_assert (!value || !TREE_SIDE_EFFECTS (value));
3566	return insert_init_debug_bind (id, bb, var, value: rhs, NULL);
3567	}
3568
3569	/* Initialize this VAR_DECL from the equivalent argument. Convert
3570	the argument to the proper type in case it was promoted. */
3571	if (value)
3572	{
3573	if (rhs == error_mark_node)
3574	{
3575	insert_decl_map (id, key: p, value: var);
3576	return insert_init_debug_bind (id, bb, var, value: rhs, NULL);
3577	}
3578
3579	STRIP_USELESS_TYPE_CONVERSION (rhs);
3580
3581	/* If we are in SSA form properly remap the default definition. */
3582	if (gimple_in_ssa_p (cfun) && is_gimple_reg (p))
3583	{
3584	if (def)
3585	{
3586	def = remap_ssa_name (name: def, id);
3587	init_stmt = gimple_build_assign (def, rhs);
3588	SSA_NAME_IS_DEFAULT_DEF (def) = 0;
3589	set_ssa_default_def (cfun, var, NULL);
3590	}
3591	}
3592	else if (!is_empty_type (TREE_TYPE (var)))
3593	init_stmt = gimple_build_assign (var, rhs);
3594
3595	if (bb && init_stmt)
3596	insert_init_stmt (id, bb, init_stmt);
3597	}
3598	return init_stmt;
3599	}
3600
3601	/* Generate code to initialize the parameters of the function at the
3602	top of the stack in ID from the GIMPLE_CALL STMT. */
3603
3604	static void
3605	initialize_inlined_parameters (copy_body_data *id, gimple *stmt,
3606	tree fn, basic_block bb)
3607	{
3608	tree parms;
3609	size_t i;
3610	tree p;
3611	tree vars = NULL_TREE;
3612	tree static_chain = gimple_call_chain (gs: stmt);
3613
3614	/* Figure out what the parameters are. */
3615	parms = DECL_ARGUMENTS (fn);
3616
3617	/* Loop through the parameter declarations, replacing each with an
3618	equivalent VAR_DECL, appropriately initialized. */
3619	for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3620	{
3621	tree val;
3622	val = i < gimple_call_num_args (gs: stmt) ? gimple_call_arg (gs: stmt, index: i) : NULL;
3623	setup_one_parameter (id, p, value: val, fn, bb, vars: &vars);
3624	}
3625	/* After remapping parameters remap their types. This has to be done
3626	in a second loop over all parameters to appropriately remap
3627	variable sized arrays when the size is specified in a
3628	parameter following the array. */
3629	for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3630	{
3631	tree *varp = id->decl_map->get (k: p);
3632	if (varp && VAR_P (*varp))
3633	{
3634	tree def = (gimple_in_ssa_p (cfun) && is_gimple_reg (p)
3635	? ssa_default_def (id->src_cfun, p) : NULL);
3636	tree var = *varp;
3637	TREE_TYPE (var) = remap_type (TREE_TYPE (var), id);
3638	/* Also remap the default definition if it was remapped
3639	to the default definition of the parameter replacement
3640	by the parameter setup. */
3641	if (def)
3642	{
3643	tree *defp = id->decl_map->get (k: def);
3644	if (defp
3645	&& TREE_CODE (*defp) == SSA_NAME
3646	&& SSA_NAME_VAR (*defp) == var)
3647	TREE_TYPE (*defp) = TREE_TYPE (var);
3648	}
3649	/* When not optimizing and the parameter is unused, assign to
3650	a dummy SSA name. Do this after remapping the type above. */
3651	else if (!optimize
3652	&& is_gimple_reg (p)
3653	&& i < gimple_call_num_args (gs: stmt))
3654	{
3655	tree val = gimple_call_arg (gs: stmt, index: i);
3656	if (val != error_mark_node)
3657	{
3658	if (!useless_type_conversion_p (TREE_TYPE (p),
3659	TREE_TYPE (val)))
3660	val = force_value_to_type (TREE_TYPE (p), value: val);
3661	def = make_ssa_name (var);
3662	gimple *init_stmt = gimple_build_assign (def, val);
3663	insert_init_stmt (id, bb, init_stmt);
3664	}
3665	}
3666	}
3667	}
3668
3669	/* Initialize the static chain. */
3670	p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl;
3671	gcc_assert (fn != current_function_decl);
3672	if (p)
3673	{
3674	/* No static chain? Seems like a bug in tree-nested.cc. */
3675	gcc_assert (static_chain);
3676
3677	setup_one_parameter (id, p, value: static_chain, fn, bb, vars: &vars);
3678	}
3679
3680	declare_inline_vars (id->block, vars);
3681	}
3682
3683
3684	/* Declare a return variable to replace the RESULT_DECL for the
3685	function we are calling. An appropriate DECL_STMT is returned.
3686	The USE_STMT is filled to contain a use of the declaration to
3687	indicate the return value of the function.
3688
3689	RETURN_SLOT, if non-null is place where to store the result. It
3690	is set only for CALL_EXPR_RETURN_SLOT_OPT. MODIFY_DEST, if non-null,
3691	was the LHS of the MODIFY_EXPR to which this call is the RHS.
3692
3693	The return value is a (possibly null) value that holds the result
3694	as seen by the caller. */
3695
3696	static tree
3697	declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest,
3698	basic_block entry_bb)
3699	{
3700	tree callee = id->src_fn;
3701	tree result = DECL_RESULT (callee);
3702	tree callee_type = TREE_TYPE (result);
3703	tree caller_type;
3704	tree var, use;
3705
3706	/* Handle type-mismatches in the function declaration return type
3707	vs. the call expression. */
3708	if (modify_dest)
3709	caller_type = TREE_TYPE (modify_dest);
3710	else if (return_slot)
3711	caller_type = TREE_TYPE (return_slot);
3712	else /* No LHS on the call. */
3713	caller_type = TREE_TYPE (TREE_TYPE (callee));
3714
3715	/* We don't need to do anything for functions that don't return anything. */
3716	if (VOID_TYPE_P (callee_type))
3717	return NULL_TREE;
3718
3719	/* If there was a return slot, then the return value is the
3720	dereferenced address of that object. */
3721	if (return_slot)
3722	{
3723	/* The front end shouldn't have used both return_slot and
3724	a modify expression. */
3725	gcc_assert (!modify_dest);
3726	if (DECL_BY_REFERENCE (result))
3727	{
3728	tree return_slot_addr = build_fold_addr_expr (return_slot);
3729	STRIP_USELESS_TYPE_CONVERSION (return_slot_addr);
3730
3731	/* We are going to construct *&return_slot and we can't do that
3732	for variables believed to be not addressable.
3733
3734	FIXME: This check possibly can match, because values returned
3735	via return slot optimization are not believed to have address
3736	taken by alias analysis. */
3737	gcc_assert (TREE_CODE (return_slot) != SSA_NAME);
3738	var = return_slot_addr;
3739	mark_addressable (return_slot);
3740	}
3741	else
3742	{
3743	var = return_slot;
3744	gcc_assert (TREE_CODE (var) != SSA_NAME);
3745	if (TREE_ADDRESSABLE (result))
3746	mark_addressable (var);
3747	}
3748	if (DECL_NOT_GIMPLE_REG_P (result)
3749	&& DECL_P (var))
3750	DECL_NOT_GIMPLE_REG_P (var) = 1;
3751
3752	if (!useless_type_conversion_p (callee_type, caller_type))
3753	var = build1 (VIEW_CONVERT_EXPR, callee_type, var);
3754
3755	use = NULL;
3756	goto done;
3757	}
3758
3759	/* All types requiring non-trivial constructors should have been handled. */
3760	gcc_assert (!TREE_ADDRESSABLE (callee_type));
3761
3762	/* Attempt to avoid creating a new temporary variable. */
3763	if (modify_dest
3764	&& TREE_CODE (modify_dest) != SSA_NAME)
3765	{
3766	bool use_it = false;
3767
3768	/* We can't use MODIFY_DEST if there's type promotion involved. */
3769	if (!useless_type_conversion_p (callee_type, caller_type))
3770	use_it = false;
3771
3772	/* ??? If we're assigning to a variable sized type, then we must
3773	reuse the destination variable, because we've no good way to
3774	create variable sized temporaries at this point. */
3775	else if (!poly_int_tree_p (TYPE_SIZE_UNIT (caller_type)))
3776	use_it = true;
3777
3778	/* If the callee cannot possibly modify MODIFY_DEST, then we can
3779	reuse it as the result of the call directly. Don't do this if
3780	it would promote MODIFY_DEST to addressable. */
3781	else if (TREE_ADDRESSABLE (result))
3782	use_it = false;
3783	else
3784	{
3785	tree base_m = get_base_address (t: modify_dest);
3786
3787	/* If the base isn't a decl, then it's a pointer, and we don't
3788	know where that's going to go. */
3789	if (!DECL_P (base_m))
3790	use_it = false;
3791	else if (is_global_var (t: base_m))
3792	use_it = false;
3793	else if (DECL_NOT_GIMPLE_REG_P (result)
3794	&& !DECL_NOT_GIMPLE_REG_P (base_m))
3795	use_it = false;
3796	else if (!TREE_ADDRESSABLE (base_m))
3797	use_it = true;
3798	}
3799
3800	if (use_it)
3801	{
3802	var = modify_dest;
3803	use = NULL;
3804	goto done;
3805	}
3806	}
3807
3808	gcc_assert (poly_int_tree_p (TYPE_SIZE_UNIT (callee_type)));
3809
3810	var = copy_result_decl_to_var (result, id);
3811	DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3812
3813	/* Do not have the rest of GCC warn about this variable as it should
3814	not be visible to the user. */
3815	suppress_warning (var /* OPT_Wuninitialized? */);
3816
3817	declare_inline_vars (id->block, var);
3818
3819	/* Build the use expr. If the return type of the function was
3820	promoted, convert it back to the expected type. */
3821	use = var;
3822	if (!useless_type_conversion_p (caller_type, TREE_TYPE (var)))
3823	{
3824	/* If we can match up types by promotion/demotion do so. */
3825	if (fold_convertible_p (caller_type, var))
3826	use = fold_convert (caller_type, var);
3827	else
3828	{
3829	/* ??? For valid programs we should not end up here.
3830	Still if we end up with truly mismatched types here, fall back
3831	to using a MEM_REF to not leak invalid GIMPLE to the following
3832	passes. */
3833	/* Prevent var from being written into SSA form. */
3834	if (is_gimple_reg_type (TREE_TYPE (var)))
3835	DECL_NOT_GIMPLE_REG_P (var) = true;
3836	use = fold_build2 (MEM_REF, caller_type,
3837	build_fold_addr_expr (var),
3838	build_int_cst (ptr_type_node, 0));
3839	}
3840	}
3841
3842	STRIP_USELESS_TYPE_CONVERSION (use);
3843
3844	if (DECL_BY_REFERENCE (result))
3845	{
3846	TREE_ADDRESSABLE (var) = 1;
3847	var = build_fold_addr_expr (var);
3848	}
3849
3850	done:
3851	/* Register the VAR_DECL as the equivalent for the RESULT_DECL; that
3852	way, when the RESULT_DECL is encountered, it will be
3853	automatically replaced by the VAR_DECL.
3854
3855	When returning by reference, ensure that RESULT_DECL remaps to
3856	gimple_val. */
3857	if (DECL_BY_REFERENCE (result)
3858	&& !is_gimple_val (var))
3859	{
3860	tree temp = create_tmp_var (TREE_TYPE (result), "retvalptr");
3861	insert_decl_map (id, key: result, value: temp);
3862	/* When RESULT_DECL is in SSA form, we need to remap and initialize
3863	it's default_def SSA_NAME. */
3864	if (gimple_in_ssa_p (fun: id->src_cfun)
3865	&& is_gimple_reg (result))
3866	if (tree default_def = ssa_default_def (id->src_cfun, result))
3867	{
3868	temp = make_ssa_name (var: temp);
3869	insert_decl_map (id, key: default_def, value: temp);
3870	}
3871	insert_init_stmt (id, bb: entry_bb, init_stmt: gimple_build_assign (temp, var));
3872	}
3873	else
3874	insert_decl_map (id, key: result, value: var);
3875
3876	/* Remember this so we can ignore it in remap_decls. */
3877	id->retvar = var;
3878	return use;
3879	}
3880
3881	/* Determine if the function can be copied. If so return NULL. If
3882	not return a string describng the reason for failure. */
3883
3884	const char *
3885	copy_forbidden (struct function *fun)
3886	{
3887	const char *reason = fun->cannot_be_copied_reason;
3888
3889	/* Only examine the function once. */
3890	if (fun->cannot_be_copied_set)
3891	return reason;
3892
3893	/* We cannot copy a function that receives a non-local goto
3894	because we cannot remap the destination label used in the
3895	function that is performing the non-local goto. */
3896	/* ??? Actually, this should be possible, if we work at it.
3897	No doubt there's just a handful of places that simply
3898	assume it doesn't happen and don't substitute properly. */
3899	if (fun->has_nonlocal_label)
3900	{
3901	reason = G_("function %q+F can never be copied "
3902	"because it receives a non-local goto");
3903	goto fail;
3904	}
3905
3906	if (fun->has_forced_label_in_static)
3907	{
3908	reason = G_("function %q+F can never be copied because it saves "
3909	"address of local label in a static variable");
3910	goto fail;
3911	}
3912
3913	fail:
3914	fun->cannot_be_copied_reason = reason;
3915	fun->cannot_be_copied_set = true;
3916	return reason;
3917	}
3918
3919
3920	static const char *inline_forbidden_reason;
3921
3922	/* A callback for walk_gimple_seq to handle statements. Returns non-null
3923	iff a function cannot be inlined. Also sets the reason why. */
3924
3925	static tree
3926	inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3927	struct walk_stmt_info *wip)
3928	{
3929	tree fn = (tree) wip->info;
3930	tree t;
3931	gimple *stmt = gsi_stmt (i: *gsi);
3932
3933	switch (gimple_code (g: stmt))
3934	{
3935	case GIMPLE_CALL:
3936	/* Refuse to inline alloca call unless user explicitly forced so as
3937	this may change program's memory overhead drastically when the
3938	function using alloca is called in loop. In GCC present in
3939	SPEC2000 inlining into schedule_block cause it to require 2GB of
3940	RAM instead of 256MB. Don't do so for alloca calls emitted for
3941	VLA objects as those can't cause unbounded growth (they're always
3942	wrapped inside stack_save/stack_restore regions. */
3943	if (gimple_maybe_alloca_call_p (stmt)
3944	&& !gimple_call_alloca_for_var_p (s: as_a <gcall *> (p: stmt))
3945	&& !lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fn)))
3946	{
3947	inline_forbidden_reason
3948	= G_("function %q+F can never be inlined because it uses "
3949	"alloca (override using the always_inline attribute)");
3950	*handled_ops_p = true;
3951	return fn;
3952	}
3953
3954	t = gimple_call_fndecl (gs: stmt);
3955	if (t == NULL_TREE)
3956	break;
3957
3958	/* We cannot inline functions that call setjmp. */
3959	if (setjmp_call_p (t))
3960	{
3961	inline_forbidden_reason
3962	= G_("function %q+F can never be inlined because it uses setjmp");
3963	*handled_ops_p = true;
3964	return t;
3965	}
3966
3967	if (DECL_BUILT_IN_CLASS (t) == BUILT_IN_NORMAL)
3968	switch (DECL_FUNCTION_CODE (decl: t))
3969	{
3970	/* We cannot inline functions that take a variable number of
3971	arguments. */
3972	case BUILT_IN_VA_START:
3973	case BUILT_IN_NEXT_ARG:
3974	case BUILT_IN_VA_END:
3975	inline_forbidden_reason
3976	= G_("function %q+F can never be inlined because it "
3977	"uses variable argument lists");
3978	*handled_ops_p = true;
3979	return t;
3980
3981	case BUILT_IN_LONGJMP:
3982	/* We can't inline functions that call __builtin_longjmp at
3983	all. The non-local goto machinery really requires the
3984	destination be in a different function. If we allow the
3985	function calling __builtin_longjmp to be inlined into the
3986	function calling __builtin_setjmp, Things will Go Awry. */
3987	inline_forbidden_reason
3988	= G_("function %q+F can never be inlined because "
3989	"it uses setjmp-longjmp exception handling");
3990	*handled_ops_p = true;
3991	return t;
3992
3993	case BUILT_IN_NONLOCAL_GOTO:
3994	/* Similarly. */
3995	inline_forbidden_reason
3996	= G_("function %q+F can never be inlined because "
3997	"it uses non-local goto");
3998	*handled_ops_p = true;
3999	return t;
4000
4001	case BUILT_IN_RETURN:
4002	case BUILT_IN_APPLY_ARGS:
4003	/* If a __builtin_apply_args caller would be inlined,
4004	it would be saving arguments of the function it has
4005	been inlined into. Similarly __builtin_return would
4006	return from the function the inline has been inlined into. */
4007	inline_forbidden_reason
4008	= G_("function %q+F can never be inlined because "
4009	"it uses %<__builtin_return%> or %<__builtin_apply_args%>");
4010	*handled_ops_p = true;
4011	return t;
4012
4013	default:
4014	break;
4015	}
4016	break;
4017
4018	case GIMPLE_GOTO:
4019	t = gimple_goto_dest (gs: stmt);
4020
4021	/* We will not inline a function which uses computed goto. The
4022	addresses of its local labels, which may be tucked into
4023	global storage, are of course not constant across
4024	instantiations, which causes unexpected behavior. */
4025	if (TREE_CODE (t) != LABEL_DECL)
4026	{
4027	inline_forbidden_reason
4028	= G_("function %q+F can never be inlined "
4029	"because it contains a computed goto");
4030	*handled_ops_p = true;
4031	return t;
4032	}
4033	break;
4034
4035	default:
4036	break;
4037	}
4038
4039	*handled_ops_p = false;
4040	return NULL_TREE;
4041	}
4042
4043	/* Return true if FNDECL is a function that cannot be inlined into
4044	another one. */
4045
4046	static bool
4047	inline_forbidden_p (tree fndecl)
4048	{
4049	struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
4050	struct walk_stmt_info wi;
4051	basic_block bb;
4052	bool forbidden_p = false;
4053
4054	/* First check for shared reasons not to copy the code. */
4055	inline_forbidden_reason = copy_forbidden (fun);
4056	if (inline_forbidden_reason != NULL)
4057	return true;
4058
4059	/* Next, walk the statements of the function looking for
4060	constraucts we can't handle, or are non-optimal for inlining. */
4061	hash_set<tree> visited_nodes;
4062	memset (s: &wi, c: 0, n: sizeof (wi));
4063	wi.info = (void *) fndecl;
4064	wi.pset = &visited_nodes;
4065
4066	/* We cannot inline a function with a variable-sized parameter because we
4067	cannot materialize a temporary of such a type in the caller if need be.
4068	Note that the return case is not symmetrical because we can guarantee
4069	that a temporary is not needed by means of CALL_EXPR_RETURN_SLOT_OPT. */
4070	for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
4071	if (!poly_int_tree_p (DECL_SIZE (parm)))
4072	{
4073	inline_forbidden_reason
4074	= G_("function %q+F can never be inlined because "
4075	"it has a VLA argument");
4076	return true;
4077	}
4078
4079	FOR_EACH_BB_FN (bb, fun)
4080	{
4081	gimple *ret;
4082	gimple_seq seq = bb_seq (bb);
4083	ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, NULL, &wi);
4084	forbidden_p = (ret != NULL);
4085	if (forbidden_p)
4086	break;
4087	}
4088
4089	return forbidden_p;
4090	}
4091
4092	/* Return false if the function FNDECL cannot be inlined on account of its
4093	attributes, true otherwise. */
4094	static bool
4095	function_attribute_inlinable_p (const_tree fndecl)
4096	{
4097	if (targetm.attribute_table)
4098	{
4099	const_tree a;
4100
4101	for (a = DECL_ATTRIBUTES (fndecl); a; a = TREE_CHAIN (a))
4102	{
4103	const_tree name = get_attribute_name (a);
4104	int i;
4105
4106	for (i = 0; targetm.attribute_table[i].name != NULL; i++)
4107	if (is_attribute_p (attr_name: targetm.attribute_table[i].name, ident: name))
4108	return targetm.function_attribute_inlinable_p (fndecl);
4109	}
4110	}
4111
4112	return true;
4113	}
4114
4115	/* Returns nonzero if FN is a function that does not have any
4116	fundamental inline blocking properties. */
4117
4118	bool
4119	tree_inlinable_function_p (tree fn)
4120	{
4121	bool inlinable = true;
4122	bool do_warning;
4123	tree always_inline;
4124
4125	/* If we've already decided this function shouldn't be inlined,
4126	there's no need to check again. */
4127	if (DECL_UNINLINABLE (fn))
4128	return false;
4129
4130	/* We only warn for functions declared `inline' by the user. */
4131	do_warning = (opt_for_fn (fn, warn_inline)
4132	&& DECL_DECLARED_INLINE_P (fn)
4133	&& !DECL_NO_INLINE_WARNING_P (fn)
4134	&& !DECL_IN_SYSTEM_HEADER (fn));
4135
4136	always_inline = lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fn));
4137
4138	if (flag_no_inline
4139	&& always_inline == NULL)
4140	{
4141	if (do_warning)
4142	warning (OPT_Winline, "function %q+F can never be inlined because it "
4143	"is suppressed using %<-fno-inline%>", fn);
4144	inlinable = false;
4145	}
4146
4147	else if (!function_attribute_inlinable_p (fndecl: fn))
4148	{
4149	if (do_warning)
4150	warning (OPT_Winline, "function %q+F can never be inlined because it "
4151	"uses attributes conflicting with inlining", fn);
4152	inlinable = false;
4153	}
4154
4155	else if (inline_forbidden_p (fndecl: fn))
4156	{
4157	/* See if we should warn about uninlinable functions. Previously,
4158	some of these warnings would be issued while trying to expand
4159	the function inline, but that would cause multiple warnings
4160	about functions that would for example call alloca. But since
4161	this a property of the function, just one warning is enough.
4162	As a bonus we can now give more details about the reason why a
4163	function is not inlinable. */
4164	if (always_inline)
4165	error (inline_forbidden_reason, fn);
4166	else if (do_warning)
4167	warning (OPT_Winline, inline_forbidden_reason, fn);
4168
4169	inlinable = false;
4170	}
4171
4172	/* Squirrel away the result so that we don't have to check again. */
4173	DECL_UNINLINABLE (fn) = !inlinable;
4174
4175	return inlinable;
4176	}
4177
4178	/* Estimate the cost of a memory move of type TYPE. Use machine dependent
4179	word size and take possible memcpy call into account and return
4180	cost based on whether optimizing for size or speed according to SPEED_P. */
4181
4182	int
4183	estimate_move_cost (tree type, bool ARG_UNUSED (speed_p))
4184	{
4185	HOST_WIDE_INT size;
4186
4187	gcc_assert (!VOID_TYPE_P (type));
4188
4189	if (VECTOR_TYPE_P (type))
4190	{
4191	scalar_mode inner = SCALAR_TYPE_MODE (TREE_TYPE (type));
4192	machine_mode simd = targetm.vectorize.preferred_simd_mode (inner);
4193	int orig_mode_size
4194	= estimated_poly_value (x: GET_MODE_SIZE (TYPE_MODE (type)));
4195	int simd_mode_size = estimated_poly_value (x: GET_MODE_SIZE (mode: simd));
4196	return ((orig_mode_size + simd_mode_size - 1)
4197	/ simd_mode_size);
4198	}
4199
4200	size = int_size_in_bytes (type);
4201
4202	if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (speed_p))
4203	/* Cost of a memcpy call, 3 arguments and the call. */
4204	return 4;
4205	else
4206	return ((size + MOVE_MAX_PIECES - 1) / MOVE_MAX_PIECES);
4207	}
4208
4209	/* Returns cost of operation CODE, according to WEIGHTS */
4210
4211	static int
4212	estimate_operator_cost (enum tree_code code, eni_weights *weights,
4213	tree op1 ATTRIBUTE_UNUSED, tree op2)
4214	{
4215	switch (code)
4216	{
4217	/* These are "free" conversions, or their presumed cost
4218	is folded into other operations. */
4219	case RANGE_EXPR:
4220	CASE_CONVERT:
4221	case COMPLEX_EXPR:
4222	case PAREN_EXPR:
4223	case VIEW_CONVERT_EXPR:
4224	return 0;
4225
4226	/* Assign cost of 1 to usual operations.
4227	??? We may consider mapping RTL costs to this. */
4228	case COND_EXPR:
4229	case VEC_COND_EXPR:
4230	case VEC_PERM_EXPR:
4231
4232	case PLUS_EXPR:
4233	case POINTER_PLUS_EXPR:
4234	case POINTER_DIFF_EXPR:
4235	case MINUS_EXPR:
4236	case MULT_EXPR:
4237	case MULT_HIGHPART_EXPR:
4238
4239	case ADDR_SPACE_CONVERT_EXPR:
4240	case FIXED_CONVERT_EXPR:
4241	case FIX_TRUNC_EXPR:
4242
4243	case NEGATE_EXPR:
4244	case FLOAT_EXPR:
4245	case MIN_EXPR:
4246	case MAX_EXPR:
4247	case ABS_EXPR:
4248	case ABSU_EXPR:
4249
4250	case LSHIFT_EXPR:
4251	case RSHIFT_EXPR:
4252	case LROTATE_EXPR:
4253	case RROTATE_EXPR:
4254
4255	case BIT_IOR_EXPR:
4256	case BIT_XOR_EXPR:
4257	case BIT_AND_EXPR:
4258	case BIT_NOT_EXPR:
4259
4260	case TRUTH_ANDIF_EXPR:
4261	case TRUTH_ORIF_EXPR:
4262	case TRUTH_AND_EXPR:
4263	case TRUTH_OR_EXPR:
4264	case TRUTH_XOR_EXPR:
4265	case TRUTH_NOT_EXPR:
4266
4267	case LT_EXPR:
4268	case LE_EXPR:
4269	case GT_EXPR:
4270	case GE_EXPR:
4271	case EQ_EXPR:
4272	case NE_EXPR:
4273	case ORDERED_EXPR:
4274	case UNORDERED_EXPR:
4275
4276	case UNLT_EXPR:
4277	case UNLE_EXPR:
4278	case UNGT_EXPR:
4279	case UNGE_EXPR:
4280	case UNEQ_EXPR:
4281	case LTGT_EXPR:
4282
4283	case CONJ_EXPR:
4284
4285	case PREDECREMENT_EXPR:
4286	case PREINCREMENT_EXPR:
4287	case POSTDECREMENT_EXPR:
4288	case POSTINCREMENT_EXPR:
4289
4290	case REALIGN_LOAD_EXPR:
4291
4292	case WIDEN_SUM_EXPR:
4293	case WIDEN_MULT_EXPR:
4294	case DOT_PROD_EXPR:
4295	case SAD_EXPR:
4296	case WIDEN_MULT_PLUS_EXPR:
4297	case WIDEN_MULT_MINUS_EXPR:
4298	case WIDEN_LSHIFT_EXPR:
4299
4300	case VEC_WIDEN_MULT_HI_EXPR:
4301	case VEC_WIDEN_MULT_LO_EXPR:
4302	case VEC_WIDEN_MULT_EVEN_EXPR:
4303	case VEC_WIDEN_MULT_ODD_EXPR:
4304	case VEC_UNPACK_HI_EXPR:
4305	case VEC_UNPACK_LO_EXPR:
4306	case VEC_UNPACK_FLOAT_HI_EXPR:
4307	case VEC_UNPACK_FLOAT_LO_EXPR:
4308	case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
4309	case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
4310	case VEC_PACK_TRUNC_EXPR:
4311	case VEC_PACK_SAT_EXPR:
4312	case VEC_PACK_FIX_TRUNC_EXPR:
4313	case VEC_PACK_FLOAT_EXPR:
4314	case VEC_WIDEN_LSHIFT_HI_EXPR:
4315	case VEC_WIDEN_LSHIFT_LO_EXPR:
4316	case VEC_DUPLICATE_EXPR:
4317	case VEC_SERIES_EXPR:
4318
4319	return 1;
4320
4321	/* Few special cases of expensive operations. This is useful
4322	to avoid inlining on functions having too many of these. */
4323	case TRUNC_DIV_EXPR:
4324	case CEIL_DIV_EXPR:
4325	case FLOOR_DIV_EXPR:
4326	case ROUND_DIV_EXPR:
4327	case EXACT_DIV_EXPR:
4328	case TRUNC_MOD_EXPR:
4329	case CEIL_MOD_EXPR:
4330	case FLOOR_MOD_EXPR:
4331	case ROUND_MOD_EXPR:
4332	case RDIV_EXPR:
4333	if (TREE_CODE (op2) != INTEGER_CST)
4334	return weights->div_mod_cost;
4335	return 1;
4336
4337	/* Bit-field insertion needs several shift and mask operations. */
4338	case BIT_INSERT_EXPR:
4339	return 3;
4340
4341	default:
4342	/* We expect a copy assignment with no operator. */
4343	gcc_assert (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS);
4344	return 0;
4345	}
4346	}
4347
4348
4349	/* Estimate number of instructions that will be created by expanding
4350	the statements in the statement sequence STMTS.
4351	WEIGHTS contains weights attributed to various constructs. */
4352
4353	int
4354	estimate_num_insns_seq (gimple_seq stmts, eni_weights *weights)
4355	{
4356	int cost;
4357	gimple_stmt_iterator gsi;
4358
4359	cost = 0;
4360	for (gsi = gsi_start (seq&: stmts); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
4361	cost += estimate_num_insns (gsi_stmt (i: gsi), weights);
4362
4363	return cost;
4364	}
4365
4366
4367	/* Estimate number of instructions that will be created by expanding STMT.
4368	WEIGHTS contains weights attributed to various constructs. */
4369
4370	int
4371	estimate_num_insns (gimple *stmt, eni_weights *weights)
4372	{
4373	unsigned cost, i;
4374	enum gimple_code code = gimple_code (g: stmt);
4375	tree lhs;
4376	tree rhs;
4377
4378	switch (code)
4379	{
4380	case GIMPLE_ASSIGN:
4381	/* Try to estimate the cost of assignments. We have three cases to
4382	deal with:
4383	1) Simple assignments to registers;
4384	2) Stores to things that must live in memory. This includes
4385	"normal" stores to scalars, but also assignments of large
4386	structures, or constructors of big arrays;
4387
4388	Let us look at the first two cases, assuming we have "a = b + C":
4389	<GIMPLE_ASSIGN <var_decl "a">
4390	<plus_expr <var_decl "b"> <constant C>>
4391	If "a" is a GIMPLE register, the assignment to it is free on almost
4392	any target, because "a" usually ends up in a real register. Hence
4393	the only cost of this expression comes from the PLUS_EXPR, and we
4394	can ignore the GIMPLE_ASSIGN.
4395	If "a" is not a GIMPLE register, the assignment to "a" will most
4396	likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost
4397	of moving something into "a", which we compute using the function
4398	estimate_move_cost. */
4399	if (gimple_clobber_p (s: stmt))
4400	return 0; /* ={v} {CLOBBER} stmt expands to nothing. */
4401
4402	lhs = gimple_assign_lhs (gs: stmt);
4403	rhs = gimple_assign_rhs1 (gs: stmt);
4404
4405	cost = 0;
4406
4407	/* Account for the cost of moving to / from memory. */
4408	if (gimple_store_p (gs: stmt))
4409	cost += estimate_move_cost (TREE_TYPE (lhs), speed_p: weights->time_based);
4410	if (gimple_assign_load_p (stmt))
4411	cost += estimate_move_cost (TREE_TYPE (rhs), speed_p: weights->time_based);
4412
4413	cost += estimate_operator_cost (code: gimple_assign_rhs_code (gs: stmt), weights,
4414	op1: gimple_assign_rhs1 (gs: stmt),
4415	op2: get_gimple_rhs_class (code: gimple_assign_rhs_code (gs: stmt))
4416	== GIMPLE_BINARY_RHS
4417	? gimple_assign_rhs2 (gs: stmt) : NULL);
4418	break;
4419
4420	case GIMPLE_COND:
4421	cost = 1 + estimate_operator_cost (code: gimple_cond_code (gs: stmt), weights,
4422	op1: gimple_op (gs: stmt, i: 0),
4423	op2: gimple_op (gs: stmt, i: 1));
4424	break;
4425
4426	case GIMPLE_SWITCH:
4427	{
4428	gswitch *switch_stmt = as_a <gswitch *> (p: stmt);
4429	/* Take into account cost of the switch + guess 2 conditional jumps for
4430	each case label.
4431
4432	TODO: once the switch expansion logic is sufficiently separated, we can
4433	do better job on estimating cost of the switch. */
4434	if (weights->time_based)
4435	cost = floor_log2 (x: gimple_switch_num_labels (gs: switch_stmt)) * 2;
4436	else
4437	cost = gimple_switch_num_labels (gs: switch_stmt) * 2;
4438	}
4439	break;
4440
4441	case GIMPLE_CALL:
4442	{
4443	tree decl;
4444
4445	if (gimple_call_internal_p (gs: stmt))
4446	return 0;
4447	else if ((decl = gimple_call_fndecl (gs: stmt))
4448	&& fndecl_built_in_p (node: decl))
4449	{
4450	/* Do not special case builtins where we see the body.
4451	This just confuse inliner. */
4452	struct cgraph_node *node;
4453	if ((node = cgraph_node::get (decl))
4454	&& node->definition)
4455	;
4456	/* For buitins that are likely expanded to nothing or
4457	inlined do not account operand costs. */
4458	else if (is_simple_builtin (decl))
4459	return 0;
4460	else if (is_inexpensive_builtin (decl))
4461	return weights->target_builtin_call_cost;
4462	else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
4463	{
4464	/* We canonicalize x * x to pow (x, 2.0) with -ffast-math, so
4465	specialize the cheap expansion we do here.
4466	??? This asks for a more general solution. */
4467	switch (DECL_FUNCTION_CODE (decl))
4468	{
4469	case BUILT_IN_POW:
4470	case BUILT_IN_POWF:
4471	case BUILT_IN_POWL:
4472	if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
4473	&& (real_equal
4474	(&TREE_REAL_CST (gimple_call_arg (stmt, 1)),
4475	&dconst2)))
4476	return estimate_operator_cost
4477	(code: MULT_EXPR, weights, op1: gimple_call_arg (gs: stmt, index: 0),
4478	op2: gimple_call_arg (gs: stmt, index: 0));
4479	break;
4480
4481	default:
4482	break;
4483	}
4484	}
4485	}
4486
4487	cost = decl ? weights->call_cost : weights->indirect_call_cost;
4488	if (gimple_call_lhs (gs: stmt))
4489	cost += estimate_move_cost (TREE_TYPE (gimple_call_lhs (stmt)),
4490	speed_p: weights->time_based);
4491	for (i = 0; i < gimple_call_num_args (gs: stmt); i++)
4492	{
4493	tree arg = gimple_call_arg (gs: stmt, index: i);
4494	cost += estimate_move_cost (TREE_TYPE (arg),
4495	speed_p: weights->time_based);
4496	}
4497	break;
4498	}
4499
4500	case GIMPLE_RETURN:
4501	return weights->return_cost;
4502
4503	case GIMPLE_GOTO:
4504	case GIMPLE_LABEL:
4505	case GIMPLE_NOP:
4506	case GIMPLE_PHI:
4507	case GIMPLE_PREDICT:
4508	case GIMPLE_DEBUG:
4509	return 0;
4510
4511	case GIMPLE_ASM:
4512	{
4513	int count = asm_str_count (templ: gimple_asm_string (asm_stmt: as_a <gasm *> (p: stmt)));
4514	/* 1000 means infinity. This avoids overflows later
4515	with very long asm statements. */
4516	if (count > 1000)
4517	count = 1000;
4518	/* If this asm is asm inline, count anything as minimum size. */
4519	if (gimple_asm_inline_p (asm_stmt: as_a <gasm *> (p: stmt)))
4520	count = MIN (1, count);
4521	return MAX (1, count);
4522	}
4523
4524	case GIMPLE_RESX:
4525	/* This is either going to be an external function call with one
4526	argument, or two register copy statements plus a goto. */
4527	return 2;
4528
4529	case GIMPLE_EH_DISPATCH:
4530	/* ??? This is going to turn into a switch statement. Ideally
4531	we'd have a look at the eh region and estimate the number of
4532	edges involved. */
4533	return 10;
4534
4535	case GIMPLE_BIND:
4536	return estimate_num_insns_seq (
4537	stmts: gimple_bind_body (gs: as_a <gbind *> (p: stmt)),
4538	weights);
4539
4540	case GIMPLE_EH_FILTER:
4541	return estimate_num_insns_seq (stmts: gimple_eh_filter_failure (gs: stmt), weights);
4542
4543	case GIMPLE_CATCH:
4544	return estimate_num_insns_seq (stmts: gimple_catch_handler (
4545	catch_stmt: as_a <gcatch *> (p: stmt)),
4546	weights);
4547
4548	case GIMPLE_TRY:
4549	return (estimate_num_insns_seq (stmts: gimple_try_eval (gs: stmt), weights)
4550	+ estimate_num_insns_seq (stmts: gimple_try_cleanup (gs: stmt), weights));
4551
4552	/* OMP directives are generally very expensive. */
4553
4554	case GIMPLE_OMP_RETURN:
4555	case GIMPLE_OMP_SECTIONS_SWITCH:
4556	case GIMPLE_OMP_ATOMIC_STORE:
4557	case GIMPLE_OMP_CONTINUE:
4558	/* ...except these, which are cheap. */
4559	return 0;
4560
4561	case GIMPLE_OMP_ATOMIC_LOAD:
4562	return weights->omp_cost;
4563
4564	case GIMPLE_OMP_FOR:
4565	return (weights->omp_cost
4566	+ estimate_num_insns_seq (stmts: gimple_omp_body (gs: stmt), weights)
4567	+ estimate_num_insns_seq (stmts: gimple_omp_for_pre_body (gs: stmt), weights));
4568
4569	case GIMPLE_OMP_PARALLEL:
4570	case GIMPLE_OMP_TASK:
4571	case GIMPLE_OMP_CRITICAL:
4572	case GIMPLE_OMP_MASTER:
4573	case GIMPLE_OMP_MASKED:
4574	case GIMPLE_OMP_SCOPE:
4575	case GIMPLE_OMP_TASKGROUP:
4576	case GIMPLE_OMP_ORDERED:
4577	case GIMPLE_OMP_SCAN:
4578	case GIMPLE_OMP_SECTION:
4579	case GIMPLE_OMP_SECTIONS:
4580	case GIMPLE_OMP_STRUCTURED_BLOCK:
4581	case GIMPLE_OMP_SINGLE:
4582	case GIMPLE_OMP_TARGET:
4583	case GIMPLE_OMP_TEAMS:
4584	return (weights->omp_cost
4585	+ estimate_num_insns_seq (stmts: gimple_omp_body (gs: stmt), weights));
4586
4587	case GIMPLE_TRANSACTION:
4588	return (weights->tm_cost
4589	+ estimate_num_insns_seq (stmts: gimple_transaction_body (
4590	transaction_stmt: as_a <gtransaction *> (p: stmt)),
4591	weights));
4592
4593	default:
4594	gcc_unreachable ();
4595	}
4596
4597	return cost;
4598	}
4599
4600	/* Estimate number of instructions that will be created by expanding
4601	function FNDECL. WEIGHTS contains weights attributed to various
4602	constructs. */
4603
4604	int
4605	estimate_num_insns_fn (tree fndecl, eni_weights *weights)
4606	{
4607	struct function *my_function = DECL_STRUCT_FUNCTION (fndecl);
4608	gimple_stmt_iterator bsi;
4609	basic_block bb;
4610	int n = 0;
4611
4612	gcc_assert (my_function && my_function->cfg);
4613	FOR_EACH_BB_FN (bb, my_function)
4614	{
4615	for (bsi = gsi_start_bb (bb); !gsi_end_p (i: bsi); gsi_next (i: &bsi))
4616	n += estimate_num_insns (stmt: gsi_stmt (i: bsi), weights);
4617	}
4618
4619	return n;
4620	}
4621
4622
4623	/* Initializes weights used by estimate_num_insns. */
4624
4625	void
4626	init_inline_once (void)
4627	{
4628	eni_size_weights.call_cost = 1;
4629	eni_size_weights.indirect_call_cost = 3;
4630	eni_size_weights.target_builtin_call_cost = 1;
4631	eni_size_weights.div_mod_cost = 1;
4632	eni_size_weights.omp_cost = 40;
4633	eni_size_weights.tm_cost = 10;
4634	eni_size_weights.time_based = false;
4635	eni_size_weights.return_cost = 1;
4636
4637	/* Estimating time for call is difficult, since we have no idea what the
4638	called function does. In the current uses of eni_time_weights,
4639	underestimating the cost does less harm than overestimating it, so
4640	we choose a rather small value here. */
4641	eni_time_weights.call_cost = 10;
4642	eni_time_weights.indirect_call_cost = 15;
4643	eni_time_weights.target_builtin_call_cost = 1;
4644	eni_time_weights.div_mod_cost = 10;
4645	eni_time_weights.omp_cost = 40;
4646	eni_time_weights.tm_cost = 40;
4647	eni_time_weights.time_based = true;
4648	eni_time_weights.return_cost = 2;
4649	}
4650
4651
4652	/* Install new lexical TREE_BLOCK underneath 'current_block'. */
4653
4654	static void
4655	prepend_lexical_block (tree current_block, tree new_block)
4656	{
4657	BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block);
4658	BLOCK_SUBBLOCKS (current_block) = new_block;
4659	BLOCK_SUPERCONTEXT (new_block) = current_block;
4660	}
4661
4662	/* Add local variables from CALLEE to CALLER. */
4663
4664	static inline void
4665	add_local_variables (struct function *callee, struct function *caller,
4666	copy_body_data *id)
4667	{
4668	tree var;
4669	unsigned ix;
4670
4671	FOR_EACH_LOCAL_DECL (callee, ix, var)
4672	if (!can_be_nonlocal (decl: var, id))
4673	{
4674	tree new_var = remap_decl (decl: var, id);
4675
4676	/* Remap debug-expressions. */
4677	if (VAR_P (new_var)
4678	&& DECL_HAS_DEBUG_EXPR_P (var)
4679	&& new_var != var)
4680	{
4681	tree tem = DECL_DEBUG_EXPR (var);
4682	bool old_regimplify = id->regimplify;
4683	id->remapping_type_depth++;
4684	walk_tree (&tem, copy_tree_body_r, id, NULL);
4685	id->remapping_type_depth--;
4686	id->regimplify = old_regimplify;
4687	SET_DECL_DEBUG_EXPR (new_var, tem);
4688	DECL_HAS_DEBUG_EXPR_P (new_var) = 1;
4689	}
4690	add_local_decl (fun: caller, d: new_var);
4691	}
4692	}
4693
4694	/* Add to BINDINGS a debug stmt resetting SRCVAR if inlining might
4695	have brought in or introduced any debug stmts for SRCVAR. */
4696
4697	static inline void
4698	reset_debug_binding (copy_body_data *id, tree srcvar, gimple_seq *bindings)
4699	{
4700	tree *remappedvarp = id->decl_map->get (k: srcvar);
4701
4702	if (!remappedvarp)
4703	return;
4704
4705	if (!VAR_P (*remappedvarp))
4706	return;
4707
4708	if (*remappedvarp == id->retvar)
4709	return;
4710
4711	tree tvar = target_for_debug_bind (*remappedvarp);
4712	if (!tvar)
4713	return;
4714
4715	gdebug *stmt = gimple_build_debug_bind (tvar, NULL_TREE,
4716	id->call_stmt);
4717	gimple_seq_add_stmt (bindings, stmt);
4718	}
4719
4720	/* For each inlined variable for which we may have debug bind stmts,
4721	add before GSI a final debug stmt resetting it, marking the end of
4722	its life, so that var-tracking knows it doesn't have to compute
4723	further locations for it. */
4724
4725	static inline void
4726	reset_debug_bindings (copy_body_data *id, gimple_stmt_iterator gsi)
4727	{
4728	tree var;
4729	unsigned ix;
4730	gimple_seq bindings = NULL;
4731
4732	if (!gimple_in_ssa_p (fun: id->src_cfun))
4733	return;
4734
4735	if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
4736	return;
4737
4738	for (var = DECL_ARGUMENTS (id->src_fn);
4739	var; var = DECL_CHAIN (var))
4740	reset_debug_binding (id, srcvar: var, bindings: &bindings);
4741
4742	FOR_EACH_LOCAL_DECL (id->src_cfun, ix, var)
4743	reset_debug_binding (id, srcvar: var, bindings: &bindings);
4744
4745	gsi_insert_seq_before_without_update (&gsi, bindings, GSI_SAME_STMT);
4746	}
4747
4748	/* If STMT is a GIMPLE_CALL, replace it with its inline expansion. */
4749
4750	static bool
4751	expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id,
4752	bitmap to_purge)
4753	{
4754	tree use_retvar;
4755	tree fn;
4756	hash_map<tree, tree> *dst;
4757	hash_map<tree, tree> *st = NULL;
4758	tree return_slot;
4759	tree modify_dest;
4760	struct cgraph_edge *cg_edge;
4761	cgraph_inline_failed_t reason;
4762	basic_block return_block;
4763	edge e;
4764	gimple_stmt_iterator gsi, stmt_gsi;
4765	bool successfully_inlined = false;
4766	bool purge_dead_abnormal_edges;
4767	gcall *call_stmt;
4768	unsigned int prop_mask, src_properties;
4769	struct function *dst_cfun;
4770	tree simduid;
4771	use_operand_p use;
4772	gimple *simtenter_stmt = NULL;
4773	vec<tree> *simtvars_save;
4774
4775	/* The gimplifier uses input_location in too many places, such as
4776	internal_get_tmp_var (). */
4777	location_t saved_location = input_location;
4778	input_location = gimple_location (g: stmt);
4779
4780	/* From here on, we're only interested in CALL_EXPRs. */
4781	call_stmt = dyn_cast <gcall *> (p: stmt);
4782	if (!call_stmt)
4783	goto egress;
4784
4785	cg_edge = id->dst_node->get_edge (call_stmt: stmt);
4786	gcc_checking_assert (cg_edge);
4787	/* First, see if we can figure out what function is being called.
4788	If we cannot, then there is no hope of inlining the function. */
4789	if (cg_edge->indirect_unknown_callee)
4790	goto egress;
4791	fn = cg_edge->callee->decl;
4792	gcc_checking_assert (fn);
4793
4794	/* If FN is a declaration of a function in a nested scope that was
4795	globally declared inline, we don't set its DECL_INITIAL.
4796	However, we can't blindly follow DECL_ABSTRACT_ORIGIN because the
4797	C++ front-end uses it for cdtors to refer to their internal
4798	declarations, that are not real functions. Fortunately those
4799	don't have trees to be saved, so we can tell by checking their
4800	gimple_body. */
4801	if (!DECL_INITIAL (fn)
4802	&& DECL_ABSTRACT_ORIGIN (fn)
4803	&& gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn)))
4804	fn = DECL_ABSTRACT_ORIGIN (fn);
4805
4806	/* Don't try to inline functions that are not well-suited to inlining. */
4807	if (cg_edge->inline_failed)
4808	{
4809	reason = cg_edge->inline_failed;
4810	/* If this call was originally indirect, we do not want to emit any
4811	inlining related warnings or sorry messages because there are no
4812	guarantees regarding those. */
4813	if (cg_edge->indirect_inlining_edge)
4814	goto egress;
4815
4816	if (lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fn))
4817	/* For extern inline functions that get redefined we always
4818	silently ignored always_inline flag. Better behavior would
4819	be to be able to keep both bodies and use extern inline body
4820	for inlining, but we can't do that because frontends overwrite
4821	the body. */
4822	&& !cg_edge->callee->redefined_extern_inline
4823	/* During early inline pass, report only when optimization is
4824	not turned on. */
4825	&& (symtab->global_info_ready
4826	|| !optimize
4827	|| cgraph_inline_failed_type (reason) == CIF_FINAL_ERROR)
4828	/* PR 20090218-1_0.c. Body can be provided by another module. */
4829	&& (reason != CIF_BODY_NOT_AVAILABLE || !flag_generate_lto))
4830	{
4831	error ("inlining failed in call to %<always_inline%> %q+F: %s", fn,
4832	cgraph_inline_failed_string (reason));
4833	if (gimple_location (g: stmt) != UNKNOWN_LOCATION)
4834	inform (gimple_location (g: stmt), "called from here");
4835	else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
4836	inform (DECL_SOURCE_LOCATION (cfun->decl),
4837	"called from this function");
4838	}
4839	else if (opt_for_fn (fn, warn_inline)
4840	&& DECL_DECLARED_INLINE_P (fn)
4841	&& !DECL_NO_INLINE_WARNING_P (fn)
4842	&& !DECL_IN_SYSTEM_HEADER (fn)
4843	&& reason != CIF_UNSPECIFIED
4844	&& !lookup_attribute (attr_name: "noinline", DECL_ATTRIBUTES (fn))
4845	/* Do not warn about not inlined recursive calls. */
4846	&& !cg_edge->recursive_p ()
4847	/* Avoid warnings during early inline pass. */
4848	&& symtab->global_info_ready)
4849	{
4850	auto_diagnostic_group d;
4851	if (warning (OPT_Winline, "inlining failed in call to %q+F: %s",
4852	fn, _(cgraph_inline_failed_string (reason))))
4853	{
4854	if (gimple_location (g: stmt) != UNKNOWN_LOCATION)
4855	inform (gimple_location (g: stmt), "called from here");
4856	else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
4857	inform (DECL_SOURCE_LOCATION (cfun->decl),
4858	"called from this function");
4859	}
4860	}
4861	goto egress;
4862	}
4863	id->src_node = cg_edge->callee;
4864
4865	/* If callee is thunk, all we need is to adjust the THIS pointer
4866	and redirect to function being thunked. */
4867	if (id->src_node->thunk)
4868	{
4869	cgraph_edge *edge;
4870	tree virtual_offset = NULL;
4871	profile_count count = cg_edge->count;
4872	tree op;
4873	gimple_stmt_iterator iter = gsi_for_stmt (stmt);
4874	thunk_info *info = thunk_info::get (node: id->src_node);
4875
4876	cgraph_edge::remove (edge: cg_edge);
4877	edge = id->src_node->callees->clone (n: id->dst_node, call_stmt,
4878	stmt_uid: gimple_uid (g: stmt),
4879	num: profile_count::one (),
4880	den: profile_count::one (),
4881	update_original: true);
4882	edge->count = count;
4883	if (info->virtual_offset_p)
4884	virtual_offset = size_int (info->virtual_value);
4885	op = create_tmp_reg_fn (cfun, TREE_TYPE (gimple_call_arg (stmt, 0)),
4886	NULL);
4887	gsi_insert_before (&iter, gimple_build_assign (op,
4888	gimple_call_arg (gs: stmt, index: 0)),
4889	GSI_NEW_STMT);
4890	gcc_assert (info->this_adjusting);
4891	op = thunk_adjust (&iter, op, 1, info->fixed_offset,
4892	virtual_offset, info->indirect_offset);
4893
4894	gimple_call_set_arg (gs: stmt, index: 0, arg: op);
4895	gimple_call_set_fndecl (gs: stmt, decl: edge->callee->decl);
4896	update_stmt (s: stmt);
4897	id->src_node->remove ();
4898	successfully_inlined = expand_call_inline (bb, stmt, id, to_purge);
4899	maybe_remove_unused_call_args (cfun, stmt);
4900	/* This used to return true even though we do fail to inline in
4901	some cases. See PR98525. */
4902	goto egress;
4903	}
4904	fn = cg_edge->callee->decl;
4905	cg_edge->callee->get_untransformed_body ();
4906
4907	if (flag_checking && cg_edge->callee->decl != id->dst_node->decl)
4908	cg_edge->callee->verify ();
4909
4910	/* We will be inlining this callee. */
4911	id->eh_lp_nr = lookup_stmt_eh_lp (stmt);
4912
4913	/* Update the callers EH personality. */
4914	if (DECL_FUNCTION_PERSONALITY (fn))
4915	DECL_FUNCTION_PERSONALITY (cg_edge->caller->decl)
4916	= DECL_FUNCTION_PERSONALITY (fn);
4917
4918	/* Split the block before the GIMPLE_CALL. */
4919	stmt_gsi = gsi_for_stmt (stmt);
4920	gsi_prev (i: &stmt_gsi);
4921	e = split_block (bb, gsi_end_p (i: stmt_gsi) ? NULL : gsi_stmt (i: stmt_gsi));
4922	bb = e->src;
4923	return_block = e->dest;
4924	remove_edge (e);
4925
4926	/* If the GIMPLE_CALL was in the last statement of BB, it may have
4927	been the source of abnormal edges. In this case, schedule
4928	the removal of dead abnormal edges. */
4929	gsi = gsi_start_bb (bb: return_block);
4930	gsi_next (i: &gsi);
4931	purge_dead_abnormal_edges = gsi_end_p (i: gsi);
4932
4933	stmt_gsi = gsi_start_bb (bb: return_block);
4934
4935	/* Build a block containing code to initialize the arguments, the
4936	actual inline expansion of the body, and a label for the return
4937	statements within the function to jump to. The type of the
4938	statement expression is the return type of the function call.
4939	??? If the call does not have an associated block then we will
4940	remap all callee blocks to NULL, effectively dropping most of
4941	its debug information. This should only happen for calls to
4942	artificial decls inserted by the compiler itself. We need to
4943	either link the inlined blocks into the caller block tree or
4944	not refer to them in any way to not break GC for locations. */
4945	if (tree block = gimple_block (g: stmt))
4946	{
4947	/* We do want to assign a not UNKNOWN_LOCATION BLOCK_SOURCE_LOCATION
4948	to make inlined_function_outer_scope_p return true on this BLOCK. */
4949	location_t loc = LOCATION_LOCUS (gimple_location (stmt));
4950	if (loc == UNKNOWN_LOCATION)
4951	loc = LOCATION_LOCUS (DECL_SOURCE_LOCATION (fn));
4952	if (loc == UNKNOWN_LOCATION)
4953	loc = BUILTINS_LOCATION;
4954	id->block = make_node (BLOCK);
4955	BLOCK_ABSTRACT_ORIGIN (id->block) = DECL_ORIGIN (fn);
4956	BLOCK_SOURCE_LOCATION (id->block) = loc;
4957	prepend_lexical_block (current_block: block, new_block: id->block);
4958	}
4959
4960	/* Local declarations will be replaced by their equivalents in this map. */
4961	st = id->decl_map;
4962	id->decl_map = new hash_map<tree, tree>;
4963	dst = id->debug_map;
4964	id->debug_map = NULL;
4965	if (flag_stack_reuse != SR_NONE)
4966	id->add_clobbers_to_eh_landing_pads = last_basic_block_for_fn (cfun);
4967
4968	/* Record the function we are about to inline. */
4969	id->src_fn = fn;
4970	id->src_cfun = DECL_STRUCT_FUNCTION (fn);
4971	id->reset_location = DECL_IGNORED_P (fn);
4972	id->call_stmt = call_stmt;
4973	cfun->cfg->full_profile &= id->src_cfun->cfg->full_profile;
4974
4975	/* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
4976	variables to be added to IFN_GOMP_SIMT_ENTER argument list. */
4977	dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
4978	simtvars_save = id->dst_simt_vars;
4979	if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
4980	&& (simduid = bb->loop_father->simduid) != NULL_TREE
4981	&& (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
4982	&& single_imm_use (var: simduid, use_p: &use, stmt: &simtenter_stmt)
4983	&& is_gimple_call (gs: simtenter_stmt)
4984	&& gimple_call_internal_p (gs: simtenter_stmt, fn: IFN_GOMP_SIMT_ENTER))
4985	vec_alloc (v&: id->dst_simt_vars, nelems: 0);
4986	else
4987	id->dst_simt_vars = NULL;
4988
4989	if (profile_status_for_fn (id->src_cfun) == PROFILE_ABSENT)
4990	profile_status_for_fn (dst_cfun) = PROFILE_ABSENT;
4991
4992	/* If the src function contains an IFN_VA_ARG, then so will the dst
4993	function after inlining. Likewise for IFN_GOMP_USE_SIMT. */
4994	prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
4995	src_properties = id->src_cfun->curr_properties & prop_mask;
4996	if (src_properties != prop_mask)
4997	dst_cfun->curr_properties &= src_properties | ~prop_mask;
4998	dst_cfun->calls_eh_return |= id->src_cfun->calls_eh_return;
4999	id->dst_node->calls_declare_variant_alt
5000	|= id->src_node->calls_declare_variant_alt;
5001
5002	gcc_assert (!id->src_cfun->after_inlining);
5003
5004	id->entry_bb = bb;
5005	if (lookup_attribute (attr_name: "cold", DECL_ATTRIBUTES (fn)))
5006	{
5007	gimple_stmt_iterator si = gsi_last_bb (bb);
5008	gsi_insert_after (&si, gimple_build_predict (predictor: PRED_COLD_FUNCTION,
5009	outcome: NOT_TAKEN),
5010	GSI_NEW_STMT);
5011	}
5012	initialize_inlined_parameters (id, stmt, fn, bb);
5013	if (debug_nonbind_markers_p && debug_inline_points && id->block
5014	&& inlined_function_outer_scope_p (block: id->block))
5015	{
5016	gimple_stmt_iterator si = gsi_last_bb (bb);
5017	gsi_insert_after (&si, gimple_build_debug_inline_entry
5018	(id->block, DECL_SOURCE_LOCATION (id->src_fn)),
5019	GSI_NEW_STMT);
5020	}
5021
5022	if (DECL_INITIAL (fn))
5023	{
5024	if (gimple_block (g: stmt))
5025	{
5026	tree *var;
5027
5028	prepend_lexical_block (current_block: id->block,
5029	new_block: remap_blocks (DECL_INITIAL (fn), id));
5030	gcc_checking_assert (BLOCK_SUBBLOCKS (id->block)
5031	&& (BLOCK_CHAIN (BLOCK_SUBBLOCKS (id->block))
5032	== NULL_TREE));
5033	/* Move vars for PARM_DECLs from DECL_INITIAL block to id->block,
5034	otherwise for DWARF DW_TAG_formal_parameter will not be children of
5035	DW_TAG_inlined_subroutine, but of a DW_TAG_lexical_block
5036	under it. The parameters can be then evaluated in the debugger,
5037	but don't show in backtraces. */
5038	for (var = &BLOCK_VARS (BLOCK_SUBBLOCKS (id->block)); *var; )
5039	if (TREE_CODE (DECL_ORIGIN (*var)) == PARM_DECL)
5040	{
5041	tree v = *var;
5042	*var = TREE_CHAIN (v);
5043	TREE_CHAIN (v) = BLOCK_VARS (id->block);
5044	BLOCK_VARS (id->block) = v;
5045	}
5046	else
5047	var = &TREE_CHAIN (*var);
5048	}
5049	else
5050	remap_blocks_to_null (DECL_INITIAL (fn), id);
5051	}
5052
5053	/* Return statements in the function body will be replaced by jumps
5054	to the RET_LABEL. */
5055	gcc_assert (DECL_INITIAL (fn));
5056	gcc_assert (TREE_CODE (DECL_INITIAL (fn)) == BLOCK);
5057
5058	/* Find the LHS to which the result of this call is assigned. */
5059	return_slot = NULL;
5060	if (gimple_call_lhs (gs: stmt))
5061	{
5062	modify_dest = gimple_call_lhs (gs: stmt);
5063
5064	/* The function which we are inlining might not return a value,
5065	in which case we should issue a warning that the function
5066	does not return a value. In that case the optimizers will
5067	see that the variable to which the value is assigned was not
5068	initialized. We do not want to issue a warning about that
5069	uninitialized variable. */
5070	if (DECL_P (modify_dest))
5071	suppress_warning (modify_dest, OPT_Wuninitialized);
5072
5073	if (gimple_call_return_slot_opt_p (s: call_stmt))
5074	{
5075	return_slot = modify_dest;
5076	modify_dest = NULL;
5077	}
5078	}
5079	else
5080	modify_dest = NULL;
5081
5082	/* If we are inlining a call to the C++ operator new, we don't want
5083	to use type based alias analysis on the return value. Otherwise
5084	we may get confused if the compiler sees that the inlined new
5085	function returns a pointer which was just deleted. See bug
5086	33407. */
5087	if (DECL_IS_OPERATOR_NEW_P (fn))
5088	{
5089	return_slot = NULL;
5090	modify_dest = NULL;
5091	}
5092
5093	/* Declare the return variable for the function. */
5094	use_retvar = declare_return_variable (id, return_slot, modify_dest, entry_bb: bb);
5095
5096	/* Add local vars in this inlined callee to caller. */
5097	add_local_variables (callee: id->src_cfun, cfun, id);
5098
5099	if (dump_enabled_p ())
5100	{
5101	char buf[128];
5102	snprintf (s: buf, maxlen: sizeof(buf), format: "%4.2f",
5103	cg_edge->sreal_frequency ().to_double ());
5104	dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
5105	call_stmt,
5106	"Inlining %C to %C with frequency %s\n",
5107	id->src_node, id->dst_node, buf);
5108	if (dump_file && (dump_flags & TDF_DETAILS))
5109	{
5110	id->src_node->dump (f: dump_file);
5111	id->dst_node->dump (f: dump_file);
5112	}
5113	}
5114
5115	/* This is it. Duplicate the callee body. Assume callee is
5116	pre-gimplified. Note that we must not alter the caller
5117	function in any way before this point, as this CALL_EXPR may be
5118	a self-referential call; if we're calling ourselves, we need to
5119	duplicate our body before altering anything. */
5120	copy_body (id, entry_block_map: bb, exit_block_map: return_block, NULL);
5121
5122	reset_debug_bindings (id, gsi: stmt_gsi);
5123
5124	if (flag_stack_reuse != SR_NONE)
5125	for (tree p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
5126	if (!TREE_THIS_VOLATILE (p))
5127	{
5128	/* The value associated with P is a local temporary only if
5129	there is no value associated with P in the debug map. */
5130	tree *varp = id->decl_map->get (k: p);
5131	if (varp
5132	&& VAR_P (*varp)
5133	&& !is_gimple_reg (*varp)
5134	&& !(id->debug_map && id->debug_map->get (k: p)))
5135	{
5136	tree clobber = build_clobber (TREE_TYPE (*varp), CLOBBER_EOL);
5137	gimple *clobber_stmt;
5138	clobber_stmt = gimple_build_assign (*varp, clobber);
5139	gimple_set_location (g: clobber_stmt, location: gimple_location (g: stmt));
5140	gsi_insert_before (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
5141	}
5142	}
5143
5144	/* Reset the escaped solution. */
5145	if (cfun->gimple_df)
5146	pt_solution_reset (&cfun->gimple_df->escaped);
5147
5148	/* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments. */
5149	if (id->dst_simt_vars && id->dst_simt_vars->length () > 0)
5150	{
5151	size_t nargs = gimple_call_num_args (gs: simtenter_stmt);
5152	vec<tree> *vars = id->dst_simt_vars;
5153	auto_vec<tree> newargs (nargs + vars->length ());
5154	for (size_t i = 0; i < nargs; i++)
5155	newargs.quick_push (obj: gimple_call_arg (gs: simtenter_stmt, index: i));
5156	for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
5157	{
5158	tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
5159	newargs.quick_push (obj: build1 (ADDR_EXPR, ptrtype, *pvar));
5160	}
5161	gcall *g = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
5162	gimple_call_set_lhs (gs: g, lhs: gimple_call_lhs (gs: simtenter_stmt));
5163	gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
5164	gsi_replace (&gsi, g, false);
5165	}
5166	vec_free (v&: id->dst_simt_vars);
5167	id->dst_simt_vars = simtvars_save;
5168
5169	/* Clean up. */
5170	if (id->debug_map)
5171	{
5172	delete id->debug_map;
5173	id->debug_map = dst;
5174	}
5175	delete id->decl_map;
5176	id->decl_map = st;
5177
5178	/* Unlink the calls virtual operands before replacing it. */
5179	unlink_stmt_vdef (stmt);
5180	if (gimple_vdef (g: stmt)
5181	&& TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
5182	release_ssa_name (name: gimple_vdef (g: stmt));
5183
5184	/* If the inlined function returns a result that we care about,
5185	substitute the GIMPLE_CALL with an assignment of the return
5186	variable to the LHS of the call. That is, if STMT was
5187	'a = foo (...)', substitute the call with 'a = USE_RETVAR'. */
5188	if (use_retvar && gimple_call_lhs (gs: stmt))
5189	{
5190	gimple *old_stmt = stmt;
5191	stmt = gimple_build_assign (gimple_call_lhs (gs: stmt), use_retvar);
5192	gimple_set_location (g: stmt, location: gimple_location (g: old_stmt));
5193	gsi_replace (&stmt_gsi, stmt, false);
5194	maybe_clean_or_replace_eh_stmt (old_stmt, stmt);
5195	/* Append a clobber for id->retvar if easily possible. */
5196	if (flag_stack_reuse != SR_NONE
5197	&& id->retvar
5198	&& VAR_P (id->retvar)
5199	&& id->retvar != return_slot
5200	&& id->retvar != modify_dest
5201	&& !TREE_THIS_VOLATILE (id->retvar)
5202	&& !is_gimple_reg (id->retvar)
5203	&& !stmt_ends_bb_p (stmt))
5204	{
5205	tree clobber = build_clobber (TREE_TYPE (id->retvar), CLOBBER_EOL);
5206	gimple *clobber_stmt;
5207	clobber_stmt = gimple_build_assign (id->retvar, clobber);
5208	gimple_set_location (g: clobber_stmt, location: gimple_location (g: old_stmt));
5209	gsi_insert_after (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
5210	}
5211	}
5212	else
5213	{
5214	/* Handle the case of inlining a function with no return
5215	statement, which causes the return value to become undefined. */
5216	if (gimple_call_lhs (gs: stmt)
5217	&& TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
5218	{
5219	tree name = gimple_call_lhs (gs: stmt);
5220	tree var = SSA_NAME_VAR (name);
5221	tree def = var ? ssa_default_def (cfun, var) : NULL;
5222
5223	if (def)
5224	{
5225	/* If the variable is used undefined, make this name
5226	undefined via a move. */
5227	stmt = gimple_build_assign (gimple_call_lhs (gs: stmt), def);
5228	gsi_replace (&stmt_gsi, stmt, true);
5229	}
5230	else
5231	{
5232	if (!var)
5233	{
5234	var = create_tmp_reg_fn (cfun, TREE_TYPE (name), NULL);
5235	SET_SSA_NAME_VAR_OR_IDENTIFIER (name, var);
5236	}
5237	/* Otherwise make this variable undefined. */
5238	gsi_remove (&stmt_gsi, true);
5239	set_ssa_default_def (cfun, var, name);
5240	SSA_NAME_DEF_STMT (name) = gimple_build_nop ();
5241	}
5242	}
5243	/* Replace with a clobber for id->retvar. */
5244	else if (flag_stack_reuse != SR_NONE
5245	&& id->retvar
5246	&& VAR_P (id->retvar)
5247	&& id->retvar != return_slot
5248	&& id->retvar != modify_dest
5249	&& !TREE_THIS_VOLATILE (id->retvar)
5250	&& !is_gimple_reg (id->retvar))
5251	{
5252	tree clobber = build_clobber (TREE_TYPE (id->retvar));
5253	gimple *clobber_stmt;
5254	clobber_stmt = gimple_build_assign (id->retvar, clobber);
5255	gimple_set_location (g: clobber_stmt, location: gimple_location (g: stmt));
5256	gsi_replace (&stmt_gsi, clobber_stmt, false);
5257	maybe_clean_or_replace_eh_stmt (stmt, clobber_stmt);
5258	}
5259	else
5260	gsi_remove (&stmt_gsi, true);
5261	}
5262
5263	if (purge_dead_abnormal_edges)
5264	bitmap_set_bit (to_purge, return_block->index);
5265
5266	/* If the value of the new expression is ignored, that's OK. We
5267	don't warn about this for CALL_EXPRs, so we shouldn't warn about
5268	the equivalent inlined version either. */
5269	if (is_gimple_assign (gs: stmt))
5270	{
5271	gcc_assert (gimple_assign_single_p (stmt)
5272	|| CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)));
5273	TREE_USED (gimple_assign_rhs1 (stmt)) = 1;
5274	}
5275
5276	id->add_clobbers_to_eh_landing_pads = 0;
5277
5278	/* Output the inlining info for this abstract function, since it has been
5279	inlined. If we don't do this now, we can lose the information about the
5280	variables in the function when the blocks get blown away as soon as we
5281	remove the cgraph node. */
5282	if (gimple_block (g: stmt))
5283	(*debug_hooks->outlining_inline_function) (fn);
5284
5285	/* Update callgraph if needed. */
5286	cg_edge->callee->remove ();
5287
5288	id->block = NULL_TREE;
5289	id->retvar = NULL_TREE;
5290	successfully_inlined = true;
5291
5292	egress:
5293	input_location = saved_location;
5294	return successfully_inlined;
5295	}
5296
5297	/* Expand call statements reachable from STMT_P.
5298	We can only have CALL_EXPRs as the "toplevel" tree code or nested
5299	in a MODIFY_EXPR. */
5300
5301	static bool
5302	gimple_expand_calls_inline (basic_block bb, copy_body_data *id,
5303	bitmap to_purge)
5304	{
5305	gimple_stmt_iterator gsi;
5306	bool inlined = false;
5307
5308	for (gsi = gsi_last_bb (bb); !gsi_end_p (i: gsi);)
5309	{
5310	gimple *stmt = gsi_stmt (i: gsi);
5311	gsi_prev (i: &gsi);
5312
5313	if (is_gimple_call (gs: stmt)
5314	&& !gimple_call_internal_p (gs: stmt))
5315	inlined |= expand_call_inline (bb, stmt, id, to_purge);
5316	}
5317
5318	return inlined;
5319	}
5320
5321
5322	/* Walk all basic blocks created after FIRST and try to fold every statement
5323	in the STATEMENTS pointer set. */
5324
5325	static void
5326	fold_marked_statements (int first, hash_set<gimple *> *statements)
5327	{
5328	auto_bitmap to_purge;
5329
5330	auto_vec<edge, 20> stack (n_basic_blocks_for_fn (cfun) + 2);
5331	auto_sbitmap visited (last_basic_block_for_fn (cfun));
5332	bitmap_clear (visited);
5333
5334	stack.quick_push (obj: single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
5335	while (!stack.is_empty ())
5336	{
5337	/* Look at the edge on the top of the stack. */
5338	edge e = stack.pop ();
5339	basic_block dest = e->dest;
5340
5341	if (dest == EXIT_BLOCK_PTR_FOR_FN (cfun)
5342	|| bitmap_bit_p (map: visited, bitno: dest->index))
5343	continue;
5344
5345	bitmap_set_bit (map: visited, bitno: dest->index);
5346
5347	if (dest->index >= first)
5348	for (gimple_stmt_iterator gsi = gsi_start_bb (bb: dest);
5349	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
5350	{
5351	if (!statements->contains (k: gsi_stmt (i: gsi)))
5352	continue;
5353
5354	gimple *old_stmt = gsi_stmt (i: gsi);
5355	tree old_decl = (is_gimple_call (gs: old_stmt)
5356	? gimple_call_fndecl (gs: old_stmt) : 0);
5357	if (old_decl && fndecl_built_in_p (node: old_decl))
5358	{
5359	/* Folding builtins can create multiple instructions,
5360	we need to look at all of them. */
5361	gimple_stmt_iterator i2 = gsi;
5362	gsi_prev (i: &i2);
5363	if (fold_stmt (&gsi))
5364	{
5365	gimple *new_stmt;
5366	/* If a builtin at the end of a bb folded into nothing,
5367	the following loop won't work. */
5368	if (gsi_end_p (i: gsi))
5369	{
5370	cgraph_update_edges_for_call_stmt (old_stmt,
5371	old_decl, NULL);
5372	break;
5373	}
5374	if (gsi_end_p (i: i2))
5375	i2 = gsi_start_bb (bb: dest);
5376	else
5377	gsi_next (i: &i2);
5378	while (1)
5379	{
5380	new_stmt = gsi_stmt (i: i2);
5381	update_stmt (s: new_stmt);
5382	cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
5383	new_stmt);
5384
5385	if (new_stmt == gsi_stmt (i: gsi))
5386	{
5387	/* It is okay to check only for the very last
5388	of these statements. If it is a throwing
5389	statement nothing will change. If it isn't
5390	this can remove EH edges. If that weren't
5391	correct then because some intermediate stmts
5392	throw, but not the last one. That would mean
5393	we'd have to split the block, which we can't
5394	here and we'd loose anyway. And as builtins
5395	probably never throw, this all
5396	is mood anyway. */
5397	if (maybe_clean_or_replace_eh_stmt (old_stmt,
5398	new_stmt))
5399	bitmap_set_bit (to_purge, dest->index);
5400	break;
5401	}
5402	gsi_next (i: &i2);
5403	}
5404	}
5405	}
5406	else if (fold_stmt (&gsi))
5407	{
5408	/* Re-read the statement from GSI as fold_stmt() may
5409	have changed it. */
5410	gimple *new_stmt = gsi_stmt (i: gsi);
5411	update_stmt (s: new_stmt);
5412
5413	if (is_gimple_call (gs: old_stmt)
5414	|| is_gimple_call (gs: new_stmt))
5415	cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
5416	new_stmt);
5417
5418	if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt))
5419	bitmap_set_bit (to_purge, dest->index);
5420	}
5421	}
5422
5423	if (EDGE_COUNT (dest->succs) > 0)
5424	{
5425	/* Avoid warnings emitted from folding statements that
5426	became unreachable because of inlined function parameter
5427	propagation. */
5428	e = find_taken_edge (dest, NULL_TREE);
5429	if (e)
5430	stack.quick_push (obj: e);
5431	else
5432	{
5433	edge_iterator ei;
5434	FOR_EACH_EDGE (e, ei, dest->succs)
5435	stack.safe_push (obj: e);
5436	}
5437	}
5438	}
5439
5440	gimple_purge_all_dead_eh_edges (to_purge);
5441	}
5442
5443	/* Expand calls to inline functions in the body of FN. */
5444
5445	unsigned int
5446	optimize_inline_calls (tree fn)
5447	{
5448	copy_body_data id;
5449	basic_block bb;
5450	int last = n_basic_blocks_for_fn (cfun);
5451	bool inlined_p = false;
5452
5453	/* Clear out ID. */
5454	memset (s: &id, c: 0, n: sizeof (id));
5455
5456	id.src_node = id.dst_node = cgraph_node::get (decl: fn);
5457	gcc_assert (id.dst_node->definition);
5458	id.dst_fn = fn;
5459	/* Or any functions that aren't finished yet. */
5460	if (current_function_decl)
5461	id.dst_fn = current_function_decl;
5462
5463	id.copy_decl = copy_decl_maybe_to_var;
5464	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5465	id.transform_new_cfg = false;
5466	id.transform_return_to_modify = true;
5467	id.transform_parameter = true;
5468	id.statements_to_fold = new hash_set<gimple *>;
5469
5470	push_gimplify_context ();
5471
5472	/* We make no attempts to keep dominance info up-to-date. */
5473	free_dominance_info (CDI_DOMINATORS);
5474	free_dominance_info (CDI_POST_DOMINATORS);
5475
5476	/* Register specific gimple functions. */
5477	gimple_register_cfg_hooks ();
5478
5479	/* Reach the trees by walking over the CFG, and note the
5480	enclosing basic-blocks in the call edges. */
5481	/* We walk the blocks going forward, because inlined function bodies
5482	will split id->current_basic_block, and the new blocks will
5483	follow it; we'll trudge through them, processing their CALL_EXPRs
5484	along the way. */
5485	auto_bitmap to_purge;
5486	FOR_EACH_BB_FN (bb, cfun)
5487	inlined_p |= gimple_expand_calls_inline (bb, id: &id, to_purge);
5488
5489	pop_gimplify_context (NULL);
5490
5491	if (flag_checking)
5492	{
5493	struct cgraph_edge *e;
5494
5495	id.dst_node->verify ();
5496
5497	/* Double check that we inlined everything we are supposed to inline. */
5498	for (e = id.dst_node->callees; e; e = e->next_callee)
5499	gcc_assert (e->inline_failed);
5500	}
5501
5502	/* If we didn't inline into the function there is nothing to do. */
5503	if (!inlined_p)
5504	{
5505	delete id.statements_to_fold;
5506	return 0;
5507	}
5508
5509	/* Fold queued statements. */
5510	update_max_bb_count ();
5511	fold_marked_statements (first: last, statements: id.statements_to_fold);
5512	delete id.statements_to_fold;
5513
5514	/* Finally purge EH and abnormal edges from the call stmts we inlined.
5515	We need to do this after fold_marked_statements since that may walk
5516	the SSA use-def chain. */
5517	unsigned i;
5518	bitmap_iterator bi;
5519	EXECUTE_IF_SET_IN_BITMAP (to_purge, 0, i, bi)
5520	{
5521	basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
5522	if (bb)
5523	{
5524	gimple_purge_dead_eh_edges (bb);
5525	gimple_purge_dead_abnormal_call_edges (bb);
5526	}
5527	}
5528
5529	gcc_assert (!id.debug_stmts.exists ());
5530
5531	/* Renumber the lexical scoping (non-code) blocks consecutively. */
5532	number_blocks (fn);
5533
5534	delete_unreachable_blocks_update_callgraph (dst_node: id.dst_node, update_clones: false);
5535	id.dst_node->calls_comdat_local = id.dst_node->check_calls_comdat_local_p ();
5536
5537	if (flag_checking)
5538	id.dst_node->verify ();
5539
5540	/* It would be nice to check SSA/CFG/statement consistency here, but it is
5541	not possible yet - the IPA passes might make various functions to not
5542	throw and they don't care to proactively update local EH info. This is
5543	done later in fixup_cfg pass that also execute the verification. */
5544	return (TODO_update_ssa
5545	| TODO_cleanup_cfg
5546	| (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0)
5547	| (gimple_in_ssa_p (cfun) ? TODO_update_address_taken : 0));
5548	}
5549
5550	/* Passed to walk_tree. Copies the node pointed to, if appropriate. */
5551
5552	tree
5553	copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
5554	{
5555	enum tree_code code = TREE_CODE (*tp);
5556	enum tree_code_class cl = TREE_CODE_CLASS (code);
5557
5558	/* We make copies of most nodes. */
5559	if (IS_EXPR_CODE_CLASS (cl)
5560	|| code == TREE_LIST
5561	|| code == TREE_VEC
5562	|| code == TYPE_DECL
5563	|| code == OMP_CLAUSE)
5564	{
5565	/* Because the chain gets clobbered when we make a copy, we save it
5566	here. */
5567	tree chain = NULL_TREE, new_tree;
5568
5569	if (CODE_CONTAINS_STRUCT (code, TS_COMMON))
5570	chain = TREE_CHAIN (*tp);
5571
5572	/* Copy the node. */
5573	new_tree = copy_node (*tp);
5574
5575	*tp = new_tree;
5576
5577	/* Now, restore the chain, if appropriate. That will cause
5578	walk_tree to walk into the chain as well. */
5579	if (code == PARM_DECL
5580	|| code == TREE_LIST
5581	|| code == OMP_CLAUSE)
5582	TREE_CHAIN (*tp) = chain;
5583
5584	/* For now, we don't update BLOCKs when we make copies. So, we
5585	have to nullify all BIND_EXPRs. */
5586	if (TREE_CODE (*tp) == BIND_EXPR)
5587	BIND_EXPR_BLOCK (*tp) = NULL_TREE;
5588	}
5589	else if (code == CONSTRUCTOR)
5590	{
5591	/* CONSTRUCTOR nodes need special handling because
5592	we need to duplicate the vector of elements. */
5593	tree new_tree;
5594
5595	new_tree = copy_node (*tp);
5596	CONSTRUCTOR_ELTS (new_tree) = vec_safe_copy (CONSTRUCTOR_ELTS (*tp));
5597	*tp = new_tree;
5598	}
5599	else if (code == STATEMENT_LIST)
5600	/* We used to just abort on STATEMENT_LIST, but we can run into them
5601	with statement-expressions (c++/40975). */
5602	copy_statement_list (tp);
5603	else if (TREE_CODE_CLASS (code) == tcc_type)
5604	*walk_subtrees = 0;
5605	else if (TREE_CODE_CLASS (code) == tcc_declaration)
5606	*walk_subtrees = 0;
5607	else if (TREE_CODE_CLASS (code) == tcc_constant)
5608	*walk_subtrees = 0;
5609	return NULL_TREE;
5610	}
5611
5612	/* The SAVE_EXPR pointed to by TP is being copied. If ST contains
5613	information indicating to what new SAVE_EXPR this one should be mapped,
5614	use that one. Otherwise, create a new node and enter it in ST. FN is
5615	the function into which the copy will be placed. */
5616
5617	static void
5618	remap_save_expr (tree *tp, hash_map<tree, tree> *st, int *walk_subtrees)
5619	{
5620	tree *n;
5621	tree t;
5622
5623	/* See if we already encountered this SAVE_EXPR. */
5624	n = st->get (k: *tp);
5625
5626	/* If we didn't already remap this SAVE_EXPR, do so now. */
5627	if (!n)
5628	{
5629	t = copy_node (*tp);
5630
5631	/* Remember this SAVE_EXPR. */
5632	st->put (k: *tp, v: t);
5633	/* Make sure we don't remap an already-remapped SAVE_EXPR. */
5634	st->put (k: t, v: t);
5635	}
5636	else
5637	{
5638	/* We've already walked into this SAVE_EXPR; don't do it again. */
5639	*walk_subtrees = 0;
5640	t = *n;
5641	}
5642
5643	/* Replace this SAVE_EXPR with the copy. */
5644	*tp = t;
5645	}
5646
5647	/* Called via walk_gimple_seq. If *GSIP points to a GIMPLE_LABEL for a local
5648	label, copies the declaration and enters it in the splay_tree in DATA (which
5649	is really a 'copy_body_data *'. */
5650
5651	static tree
5652	mark_local_labels_stmt (gimple_stmt_iterator *gsip,
5653	bool *handled_ops_p ATTRIBUTE_UNUSED,
5654	struct walk_stmt_info *wi)
5655	{
5656	copy_body_data *id = (copy_body_data *) wi->info;
5657	glabel *stmt = dyn_cast <glabel *> (p: gsi_stmt (i: *gsip));
5658
5659	if (stmt)
5660	{
5661	tree decl = gimple_label_label (gs: stmt);
5662
5663	/* Copy the decl and remember the copy. */
5664	insert_decl_map (id, key: decl, value: id->copy_decl (decl, id));
5665	}
5666
5667	return NULL_TREE;
5668	}
5669
5670	static gimple_seq duplicate_remap_omp_clause_seq (gimple_seq seq,
5671	struct walk_stmt_info *wi);
5672
5673	/* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5674	Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5675	remaps all local declarations to appropriate replacements in gimple
5676	operands. */
5677
5678	static tree
5679	replace_locals_op (tree *tp, int *walk_subtrees, void *data)
5680	{
5681	struct walk_stmt_info *wi = (struct walk_stmt_info*) data;
5682	copy_body_data *id = (copy_body_data *) wi->info;
5683	hash_map<tree, tree> *st = id->decl_map;
5684	tree *n;
5685	tree expr = *tp;
5686
5687	/* For recursive invocations this is no longer the LHS itself. */
5688	bool is_lhs = wi->is_lhs;
5689	wi->is_lhs = false;
5690
5691	if (TREE_CODE (expr) == SSA_NAME)
5692	{
5693	*tp = remap_ssa_name (name: *tp, id);
5694	*walk_subtrees = 0;
5695	if (is_lhs)
5696	SSA_NAME_DEF_STMT (*tp) = gsi_stmt (i: wi->gsi);
5697	}
5698	/* Only a local declaration (variable or label). */
5699	else if ((VAR_P (expr) && !TREE_STATIC (expr))
5700	|| TREE_CODE (expr) == LABEL_DECL)
5701	{
5702	/* Lookup the declaration. */
5703	n = st->get (k: expr);
5704
5705	/* If it's there, remap it. */
5706	if (n)
5707	*tp = *n;
5708	*walk_subtrees = 0;
5709	}
5710	else if (TREE_CODE (expr) == STATEMENT_LIST
5711	|| TREE_CODE (expr) == BIND_EXPR
5712	|| TREE_CODE (expr) == SAVE_EXPR)
5713	gcc_unreachable ();
5714	else if (TREE_CODE (expr) == TARGET_EXPR)
5715	{
5716	/* Don't mess with a TARGET_EXPR that hasn't been expanded.
5717	It's OK for this to happen if it was part of a subtree that
5718	isn't immediately expanded, such as operand 2 of another
5719	TARGET_EXPR. */
5720	if (!TREE_OPERAND (expr, 1))
5721	{
5722	TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
5723	TREE_OPERAND (expr, 3) = NULL_TREE;
5724	}
5725	}
5726	else if (TREE_CODE (expr) == OMP_CLAUSE)
5727	{
5728	/* Before the omplower pass completes, some OMP clauses can contain
5729	sequences that are neither copied by gimple_seq_copy nor walked by
5730	walk_gimple_seq. To make copy_gimple_seq_and_replace_locals work even
5731	in those situations, we have to copy and process them explicitely. */
5732
5733	if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LASTPRIVATE)
5734	{
5735	gimple_seq seq = OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr);
5736	seq = duplicate_remap_omp_clause_seq (seq, wi);
5737	OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr) = seq;
5738	}
5739	else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LINEAR)
5740	{
5741	gimple_seq seq = OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr);
5742	seq = duplicate_remap_omp_clause_seq (seq, wi);
5743	OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr) = seq;
5744	}
5745	else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_REDUCTION)
5746	{
5747	gimple_seq seq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr);
5748	seq = duplicate_remap_omp_clause_seq (seq, wi);
5749	OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr) = seq;
5750	seq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr);
5751	seq = duplicate_remap_omp_clause_seq (seq, wi);
5752	OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr) = seq;
5753	}
5754	}
5755
5756	/* Keep iterating. */
5757	return NULL_TREE;
5758	}
5759
5760
5761	/* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5762	Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5763	remaps all local declarations to appropriate replacements in gimple
5764	statements. */
5765
5766	static tree
5767	replace_locals_stmt (gimple_stmt_iterator *gsip,
5768	bool *handled_ops_p ATTRIBUTE_UNUSED,
5769	struct walk_stmt_info *wi)
5770	{
5771	copy_body_data *id = (copy_body_data *) wi->info;
5772	gimple *gs = gsi_stmt (i: *gsip);
5773
5774	if (gbind *stmt = dyn_cast <gbind *> (p: gs))
5775	{
5776	tree block = gimple_bind_block (bind_stmt: stmt);
5777
5778	if (block)
5779	{
5780	remap_block (block: &block, id);
5781	gimple_bind_set_block (bind_stmt: stmt, block);
5782	}
5783
5784	/* This will remap a lot of the same decls again, but this should be
5785	harmless. */
5786	if (gimple_bind_vars (bind_stmt: stmt))
5787	{
5788	tree old_var, decls = gimple_bind_vars (bind_stmt: stmt);
5789
5790	for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
5791	if (!can_be_nonlocal (decl: old_var, id)
5792	&& ! variably_modified_type_p (TREE_TYPE (old_var), id->src_fn))
5793	remap_decl (decl: old_var, id);
5794
5795	gcc_checking_assert (!id->prevent_decl_creation_for_types);
5796	id->prevent_decl_creation_for_types = true;
5797	gimple_bind_set_vars (bind_stmt: stmt, vars: remap_decls (decls, NULL, id));
5798	id->prevent_decl_creation_for_types = false;
5799	}
5800	}
5801
5802	/* Keep iterating. */
5803	return NULL_TREE;
5804	}
5805
5806	/* Create a copy of SEQ and remap all decls in it. */
5807
5808	static gimple_seq
5809	duplicate_remap_omp_clause_seq (gimple_seq seq, struct walk_stmt_info *wi)
5810	{
5811	if (!seq)
5812	return NULL;
5813
5814	/* If there are any labels in OMP sequences, they can be only referred to in
5815	the sequence itself and therefore we can do both here. */
5816	walk_gimple_seq (seq, mark_local_labels_stmt, NULL, wi);
5817	gimple_seq copy = gimple_seq_copy (seq);
5818	walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, wi);
5819	return copy;
5820	}
5821
5822	/* Copies everything in SEQ and replaces variables and labels local to
5823	current_function_decl. */
5824
5825	gimple_seq
5826	copy_gimple_seq_and_replace_locals (gimple_seq seq)
5827	{
5828	copy_body_data id;
5829	struct walk_stmt_info wi;
5830	gimple_seq copy;
5831
5832	/* There's nothing to do for NULL_TREE. */
5833	if (seq == NULL)
5834	return seq;
5835
5836	/* Set up ID. */
5837	memset (s: &id, c: 0, n: sizeof (id));
5838	id.src_fn = current_function_decl;
5839	id.dst_fn = current_function_decl;
5840	id.src_cfun = cfun;
5841	id.decl_map = new hash_map<tree, tree>;
5842	id.debug_map = NULL;
5843
5844	id.copy_decl = copy_decl_no_change;
5845	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5846	id.transform_new_cfg = false;
5847	id.transform_return_to_modify = false;
5848	id.transform_parameter = false;
5849
5850	/* Walk the tree once to find local labels. */
5851	memset (s: &wi, c: 0, n: sizeof (wi));
5852	hash_set<tree> visited;
5853	wi.info = &id;
5854	wi.pset = &visited;
5855	walk_gimple_seq (seq, mark_local_labels_stmt, NULL, &wi);
5856
5857	copy = gimple_seq_copy (seq);
5858
5859	/* Walk the copy, remapping decls. */
5860	memset (s: &wi, c: 0, n: sizeof (wi));
5861	wi.info = &id;
5862	walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, &wi);
5863
5864	/* Clean up. */
5865	delete id.decl_map;
5866	if (id.debug_map)
5867	delete id.debug_map;
5868	if (id.dependence_map)
5869	{
5870	delete id.dependence_map;
5871	id.dependence_map = NULL;
5872	}
5873
5874	return copy;
5875	}
5876
5877
5878	/* Allow someone to determine if SEARCH is a child of TOP from gdb. */
5879
5880	static tree
5881	debug_find_tree_1 (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED, void *data)
5882	{
5883	if (*tp == data)
5884	return (tree) data;
5885	else
5886	return NULL;
5887	}
5888
5889	DEBUG_FUNCTION bool
5890	debug_find_tree (tree top, tree search)
5891	{
5892	return walk_tree_without_duplicates (&top, debug_find_tree_1, search) != 0;
5893	}
5894
5895
5896	/* Declare the variables created by the inliner. Add all the variables in
5897	VARS to BIND_EXPR. */
5898
5899	static void
5900	declare_inline_vars (tree block, tree vars)
5901	{
5902	tree t;
5903	for (t = vars; t; t = DECL_CHAIN (t))
5904	{
5905	DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
5906	gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t));
5907	add_local_decl (cfun, d: t);
5908	}
5909
5910	if (block)
5911	BLOCK_VARS (block) = chainon (BLOCK_VARS (block), vars);
5912	}
5913
5914	/* Copy NODE (which must be a DECL). The DECL originally was in the FROM_FN,
5915	but now it will be in the TO_FN. PARM_TO_VAR means enable PARM_DECL to
5916	VAR_DECL translation. */
5917
5918	tree
5919	copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy)
5920	{
5921	/* Don't generate debug information for the copy if we wouldn't have
5922	generated it for the copy either. */
5923	DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (decl);
5924	DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl);
5925
5926	/* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what
5927	declaration inspired this copy. */
5928	DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl);
5929
5930	/* The new variable/label has no RTL, yet. */
5931	if (HAS_RTL_P (copy)
5932	&& !TREE_STATIC (copy) && !DECL_EXTERNAL (copy))
5933	SET_DECL_RTL (copy, 0);
5934	/* For vector typed decls make sure to update DECL_MODE according
5935	to the new function context. */
5936	if (VECTOR_TYPE_P (TREE_TYPE (copy)))
5937	SET_DECL_MODE (copy, TYPE_MODE (TREE_TYPE (copy)));
5938
5939	/* These args would always appear unused, if not for this. */
5940	TREE_USED (copy) = 1;
5941
5942	/* Set the context for the new declaration. */
5943	if (!DECL_CONTEXT (decl))
5944	/* Globals stay global. */
5945	;
5946	else if (DECL_CONTEXT (decl) != id->src_fn)
5947	/* Things that weren't in the scope of the function we're inlining
5948	from aren't in the scope we're inlining to, either. */
5949	;
5950	else if (TREE_STATIC (decl))
5951	/* Function-scoped static variables should stay in the original
5952	function. */
5953	;
5954	else
5955	{
5956	/* Ordinary automatic local variables are now in the scope of the
5957	new function. */
5958	DECL_CONTEXT (copy) = id->dst_fn;
5959	if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
5960	{
5961	if (!lookup_attribute (attr_name: "omp simt private", DECL_ATTRIBUTES (copy)))
5962	DECL_ATTRIBUTES (copy)
5963	= tree_cons (get_identifier ("omp simt private"), NULL,
5964	DECL_ATTRIBUTES (copy));
5965	id->dst_simt_vars->safe_push (obj: copy);
5966	}
5967	}
5968
5969	return copy;
5970	}
5971
5972	/* Create a new VAR_DECL that is indentical in all respect to DECL except that
5973	DECL can be either a VAR_DECL, a PARM_DECL or RESULT_DECL. The original
5974	DECL must come from ID->src_fn and the copy will be part of ID->dst_fn. */
5975
5976	tree
5977	copy_decl_to_var (tree decl, copy_body_data *id)
5978	{
5979	tree copy, type;
5980
5981	gcc_assert (TREE_CODE (decl) == PARM_DECL
5982	|| TREE_CODE (decl) == RESULT_DECL);
5983
5984	type = TREE_TYPE (decl);
5985
5986	copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
5987	VAR_DECL, DECL_NAME (decl), type);
5988	if (DECL_PT_UID_SET_P (decl))
5989	SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
5990	TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
5991	TREE_READONLY (copy) = TREE_READONLY (decl);
5992	TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
5993	DECL_NOT_GIMPLE_REG_P (copy) = DECL_NOT_GIMPLE_REG_P (decl);
5994	DECL_BY_REFERENCE (copy) = DECL_BY_REFERENCE (decl);
5995
5996	return copy_decl_for_dup_finish (id, decl, copy);
5997	}
5998
5999	/* Like copy_decl_to_var, but create a return slot object instead of a
6000	pointer variable for return by invisible reference. */
6001
6002	static tree
6003	copy_result_decl_to_var (tree decl, copy_body_data *id)
6004	{
6005	tree copy, type;
6006
6007	gcc_assert (TREE_CODE (decl) == PARM_DECL
6008	|| TREE_CODE (decl) == RESULT_DECL);
6009
6010	type = TREE_TYPE (decl);
6011	if (DECL_BY_REFERENCE (decl))
6012	type = TREE_TYPE (type);
6013
6014	copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
6015	VAR_DECL, DECL_NAME (decl), type);
6016	if (DECL_PT_UID_SET_P (decl))
6017	SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
6018	TREE_READONLY (copy) = TREE_READONLY (decl);
6019	TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
6020	if (!DECL_BY_REFERENCE (decl))
6021	{
6022	TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
6023	DECL_NOT_GIMPLE_REG_P (copy)
6024	= (DECL_NOT_GIMPLE_REG_P (decl)
6025	/* RESULT_DECLs are treated special by needs_to_live_in_memory,
6026	mirror that to the created VAR_DECL. */
6027	|| (TREE_CODE (decl) == RESULT_DECL
6028	&& aggregate_value_p (decl, id->src_fn)));
6029	}
6030
6031	return copy_decl_for_dup_finish (id, decl, copy);
6032	}
6033
6034	tree
6035	copy_decl_no_change (tree decl, copy_body_data *id)
6036	{
6037	tree copy;
6038
6039	copy = copy_node (decl);
6040
6041	/* The COPY is not abstract; it will be generated in DST_FN. */
6042	DECL_ABSTRACT_P (copy) = false;
6043	lang_hooks.dup_lang_specific_decl (copy);
6044
6045	/* TREE_ADDRESSABLE isn't used to indicate that a label's address has
6046	been taken; it's for internal bookkeeping in expand_goto_internal. */
6047	if (TREE_CODE (copy) == LABEL_DECL)
6048	{
6049	TREE_ADDRESSABLE (copy) = 0;
6050	LABEL_DECL_UID (copy) = -1;
6051	}
6052
6053	return copy_decl_for_dup_finish (id, decl, copy);
6054	}
6055
6056	static tree
6057	copy_decl_maybe_to_var (tree decl, copy_body_data *id)
6058	{
6059	if (TREE_CODE (decl) == PARM_DECL || TREE_CODE (decl) == RESULT_DECL)
6060	return copy_decl_to_var (decl, id);
6061	else
6062	return copy_decl_no_change (decl, id);
6063	}
6064
6065	/* Return a copy of the function's argument tree without any modifications. */
6066
6067	static tree
6068	copy_arguments_nochange (tree orig_parm, copy_body_data * id)
6069	{
6070	tree arg, *parg;
6071	tree new_parm = NULL;
6072
6073	parg = &new_parm;
6074	for (arg = orig_parm; arg; arg = DECL_CHAIN (arg))
6075	{
6076	tree new_tree = remap_decl (decl: arg, id);
6077	if (TREE_CODE (new_tree) != PARM_DECL)
6078	new_tree = id->copy_decl (arg, id);
6079	lang_hooks.dup_lang_specific_decl (new_tree);
6080	*parg = new_tree;
6081	parg = &DECL_CHAIN (new_tree);
6082	}
6083	return new_parm;
6084	}
6085
6086	/* Return a copy of the function's static chain. */
6087	static tree
6088	copy_static_chain (tree static_chain, copy_body_data * id)
6089	{
6090	tree *chain_copy, *pvar;
6091
6092	chain_copy = &static_chain;
6093	for (pvar = chain_copy; *pvar; pvar = &DECL_CHAIN (*pvar))
6094	{
6095	tree new_tree = remap_decl (decl: *pvar, id);
6096	lang_hooks.dup_lang_specific_decl (new_tree);
6097	DECL_CHAIN (new_tree) = DECL_CHAIN (*pvar);
6098	*pvar = new_tree;
6099	}
6100	return static_chain;
6101	}
6102
6103	/* Return true if the function is allowed to be versioned.
6104	This is a guard for the versioning functionality. */
6105
6106	bool
6107	tree_versionable_function_p (tree fndecl)
6108	{
6109	return (!lookup_attribute (attr_name: "noclone", DECL_ATTRIBUTES (fndecl))
6110	&& copy_forbidden (DECL_STRUCT_FUNCTION (fndecl)) == NULL);
6111	}
6112
6113	/* Update clone info after duplication. */
6114
6115	static void
6116	update_clone_info (copy_body_data * id)
6117	{
6118	struct cgraph_node *this_node = id->dst_node;
6119	if (!this_node->clones)
6120	return;
6121	for (cgraph_node *node = this_node->clones; node != this_node;)
6122	{
6123	/* First update replace maps to match the new body. */
6124	clone_info *info = clone_info::get (node);
6125	if (info && info->tree_map)
6126	{
6127	unsigned int i;
6128	for (i = 0; i < vec_safe_length (v: info->tree_map); i++)
6129	{
6130	struct ipa_replace_map *replace_info;
6131	replace_info = (*info->tree_map)[i];
6132	walk_tree (&replace_info->new_tree, copy_tree_body_r, id, NULL);
6133	}
6134	}
6135
6136	if (node->clones)
6137	node = node->clones;
6138	else if (node->next_sibling_clone)
6139	node = node->next_sibling_clone;
6140	else
6141	{
6142	while (node != id->dst_node && !node->next_sibling_clone)
6143	node = node->clone_of;
6144	if (node != id->dst_node)
6145	node = node->next_sibling_clone;
6146	}
6147	}
6148	}
6149
6150	/* Create a copy of a function's tree.
6151	OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes
6152	of the original function and the new copied function
6153	respectively. In case we want to replace a DECL
6154	tree with another tree while duplicating the function's
6155	body, TREE_MAP represents the mapping between these
6156	trees. If UPDATE_CLONES is set, the call_stmt fields
6157	of edges of clones of the function will be updated.
6158
6159	If non-NULL PARAM_ADJUSTMENTS determines how function prototype (i.e. the
6160	function parameters and return value) should be modified).
6161	If non-NULL BLOCKS_TO_COPY determine what basic blocks to copy.
6162	If non_NULL NEW_ENTRY determine new entry BB of the clone.
6163	*/
6164	void
6165	tree_function_versioning (tree old_decl, tree new_decl,
6166	vec<ipa_replace_map *, va_gc> *tree_map,
6167	ipa_param_adjustments *param_adjustments,
6168	bool update_clones, bitmap blocks_to_copy,
6169	basic_block new_entry)
6170	{
6171	struct cgraph_node *old_version_node;
6172	struct cgraph_node *new_version_node;
6173	copy_body_data id;
6174	tree p;
6175	unsigned i;
6176	struct ipa_replace_map *replace_info;
6177	basic_block old_entry_block, bb;
6178	auto_vec<gimple *, 10> init_stmts;
6179	tree vars = NULL_TREE;
6180
6181	/* We can get called recursively from expand_call_inline via clone
6182	materialization. While expand_call_inline maintains input_location
6183	we cannot tolerate it to leak into the materialized clone. */
6184	location_t saved_location = input_location;
6185	input_location = UNKNOWN_LOCATION;
6186
6187	gcc_assert (TREE_CODE (old_decl) == FUNCTION_DECL
6188	&& TREE_CODE (new_decl) == FUNCTION_DECL);
6189	DECL_POSSIBLY_INLINED (old_decl) = 1;
6190
6191	old_version_node = cgraph_node::get (decl: old_decl);
6192	gcc_checking_assert (old_version_node);
6193	new_version_node = cgraph_node::get (decl: new_decl);
6194	gcc_checking_assert (new_version_node);
6195
6196	/* Copy over debug args. */
6197	if (DECL_HAS_DEBUG_ARGS_P (old_decl))
6198	{
6199	vec<tree, va_gc> **new_debug_args, **old_debug_args;
6200	gcc_checking_assert (decl_debug_args_lookup (new_decl) == NULL);
6201	DECL_HAS_DEBUG_ARGS_P (new_decl) = 0;
6202	old_debug_args = decl_debug_args_lookup (old_decl);
6203	if (old_debug_args)
6204	{
6205	new_debug_args = decl_debug_args_insert (new_decl);
6206	*new_debug_args = vec_safe_copy (src: *old_debug_args);
6207	}
6208	}
6209
6210	/* Output the inlining info for this abstract function, since it has been
6211	inlined. If we don't do this now, we can lose the information about the
6212	variables in the function when the blocks get blown away as soon as we
6213	remove the cgraph node. */
6214	(*debug_hooks->outlining_inline_function) (old_decl);
6215
6216	DECL_ARTIFICIAL (new_decl) = 1;
6217	DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl);
6218	if (DECL_ORIGIN (old_decl) == old_decl)
6219	old_version_node->used_as_abstract_origin = true;
6220	DECL_FUNCTION_PERSONALITY (new_decl) = DECL_FUNCTION_PERSONALITY (old_decl);
6221
6222	/* Prepare the data structures for the tree copy. */
6223	memset (s: &id, c: 0, n: sizeof (id));
6224
6225	/* Generate a new name for the new version. */
6226	id.statements_to_fold = new hash_set<gimple *>;
6227
6228	id.decl_map = new hash_map<tree, tree>;
6229	id.debug_map = NULL;
6230	id.src_fn = old_decl;
6231	id.dst_fn = new_decl;
6232	id.src_node = old_version_node;
6233	id.dst_node = new_version_node;
6234	id.src_cfun = DECL_STRUCT_FUNCTION (old_decl);
6235	id.blocks_to_copy = blocks_to_copy;
6236
6237	id.copy_decl = copy_decl_no_change;
6238	id.transform_call_graph_edges
6239	= update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE;
6240	id.transform_new_cfg = true;
6241	id.transform_return_to_modify = false;
6242	id.transform_parameter = false;
6243
6244	old_entry_block = ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (old_decl));
6245	DECL_RESULT (new_decl) = DECL_RESULT (old_decl);
6246	DECL_ARGUMENTS (new_decl) = DECL_ARGUMENTS (old_decl);
6247	initialize_cfun (new_fndecl: new_decl, callee_fndecl: old_decl,
6248	count: new_entry ? new_entry->count : old_entry_block->count);
6249	new_version_node->calls_declare_variant_alt
6250	= old_version_node->calls_declare_variant_alt;
6251	if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
6252	DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
6253	= id.src_cfun->gimple_df->ipa_pta;
6254
6255	/* Copy the function's static chain. */
6256	p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl;
6257	if (p)
6258	DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl
6259	= copy_static_chain (static_chain: p, id: &id);
6260
6261	auto_vec<int, 16> new_param_indices;
6262	clone_info *info = clone_info::get (node: old_version_node);
6263	ipa_param_adjustments *old_param_adjustments
6264	= info ? info->param_adjustments : NULL;
6265	if (old_param_adjustments)
6266	old_param_adjustments->get_updated_indices (new_indices: &new_param_indices);
6267
6268	/* If there's a tree_map, prepare for substitution. */
6269	if (tree_map)
6270	for (i = 0; i < tree_map->length (); i++)
6271	{
6272	gimple *init;
6273	replace_info = (*tree_map)[i];
6274
6275	int p = replace_info->parm_num;
6276	if (old_param_adjustments)
6277	p = new_param_indices[p];
6278
6279	tree parm;
6280	for (parm = DECL_ARGUMENTS (old_decl); p;
6281	parm = DECL_CHAIN (parm))
6282	p--;
6283	gcc_assert (parm);
6284	init = setup_one_parameter (id: &id, p: parm, value: replace_info->new_tree,
6285	fn: id.src_fn, NULL, vars: &vars);
6286	if (init)
6287	init_stmts.safe_push (obj: init);
6288	}
6289
6290	ipa_param_body_adjustments *param_body_adjs = NULL;
6291	if (param_adjustments)
6292	{
6293	param_body_adjs = new ipa_param_body_adjustments (param_adjustments,
6294	new_decl, old_decl,
6295	&id, &vars, tree_map);
6296	id.param_body_adjs = param_body_adjs;
6297	DECL_ARGUMENTS (new_decl) = param_body_adjs->get_new_param_chain ();
6298	}
6299	else if (DECL_ARGUMENTS (old_decl) != NULL_TREE)
6300	DECL_ARGUMENTS (new_decl)
6301	= copy_arguments_nochange (DECL_ARGUMENTS (old_decl), id: &id);
6302
6303	DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), id: &id);
6304	BLOCK_SUPERCONTEXT (DECL_INITIAL (new_decl)) = new_decl;
6305
6306	declare_inline_vars (DECL_INITIAL (new_decl), vars);
6307
6308	if (!vec_safe_is_empty (DECL_STRUCT_FUNCTION (old_decl)->local_decls))
6309	/* Add local vars. */
6310	add_local_variables (DECL_STRUCT_FUNCTION (old_decl), cfun, id: &id);
6311
6312	if (DECL_RESULT (old_decl) == NULL_TREE)
6313	;
6314	else if (param_adjustments && param_adjustments->m_skip_return
6315	&& !VOID_TYPE_P (TREE_TYPE (DECL_RESULT (old_decl))))
6316	{
6317	tree resdecl_repl = copy_result_decl_to_var (DECL_RESULT (old_decl),
6318	id: &id);
6319	declare_inline_vars (NULL, vars: resdecl_repl);
6320	if (DECL_BY_REFERENCE (DECL_RESULT (old_decl)))
6321	resdecl_repl = build_fold_addr_expr (resdecl_repl);
6322	insert_decl_map (id: &id, DECL_RESULT (old_decl), value: resdecl_repl);
6323
6324	DECL_RESULT (new_decl)
6325	= build_decl (DECL_SOURCE_LOCATION (DECL_RESULT (old_decl)),
6326	RESULT_DECL, NULL_TREE, void_type_node);
6327	DECL_CONTEXT (DECL_RESULT (new_decl)) = new_decl;
6328	DECL_IS_MALLOC (new_decl) = false;
6329	cfun->returns_struct = 0;
6330	cfun->returns_pcc_struct = 0;
6331	}
6332	else
6333	{
6334	tree old_name;
6335	DECL_RESULT (new_decl) = remap_decl (DECL_RESULT (old_decl), id: &id);
6336	lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl));
6337	if (gimple_in_ssa_p (fun: id.src_cfun)
6338	&& DECL_BY_REFERENCE (DECL_RESULT (old_decl))
6339	&& (old_name = ssa_default_def (id.src_cfun, DECL_RESULT (old_decl))))
6340	{
6341	tree new_name = make_ssa_name (DECL_RESULT (new_decl));
6342	insert_decl_map (id: &id, key: old_name, value: new_name);
6343	SSA_NAME_DEF_STMT (new_name) = gimple_build_nop ();
6344	set_ssa_default_def (cfun, DECL_RESULT (new_decl), new_name);
6345	}
6346	}
6347
6348	/* Set up the destination functions loop tree. */
6349	if (loops_for_fn (DECL_STRUCT_FUNCTION (old_decl)) != NULL)
6350	{
6351	cfun->curr_properties &= ~PROP_loops;
6352	loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6353	cfun->curr_properties |= PROP_loops;
6354	}
6355
6356	/* Copy the Function's body. */
6357	copy_body (id: &id, ENTRY_BLOCK_PTR_FOR_FN (cfun), EXIT_BLOCK_PTR_FOR_FN (cfun),
6358	new_entry);
6359
6360	/* Renumber the lexical scoping (non-code) blocks consecutively. */
6361	number_blocks (new_decl);
6362
6363	/* We want to create the BB unconditionally, so that the addition of
6364	debug stmts doesn't affect BB count, which may in the end cause
6365	codegen differences. */
6366	bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6367	while (init_stmts.length ())
6368	insert_init_stmt (id: &id, bb, init_stmt: init_stmts.pop ());
6369	if (param_body_adjs)
6370	param_body_adjs->append_init_stmts (bb);
6371	update_clone_info (id: &id);
6372
6373	/* Remap the nonlocal_goto_save_area, if any. */
6374	if (cfun->nonlocal_goto_save_area)
6375	{
6376	struct walk_stmt_info wi;
6377
6378	memset (s: &wi, c: 0, n: sizeof (wi));
6379	wi.info = &id;
6380	walk_tree (&cfun->nonlocal_goto_save_area, remap_gimple_op_r, &wi, NULL);
6381	}
6382
6383	/* Clean up. */
6384	delete id.decl_map;
6385	if (id.debug_map)
6386	delete id.debug_map;
6387	free_dominance_info (CDI_DOMINATORS);
6388	free_dominance_info (CDI_POST_DOMINATORS);
6389
6390	update_max_bb_count ();
6391	fold_marked_statements (first: 0, statements: id.statements_to_fold);
6392	delete id.statements_to_fold;
6393	delete_unreachable_blocks_update_callgraph (dst_node: id.dst_node, update_clones);
6394	if (id.dst_node->definition)
6395	cgraph_edge::rebuild_references ();
6396	if (loops_state_satisfies_p (flags: LOOPS_NEED_FIXUP))
6397	{
6398	calculate_dominance_info (CDI_DOMINATORS);
6399	fix_loop_structure (NULL);
6400	}
6401	update_ssa (TODO_update_ssa);
6402
6403	/* After partial cloning we need to rescale frequencies, so they are
6404	within proper range in the cloned function. */
6405	if (new_entry)
6406	{
6407	struct cgraph_edge *e;
6408	rebuild_frequencies ();
6409
6410	new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6411	for (e = new_version_node->callees; e; e = e->next_callee)
6412	{
6413	basic_block bb = gimple_bb (g: e->call_stmt);
6414	e->count = bb->count;
6415	}
6416	for (e = new_version_node->indirect_calls; e; e = e->next_callee)
6417	{
6418	basic_block bb = gimple_bb (g: e->call_stmt);
6419	e->count = bb->count;
6420	}
6421	}
6422
6423	if (param_body_adjs && MAY_HAVE_DEBUG_BIND_STMTS)
6424	{
6425	vec<tree, va_gc> **debug_args = NULL;
6426	unsigned int len = 0;
6427	unsigned reset_len = param_body_adjs->m_reset_debug_decls.length ();
6428
6429	for (i = 0; i < reset_len; i++)
6430	{
6431	tree parm = param_body_adjs->m_reset_debug_decls[i];
6432	gcc_assert (is_gimple_reg (parm));
6433	tree ddecl;
6434
6435	if (debug_args == NULL)
6436	{
6437	debug_args = decl_debug_args_insert (new_decl);
6438	len = vec_safe_length (v: *debug_args);
6439	}
6440	ddecl = build_debug_expr_decl (TREE_TYPE (parm));
6441	/* FIXME: Is setting the mode really necessary? */
6442	SET_DECL_MODE (ddecl, DECL_MODE (parm));
6443	vec_safe_push (v&: *debug_args, DECL_ORIGIN (parm));
6444	vec_safe_push (v&: *debug_args, obj: ddecl);
6445	}
6446	if (debug_args != NULL)
6447	{
6448	/* On the callee side, add
6449	DEBUG D#Y s=> parm
6450	DEBUG var => D#Y
6451	stmts to the first bb where var is a VAR_DECL created for the
6452	optimized away parameter in DECL_INITIAL block. This hints
6453	in the debug info that var (whole DECL_ORIGIN is the parm
6454	PARM_DECL) is optimized away, but could be looked up at the
6455	call site as value of D#X there. */
6456	gimple_stmt_iterator cgsi
6457	= gsi_after_labels (bb: single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6458	gimple *def_temp;
6459	tree var = vars;
6460	i = vec_safe_length (v: *debug_args);
6461	do
6462	{
6463	tree vexpr = NULL_TREE;
6464	i -= 2;
6465	while (var != NULL_TREE
6466	&& DECL_ABSTRACT_ORIGIN (var) != (**debug_args)[i])
6467	var = TREE_CHAIN (var);
6468	if (var == NULL_TREE)
6469	break;
6470	tree parm = (**debug_args)[i];
6471	if (tree parm_ddef = ssa_default_def (id.src_cfun, parm))
6472	if (tree *d
6473	= param_body_adjs->m_dead_ssa_debug_equiv.get (k: parm_ddef))
6474	vexpr = *d;
6475	if (!vexpr)
6476	{
6477	vexpr = build_debug_expr_decl (TREE_TYPE (parm));
6478	/* FIXME: Is setting the mode really necessary? */
6479	SET_DECL_MODE (vexpr, DECL_MODE (parm));
6480	}
6481	def_temp = gimple_build_debug_bind (var, vexpr, NULL);
6482	gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
6483	def_temp = gimple_build_debug_source_bind (vexpr, parm, NULL);
6484	gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
6485	}
6486	while (i > len);
6487	}
6488	}
6489	delete param_body_adjs;
6490	free_dominance_info (CDI_DOMINATORS);
6491	free_dominance_info (CDI_POST_DOMINATORS);
6492
6493	gcc_assert (!id.debug_stmts.exists ());
6494	pop_cfun ();
6495	input_location = saved_location;
6496	return;
6497	}
6498
6499	/* EXP is CALL_EXPR present in a GENERIC expression tree. Try to integrate
6500	the callee and return the inlined body on success. */
6501
6502	tree
6503	maybe_inline_call_in_expr (tree exp)
6504	{
6505	tree fn = get_callee_fndecl (exp);
6506
6507	/* We can only try to inline "const" functions. */
6508	if (fn && TREE_READONLY (fn) && DECL_SAVED_TREE (fn))
6509	{
6510	call_expr_arg_iterator iter;
6511	copy_body_data id;
6512	tree param, arg, t;
6513	hash_map<tree, tree> decl_map;
6514
6515	/* Remap the parameters. */
6516	for (param = DECL_ARGUMENTS (fn), arg = first_call_expr_arg (exp, iter: &iter);
6517	param;
6518	param = DECL_CHAIN (param), arg = next_call_expr_arg (iter: &iter))
6519	decl_map.put (k: param, v: arg);
6520
6521	memset (s: &id, c: 0, n: sizeof (id));
6522	id.src_fn = fn;
6523	id.dst_fn = current_function_decl;
6524	id.src_cfun = DECL_STRUCT_FUNCTION (fn);
6525	id.decl_map = &decl_map;
6526
6527	id.copy_decl = copy_decl_no_change;
6528	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6529	id.transform_new_cfg = false;
6530	id.transform_return_to_modify = true;
6531	id.transform_parameter = true;
6532
6533	/* Make sure not to unshare trees behind the front-end's back
6534	since front-end specific mechanisms may rely on sharing. */
6535	id.regimplify = false;
6536	id.do_not_unshare = true;
6537
6538	/* We're not inside any EH region. */
6539	id.eh_lp_nr = 0;
6540
6541	t = copy_tree_body (id: &id);
6542
6543	/* We can only return something suitable for use in a GENERIC
6544	expression tree. */
6545	if (TREE_CODE (t) == MODIFY_EXPR)
6546	return TREE_OPERAND (t, 1);
6547	}
6548
6549	return NULL_TREE;
6550	}
6551
6552	/* Duplicate a type, fields and all. */
6553
6554	tree
6555	build_duplicate_type (tree type)
6556	{
6557	struct copy_body_data id;
6558
6559	memset (s: &id, c: 0, n: sizeof (id));
6560	id.src_fn = current_function_decl;
6561	id.dst_fn = current_function_decl;
6562	id.src_cfun = cfun;
6563	id.decl_map = new hash_map<tree, tree>;
6564	id.debug_map = NULL;
6565	id.copy_decl = copy_decl_no_change;
6566
6567	type = remap_type_1 (type, id: &id);
6568
6569	delete id.decl_map;
6570	if (id.debug_map)
6571	delete id.debug_map;
6572
6573	TYPE_CANONICAL (type) = type;
6574
6575	return type;
6576	}
6577
6578	/* Unshare the entire DECL_SAVED_TREE of FN and return the remapped
6579	parameters and RESULT_DECL in PARMS and RESULT. Used by C++ constexpr
6580	evaluation. */
6581
6582	tree
6583	copy_fn (tree fn, tree& parms, tree& result)
6584	{
6585	copy_body_data id;
6586	tree param;
6587	hash_map<tree, tree> decl_map;
6588
6589	tree *p = &parms;
6590	*p = NULL_TREE;
6591
6592	memset (s: &id, c: 0, n: sizeof (id));
6593	id.src_fn = fn;
6594	id.dst_fn = current_function_decl;
6595	id.src_cfun = DECL_STRUCT_FUNCTION (fn);
6596	id.decl_map = &decl_map;
6597
6598	id.copy_decl = copy_decl_no_change;
6599	id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6600	id.transform_new_cfg = false;
6601	id.transform_return_to_modify = false;
6602	id.transform_parameter = true;
6603
6604	/* Make sure not to unshare trees behind the front-end's back
6605	since front-end specific mechanisms may rely on sharing. */
6606	id.regimplify = false;
6607	id.do_not_unshare = true;
6608	id.do_not_fold = true;
6609
6610	/* We're not inside any EH region. */
6611	id.eh_lp_nr = 0;
6612
6613	/* Remap the parameters and result and return them to the caller. */
6614	for (param = DECL_ARGUMENTS (fn);
6615	param;
6616	param = DECL_CHAIN (param))
6617	{
6618	*p = remap_decl (decl: param, id: &id);
6619	p = &DECL_CHAIN (*p);
6620	}
6621
6622	if (DECL_RESULT (fn))
6623	result = remap_decl (DECL_RESULT (fn), id: &id);
6624	else
6625	result = NULL_TREE;
6626
6627	return copy_tree_body (id: &id);
6628	}
6629