1	/* A pass for lowering trees to RTL.
2	Copyright (C) 2004-2023 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify
7	it under the terms of the GNU General Public License as published by
8	the Free Software Foundation; either version 3, or (at your option)
9	any later version.
10
11	GCC is distributed in the hope that it will be useful,
12	but WITHOUT ANY WARRANTY; without even the implied warranty of
13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	GNU General Public License for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#include "system.h"
22	#include "coretypes.h"
23	#include "backend.h"
24	#include "target.h"
25	#include "rtl.h"
26	#include "tree.h"
27	#include "gimple.h"
28	#include "cfghooks.h"
29	#include "tree-pass.h"
30	#include "memmodel.h"
31	#include "tm_p.h"
32	#include "ssa.h"
33	#include "optabs.h"
34	#include "regs.h" /* For reg_renumber. */
35	#include "emit-rtl.h"
36	#include "recog.h"
37	#include "cgraph.h"
38	#include "diagnostic.h"
39	#include "fold-const.h"
40	#include "varasm.h"
41	#include "stor-layout.h"
42	#include "stmt.h"
43	#include "print-tree.h"
44	#include "cfgrtl.h"
45	#include "cfganal.h"
46	#include "cfgbuild.h"
47	#include "cfgcleanup.h"
48	#include "dojump.h"
49	#include "explow.h"
50	#include "calls.h"
51	#include "expr.h"
52	#include "internal-fn.h"
53	#include "tree-eh.h"
54	#include "gimple-iterator.h"
55	#include "gimple-expr.h"
56	#include "gimple-walk.h"
57	#include "tree-cfg.h"
58	#include "tree-dfa.h"
59	#include "tree-ssa.h"
60	#include "except.h"
61	#include "gimple-pretty-print.h"
62	#include "toplev.h"
63	#include "debug.h"
64	#include "tree-inline.h"
65	#include "value-prof.h"
66	#include "tree-ssa-live.h"
67	#include "tree-outof-ssa.h"
68	#include "cfgloop.h"
69	#include "insn-attr.h" /* For INSN_SCHEDULING. */
70	#include "stringpool.h"
71	#include "attribs.h"
72	#include "asan.h"
73	#include "tree-ssa-address.h"
74	#include "output.h"
75	#include "builtins.h"
76	#include "opts.h"
77
78	/* Some systems use __main in a way incompatible with its use in gcc, in these
79	cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
80	give the same symbol without quotes for an alternative entry point. You
81	must define both, or neither. */
82	#ifndef NAME__MAIN
83	#define NAME__MAIN "__main"
84	#endif
85
86	/* This variable holds information helping the rewriting of SSA trees
87	into RTL. */
88	struct ssaexpand SA;
89
90	/* This variable holds the currently expanded gimple statement for purposes
91	of comminucating the profile info to the builtin expanders. */
92	gimple *currently_expanding_gimple_stmt;
93
94	static rtx expand_debug_expr (tree);
95
96	static bool defer_stack_allocation (tree, bool);
97
98	static void record_alignment_for_reg_var (unsigned int);
99
100	/* Return an expression tree corresponding to the RHS of GIMPLE
101	statement STMT. */
102
103	tree
104	gimple_assign_rhs_to_tree (gimple *stmt)
105	{
106	tree t;
107	switch (gimple_assign_rhs_class (gs: stmt))
108	{
109	case GIMPLE_TERNARY_RHS:
110	t = build3 (gimple_assign_rhs_code (gs: stmt),
111	TREE_TYPE (gimple_assign_lhs (stmt)),
112	gimple_assign_rhs1 (gs: stmt), gimple_assign_rhs2 (gs: stmt),
113	gimple_assign_rhs3 (gs: stmt));
114	break;
115	case GIMPLE_BINARY_RHS:
116	t = build2 (gimple_assign_rhs_code (gs: stmt),
117	TREE_TYPE (gimple_assign_lhs (stmt)),
118	gimple_assign_rhs1 (gs: stmt), gimple_assign_rhs2 (gs: stmt));
119	break;
120	case GIMPLE_UNARY_RHS:
121	t = build1 (gimple_assign_rhs_code (gs: stmt),
122	TREE_TYPE (gimple_assign_lhs (stmt)),
123	gimple_assign_rhs1 (gs: stmt));
124	break;
125	case GIMPLE_SINGLE_RHS:
126	{
127	t = gimple_assign_rhs1 (gs: stmt);
128	/* Avoid modifying this tree in place below. */
129	if ((gimple_has_location (g: stmt) && CAN_HAVE_LOCATION_P (t)
130	&& gimple_location (g: stmt) != EXPR_LOCATION (t))
131	|| (gimple_block (g: stmt) && currently_expanding_to_rtl
132	&& EXPR_P (t)))
133	t = copy_node (t);
134	break;
135	}
136	default:
137	gcc_unreachable ();
138	}
139
140	if (gimple_has_location (g: stmt) && CAN_HAVE_LOCATION_P (t))
141	SET_EXPR_LOCATION (t, gimple_location (stmt));
142
143	return t;
144	}
145
146
147	#ifndef STACK_ALIGNMENT_NEEDED
148	#define STACK_ALIGNMENT_NEEDED 1
149	#endif
150
151	#define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
152
153	/* Choose either CUR or NEXT as the leader DECL for a partition.
154	Prefer ignored decls, to simplify debug dumps and reduce ambiguity
155	out of the same user variable being in multiple partitions (this is
156	less likely for compiler-introduced temps). */
157
158	static tree
159	leader_merge (tree cur, tree next)
160	{
161	if (cur == NULL || cur == next)
162	return next;
163
164	if (DECL_P (cur) && DECL_IGNORED_P (cur))
165	return cur;
166
167	if (DECL_P (next) && DECL_IGNORED_P (next))
168	return next;
169
170	return cur;
171	}
172
173	/* Associate declaration T with storage space X. If T is no
174	SSA name this is exactly SET_DECL_RTL, otherwise make the
175	partition of T associated with X. */
176	static inline void
177	set_rtl (tree t, rtx x)
178	{
179	gcc_checking_assert (!x
180	|| !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
181	|| (use_register_for_decl (t)
182	? (REG_P (x)
183	|| (GET_CODE (x) == CONCAT
184	&& (REG_P (XEXP (x, 0))
185	|| SUBREG_P (XEXP (x, 0)))
186	&& (REG_P (XEXP (x, 1))
187	|| SUBREG_P (XEXP (x, 1))))
188	/* We need to accept PARALLELs for RESUT_DECLs
189	because of vector types with BLKmode returned
190	in multiple registers, but they are supposed
191	to be uncoalesced. */
192	|| (GET_CODE (x) == PARALLEL
193	&& SSAVAR (t)
194	&& TREE_CODE (SSAVAR (t)) == RESULT_DECL
195	&& (GET_MODE (x) == BLKmode
196	|| !flag_tree_coalesce_vars)))
197	: (MEM_P (x) || x == pc_rtx
198	|| (GET_CODE (x) == CONCAT
199	&& MEM_P (XEXP (x, 0))
200	&& MEM_P (XEXP (x, 1))))));
201	/* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
202	RESULT_DECLs has the expected mode. For memory, we accept
203	unpromoted modes, since that's what we're likely to get. For
204	PARM_DECLs and RESULT_DECLs, we'll have been called by
205	set_parm_rtl, which will give us the default def, so we don't
206	have to compute it ourselves. For RESULT_DECLs, we accept mode
207	mismatches too, as long as we have BLKmode or are not coalescing
208	across variables, so that we don't reject BLKmode PARALLELs or
209	unpromoted REGs. */
210	gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
211	|| (SSAVAR (t)
212	&& TREE_CODE (SSAVAR (t)) == RESULT_DECL
213	&& (promote_ssa_mode (t, NULL) == BLKmode
214	|| !flag_tree_coalesce_vars))
215	|| !use_register_for_decl (t)
216	|| GET_MODE (x) == promote_ssa_mode (t, NULL));
217
218	if (x)
219	{
220	bool skip = false;
221	tree cur = NULL_TREE;
222	rtx xm = x;
223
224	retry:
225	if (MEM_P (xm))
226	cur = MEM_EXPR (xm);
227	else if (REG_P (xm))
228	cur = REG_EXPR (xm);
229	else if (SUBREG_P (xm))
230	{
231	gcc_assert (subreg_lowpart_p (xm));
232	xm = SUBREG_REG (xm);
233	goto retry;
234	}
235	else if (GET_CODE (xm) == CONCAT)
236	{
237	xm = XEXP (xm, 0);
238	goto retry;
239	}
240	else if (GET_CODE (xm) == PARALLEL)
241	{
242	xm = XVECEXP (xm, 0, 0);
243	gcc_assert (GET_CODE (xm) == EXPR_LIST);
244	xm = XEXP (xm, 0);
245	goto retry;
246	}
247	else if (xm == pc_rtx)
248	skip = true;
249	else
250	gcc_unreachable ();
251
252	tree next = skip ? cur : leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
253
254	if (cur != next)
255	{
256	if (MEM_P (x))
257	set_mem_attributes (x,
258	next && TREE_CODE (next) == SSA_NAME
259	? TREE_TYPE (next)
260	: next, true);
261	else
262	set_reg_attrs_for_decl_rtl (t: next, x);
263	}
264	}
265
266	if (TREE_CODE (t) == SSA_NAME)
267	{
268	int part = var_to_partition (map: SA.map, var: t);
269	if (part != NO_PARTITION)
270	{
271	if (SA.partition_to_pseudo[part])
272	gcc_assert (SA.partition_to_pseudo[part] == x);
273	else if (x != pc_rtx)
274	SA.partition_to_pseudo[part] = x;
275	}
276	/* For the benefit of debug information at -O0 (where
277	vartracking doesn't run) record the place also in the base
278	DECL. For PARMs and RESULTs, do so only when setting the
279	default def. */
280	if (x && x != pc_rtx && SSA_NAME_VAR (t)
281	&& (VAR_P (SSA_NAME_VAR (t))
282	|| SSA_NAME_IS_DEFAULT_DEF (t)))
283	{
284	tree var = SSA_NAME_VAR (t);
285	/* If we don't yet have something recorded, just record it now. */
286	if (!DECL_RTL_SET_P (var))
287	SET_DECL_RTL (var, x);
288	/* If we have it set already to "multiple places" don't
289	change this. */
290	else if (DECL_RTL (var) == pc_rtx)
291	;
292	/* If we have something recorded and it's not the same place
293	as we want to record now, we have multiple partitions for the
294	same base variable, with different places. We can't just
295	randomly chose one, hence we have to say that we don't know.
296	This only happens with optimization, and there var-tracking
297	will figure out the right thing. */
298	else if (DECL_RTL (var) != x)
299	SET_DECL_RTL (var, pc_rtx);
300	}
301	}
302	else
303	SET_DECL_RTL (t, x);
304	}
305
306	/* This structure holds data relevant to one variable that will be
307	placed in a stack slot. */
308	class stack_var
309	{
310	public:
311	/* The Variable. */
312	tree decl;
313
314	/* Initially, the size of the variable. Later, the size of the partition,
315	if this variable becomes it's partition's representative. */
316	poly_uint64 size;
317
318	/* The *byte* alignment required for this variable. Or as, with the
319	size, the alignment for this partition. */
320	unsigned int alignb;
321
322	/* The partition representative. */
323	size_t representative;
324
325	/* The next stack variable in the partition, or EOC. */
326	size_t next;
327
328	/* The numbers of conflicting stack variables. */
329	bitmap conflicts;
330	};
331
332	#define EOC ((size_t)-1)
333
334	/* We have an array of such objects while deciding allocation. */
335	static class stack_var *stack_vars;
336	static size_t stack_vars_alloc;
337	static size_t stack_vars_num;
338	static hash_map<tree, size_t> *decl_to_stack_part;
339
340	/* Conflict bitmaps go on this obstack. This allows us to destroy
341	all of them in one big sweep. */
342	static bitmap_obstack stack_var_bitmap_obstack;
343
344	/* An array of indices such that stack_vars[stack_vars_sorted[i]].size
345	is non-decreasing. */
346	static size_t *stack_vars_sorted;
347
348	/* The phase of the stack frame. This is the known misalignment of
349	virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
350	(frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
351	static int frame_phase;
352
353	/* Used during expand_used_vars to remember if we saw any decls for
354	which we'd like to enable stack smashing protection. */
355	static bool has_protected_decls;
356
357	/* Used during expand_used_vars. Remember if we say a character buffer
358	smaller than our cutoff threshold. Used for -Wstack-protector. */
359	static bool has_short_buffer;
360
361	/* Compute the byte alignment to use for DECL. Ignore alignment
362	we can't do with expected alignment of the stack boundary. */
363
364	static unsigned int
365	align_local_variable (tree decl, bool really_expand)
366	{
367	unsigned int align;
368
369	if (TREE_CODE (decl) == SSA_NAME)
370	{
371	tree type = TREE_TYPE (decl);
372	machine_mode mode = TYPE_MODE (type);
373
374	align = TYPE_ALIGN (type);
375	if (mode != BLKmode
376	&& align < GET_MODE_ALIGNMENT (mode))
377	align = GET_MODE_ALIGNMENT (mode);
378	}
379	else
380	align = LOCAL_DECL_ALIGNMENT (decl);
381
382	if (hwasan_sanitize_stack_p ())
383	align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
384
385	if (TREE_CODE (decl) != SSA_NAME && really_expand)
386	/* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
387	That is done before IPA and could bump alignment based on host
388	backend even for offloaded code which wants different
389	LOCAL_DECL_ALIGNMENT. */
390	SET_DECL_ALIGN (decl, align);
391
392	return align / BITS_PER_UNIT;
393	}
394
395	/* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
396	down otherwise. Return truncated BASE value. */
397
398	static inline unsigned HOST_WIDE_INT
399	align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
400	{
401	return align_up ? (base + align - 1) & -align : base & -align;
402	}
403
404	/* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
405	Return the frame offset. */
406
407	static poly_int64
408	alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
409	{
410	poly_int64 offset, new_frame_offset;
411
412	if (FRAME_GROWS_DOWNWARD)
413	{
414	new_frame_offset
415	= aligned_lower_bound (frame_offset - frame_phase - size,
416	align) + frame_phase;
417	offset = new_frame_offset;
418	}
419	else
420	{
421	new_frame_offset
422	= aligned_upper_bound (frame_offset - frame_phase,
423	align) + frame_phase;
424	offset = new_frame_offset;
425	new_frame_offset += size;
426	}
427	frame_offset = new_frame_offset;
428
429	if (frame_offset_overflow (frame_offset, cfun->decl))
430	frame_offset = offset = 0;
431
432	return offset;
433	}
434
435	/* Ensure that the stack is aligned to ALIGN bytes.
436	Return the new frame offset. */
437	static poly_int64
438	align_frame_offset (unsigned HOST_WIDE_INT align)
439	{
440	return alloc_stack_frame_space (size: 0, align);
441	}
442
443	/* Accumulate DECL into STACK_VARS. */
444
445	static void
446	add_stack_var (tree decl, bool really_expand)
447	{
448	class stack_var *v;
449
450	if (stack_vars_num >= stack_vars_alloc)
451	{
452	if (stack_vars_alloc)
453	stack_vars_alloc = stack_vars_alloc * 3 / 2;
454	else
455	stack_vars_alloc = 32;
456	stack_vars
457	= XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
458	}
459	if (!decl_to_stack_part)
460	decl_to_stack_part = new hash_map<tree, size_t>;
461
462	v = &stack_vars[stack_vars_num];
463	decl_to_stack_part->put (k: decl, v: stack_vars_num);
464
465	v->decl = decl;
466	tree size = TREE_CODE (decl) == SSA_NAME
467	? TYPE_SIZE_UNIT (TREE_TYPE (decl))
468	: DECL_SIZE_UNIT (decl);
469	v->size = tree_to_poly_uint64 (size);
470	/* Ensure that all variables have size, so that &a != &b for any two
471	variables that are simultaneously live. */
472	if (known_eq (v->size, 0U))
473	v->size = 1;
474	v->alignb = align_local_variable (decl, really_expand);
475	/* An alignment of zero can mightily confuse us later. */
476	gcc_assert (v->alignb != 0);
477
478	/* All variables are initially in their own partition. */
479	v->representative = stack_vars_num;
480	v->next = EOC;
481
482	/* All variables initially conflict with no other. */
483	v->conflicts = NULL;
484
485	/* Ensure that this decl doesn't get put onto the list twice. */
486	set_rtl (t: decl, x: pc_rtx);
487
488	stack_vars_num++;
489	}
490
491	/* Make the decls associated with luid's X and Y conflict. */
492
493	static void
494	add_stack_var_conflict (size_t x, size_t y)
495	{
496	class stack_var *a = &stack_vars[x];
497	class stack_var *b = &stack_vars[y];
498	if (x == y)
499	return;
500	if (!a->conflicts)
501	a->conflicts = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
502	if (!b->conflicts)
503	b->conflicts = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
504	bitmap_set_bit (a->conflicts, y);
505	bitmap_set_bit (b->conflicts, x);
506	}
507
508	/* Check whether the decls associated with luid's X and Y conflict. */
509
510	static bool
511	stack_var_conflict_p (size_t x, size_t y)
512	{
513	class stack_var *a = &stack_vars[x];
514	class stack_var *b = &stack_vars[y];
515	if (x == y)
516	return false;
517	/* Partitions containing an SSA name result from gimple registers
518	with things like unsupported modes. They are top-level and
519	hence conflict with everything else. */
520	if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
521	return true;
522
523	if (!a->conflicts || !b->conflicts)
524	return false;
525	return bitmap_bit_p (a->conflicts, y);
526	}
527
528	/* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
529	enter its partition number into bitmap DATA. */
530
531	static bool
532	visit_op (gimple *, tree op, tree, void *data)
533	{
534	bitmap active = (bitmap)data;
535	op = get_base_address (t: op);
536	if (op
537	&& DECL_P (op)
538	&& DECL_RTL_IF_SET (op) == pc_rtx)
539	{
540	size_t *v = decl_to_stack_part->get (k: op);
541	if (v)
542	bitmap_set_bit (active, *v);
543	}
544	return false;
545	}
546
547	/* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
548	record conflicts between it and all currently active other partitions
549	from bitmap DATA. */
550
551	static bool
552	visit_conflict (gimple *, tree op, tree, void *data)
553	{
554	bitmap active = (bitmap)data;
555	op = get_base_address (t: op);
556	if (op
557	&& DECL_P (op)
558	&& DECL_RTL_IF_SET (op) == pc_rtx)
559	{
560	size_t *v = decl_to_stack_part->get (k: op);
561	if (v && bitmap_set_bit (active, *v))
562	{
563	size_t num = *v;
564	bitmap_iterator bi;
565	unsigned i;
566	gcc_assert (num < stack_vars_num);
567	EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
568	add_stack_var_conflict (x: num, y: i);
569	}
570	}
571	return false;
572	}
573
574	/* Helper routine for add_scope_conflicts, calculating the active partitions
575	at the end of BB, leaving the result in WORK. We're called to generate
576	conflicts when FOR_CONFLICT is true, otherwise we're just tracking
577	liveness. */
578
579	static void
580	add_scope_conflicts_1 (basic_block bb, bitmap work, bool for_conflict)
581	{
582	edge e;
583	edge_iterator ei;
584	gimple_stmt_iterator gsi;
585	walk_stmt_load_store_addr_fn visit;
586
587	bitmap_clear (work);
588	FOR_EACH_EDGE (e, ei, bb->preds)
589	bitmap_ior_into (work, (bitmap)e->src->aux);
590
591	visit = visit_op;
592
593	for (gsi = gsi_start_phis (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
594	{
595	gimple *stmt = gsi_stmt (i: gsi);
596	walk_stmt_load_store_addr_ops (stmt, work, NULL, NULL, visit);
597	}
598	for (gsi = gsi_after_labels (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
599	{
600	gimple *stmt = gsi_stmt (i: gsi);
601
602	if (gimple_clobber_p (s: stmt))
603	{
604	tree lhs = gimple_assign_lhs (gs: stmt);
605	size_t *v;
606	/* Nested function lowering might introduce LHSs
607	that are COMPONENT_REFs. */
608	if (!VAR_P (lhs))
609	continue;
610	if (DECL_RTL_IF_SET (lhs) == pc_rtx
611	&& (v = decl_to_stack_part->get (k: lhs)))
612	bitmap_clear_bit (work, *v);
613	}
614	else if (!is_gimple_debug (gs: stmt))
615	{
616	if (for_conflict
617	&& visit == visit_op)
618	{
619	/* If this is the first real instruction in this BB we need
620	to add conflicts for everything live at this point now.
621	Unlike classical liveness for named objects we can't
622	rely on seeing a def/use of the names we're interested in.
623	There might merely be indirect loads/stores. We'd not add any
624	conflicts for such partitions. */
625	bitmap_iterator bi;
626	unsigned i;
627	EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
628	{
629	class stack_var *a = &stack_vars[i];
630	if (!a->conflicts)
631	a->conflicts = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
632	bitmap_ior_into (a->conflicts, work);
633	}
634	visit = visit_conflict;
635	}
636	walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
637	}
638	}
639	}
640
641	/* Generate stack partition conflicts between all partitions that are
642	simultaneously live. */
643
644	static void
645	add_scope_conflicts (void)
646	{
647	basic_block bb;
648	bool changed;
649	bitmap work = BITMAP_ALLOC (NULL);
650	int *rpo;
651	int n_bbs;
652
653	/* We approximate the live range of a stack variable by taking the first
654	mention of its name as starting point(s), and by the end-of-scope
655	death clobber added by gimplify as ending point(s) of the range.
656	This overapproximates in the case we for instance moved an address-taken
657	operation upward, without also moving a dereference to it upwards.
658	But it's conservatively correct as a variable never can hold values
659	before its name is mentioned at least once.
660
661	We then do a mostly classical bitmap liveness algorithm. */
662
663	FOR_ALL_BB_FN (bb, cfun)
664	bb->aux = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
665
666	rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
667	n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
668
669	changed = true;
670	while (changed)
671	{
672	int i;
673	changed = false;
674	for (i = 0; i < n_bbs; i++)
675	{
676	bitmap active;
677	bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
678	active = (bitmap)bb->aux;
679	add_scope_conflicts_1 (bb, work, for_conflict: false);
680	if (bitmap_ior_into (active, work))
681	changed = true;
682	}
683	}
684
685	FOR_EACH_BB_FN (bb, cfun)
686	add_scope_conflicts_1 (bb, work, for_conflict: true);
687
688	free (ptr: rpo);
689	BITMAP_FREE (work);
690	FOR_ALL_BB_FN (bb, cfun)
691	BITMAP_FREE (bb->aux);
692	}
693
694	/* A subroutine of partition_stack_vars. A comparison function for qsort,
695	sorting an array of indices by the properties of the object. */
696
697	static int
698	stack_var_cmp (const void *a, const void *b)
699	{
700	size_t ia = *(const size_t *)a;
701	size_t ib = *(const size_t *)b;
702	unsigned int aligna = stack_vars[ia].alignb;
703	unsigned int alignb = stack_vars[ib].alignb;
704	poly_int64 sizea = stack_vars[ia].size;
705	poly_int64 sizeb = stack_vars[ib].size;
706	tree decla = stack_vars[ia].decl;
707	tree declb = stack_vars[ib].decl;
708	bool largea, largeb;
709	unsigned int uida, uidb;
710
711	/* Primary compare on "large" alignment. Large comes first. */
712	largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
713	largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
714	if (largea != largeb)
715	return (int)largeb - (int)largea;
716
717	/* Secondary compare on size, decreasing */
718	int diff = compare_sizes_for_sort (a: sizeb, b: sizea);
719	if (diff != 0)
720	return diff;
721
722	/* Tertiary compare on true alignment, decreasing. */
723	if (aligna < alignb)
724	return -1;
725	if (aligna > alignb)
726	return 1;
727
728	/* Final compare on ID for sort stability, increasing.
729	Two SSA names are compared by their version, SSA names come before
730	non-SSA names, and two normal decls are compared by their DECL_UID. */
731	if (TREE_CODE (decla) == SSA_NAME)
732	{
733	if (TREE_CODE (declb) == SSA_NAME)
734	uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
735	else
736	return -1;
737	}
738	else if (TREE_CODE (declb) == SSA_NAME)
739	return 1;
740	else
741	uida = DECL_UID (decla), uidb = DECL_UID (declb);
742	if (uida < uidb)
743	return 1;
744	if (uida > uidb)
745	return -1;
746	return 0;
747	}
748
749	struct part_traits : unbounded_int_hashmap_traits <size_t, bitmap> {};
750	typedef hash_map<size_t, bitmap, part_traits> part_hashmap;
751
752	/* If the points-to solution *PI points to variables that are in a partition
753	together with other variables add all partition members to the pointed-to
754	variables bitmap. */
755
756	static void
757	add_partitioned_vars_to_ptset (struct pt_solution *pt,
758	part_hashmap *decls_to_partitions,
759	hash_set<bitmap> *visited, bitmap temp)
760	{
761	bitmap_iterator bi;
762	unsigned i;
763	bitmap *part;
764
765	if (pt->anything
766	|| pt->vars == NULL
767	/* The pointed-to vars bitmap is shared, it is enough to
768	visit it once. */
769	|| visited->add (k: pt->vars))
770	return;
771
772	bitmap_clear (temp);
773
774	/* By using a temporary bitmap to store all members of the partitions
775	we have to add we make sure to visit each of the partitions only
776	once. */
777	EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
778	if ((!temp
779	|| !bitmap_bit_p (temp, i))
780	&& (part = decls_to_partitions->get (k: i)))
781	bitmap_ior_into (temp, *part);
782	if (!bitmap_empty_p (map: temp))
783	bitmap_ior_into (pt->vars, temp);
784	}
785
786	/* Update points-to sets based on partition info, so we can use them on RTL.
787	The bitmaps representing stack partitions will be saved until expand,
788	where partitioned decls used as bases in memory expressions will be
789	rewritten. */
790
791	static void
792	update_alias_info_with_stack_vars (void)
793	{
794	part_hashmap *decls_to_partitions = NULL;
795	size_t i, j;
796	tree var = NULL_TREE;
797
798	for (i = 0; i < stack_vars_num; i++)
799	{
800	bitmap part = NULL;
801	tree name;
802	struct ptr_info_def *pi;
803
804	/* Not interested in partitions with single variable. */
805	if (stack_vars[i].representative != i
806	|| stack_vars[i].next == EOC)
807	continue;
808
809	if (!decls_to_partitions)
810	{
811	decls_to_partitions = new part_hashmap;
812	cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
813	}
814
815	/* Create an SSA_NAME that points to the partition for use
816	as base during alias-oracle queries on RTL for bases that
817	have been partitioned. */
818	if (var == NULL_TREE)
819	var = create_tmp_var (ptr_type_node);
820	name = make_ssa_name (var);
821
822	/* Create bitmaps representing partitions. They will be used for
823	points-to sets later, so use GGC alloc. */
824	part = BITMAP_GGC_ALLOC ();
825	for (j = i; j != EOC; j = stack_vars[j].next)
826	{
827	tree decl = stack_vars[j].decl;
828	unsigned int uid = DECL_PT_UID (decl);
829	bitmap_set_bit (part, uid);
830	decls_to_partitions->put (k: uid, v: part);
831	cfun->gimple_df->decls_to_pointers->put (k: decl, v: name);
832	if (TREE_ADDRESSABLE (decl))
833	TREE_ADDRESSABLE (name) = 1;
834	}
835
836	/* Make the SSA name point to all partition members. */
837	pi = get_ptr_info (name);
838	pt_solution_set (&pi->pt, part, false);
839	}
840
841	/* Make all points-to sets that contain one member of a partition
842	contain all members of the partition. */
843	if (decls_to_partitions)
844	{
845	unsigned i;
846	tree name;
847	hash_set<bitmap> visited;
848	bitmap temp = BITMAP_ALLOC (obstack: &stack_var_bitmap_obstack);
849
850	FOR_EACH_SSA_NAME (i, name, cfun)
851	{
852	struct ptr_info_def *pi;
853
854	if (POINTER_TYPE_P (TREE_TYPE (name))
855	&& ((pi = SSA_NAME_PTR_INFO (name)) != NULL))
856	add_partitioned_vars_to_ptset (pt: &pi->pt, decls_to_partitions,
857	visited: &visited, temp);
858	}
859
860	add_partitioned_vars_to_ptset (pt: &cfun->gimple_df->escaped,
861	decls_to_partitions, visited: &visited, temp);
862
863	delete decls_to_partitions;
864	BITMAP_FREE (temp);
865	}
866	}
867
868	/* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
869	partitioning algorithm. Partitions A and B are known to be non-conflicting.
870	Merge them into a single partition A. */
871
872	static void
873	union_stack_vars (size_t a, size_t b)
874	{
875	class stack_var *vb = &stack_vars[b];
876	bitmap_iterator bi;
877	unsigned u;
878
879	gcc_assert (stack_vars[b].next == EOC);
880	/* Add B to A's partition. */
881	stack_vars[b].next = stack_vars[a].next;
882	stack_vars[b].representative = a;
883	stack_vars[a].next = b;
884
885	/* Make sure A is big enough to hold B. */
886	stack_vars[a].size = upper_bound (a: stack_vars[a].size, b: stack_vars[b].size);
887
888	/* Update the required alignment of partition A to account for B. */
889	if (stack_vars[a].alignb < stack_vars[b].alignb)
890	stack_vars[a].alignb = stack_vars[b].alignb;
891
892	/* Update the interference graph and merge the conflicts. */
893	if (vb->conflicts)
894	{
895	EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
896	add_stack_var_conflict (x: a, y: stack_vars[u].representative);
897	BITMAP_FREE (vb->conflicts);
898	}
899	}
900
901	/* A subroutine of expand_used_vars. Binpack the variables into
902	partitions constrained by the interference graph. The overall
903	algorithm used is as follows:
904
905	Sort the objects by size in descending order.
906	For each object A {
907	S = size(A)
908	O = 0
909	loop {
910	Look for the largest non-conflicting object B with size <= S.
911	UNION (A, B)
912	}
913	}
914	*/
915
916	static void
917	partition_stack_vars (void)
918	{
919	size_t si, sj, n = stack_vars_num;
920
921	stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
922	for (si = 0; si < n; ++si)
923	stack_vars_sorted[si] = si;
924
925	if (n == 1)
926	return;
927
928	qsort (stack_vars_sorted, n, sizeof (size_t), stack_var_cmp);
929
930	for (si = 0; si < n; ++si)
931	{
932	size_t i = stack_vars_sorted[si];
933	unsigned int ialign = stack_vars[i].alignb;
934	poly_int64 isize = stack_vars[i].size;
935
936	/* Ignore objects that aren't partition representatives. If we
937	see a var that is not a partition representative, it must
938	have been merged earlier. */
939	if (stack_vars[i].representative != i)
940	continue;
941
942	for (sj = si + 1; sj < n; ++sj)
943	{
944	size_t j = stack_vars_sorted[sj];
945	unsigned int jalign = stack_vars[j].alignb;
946	poly_int64 jsize = stack_vars[j].size;
947
948	/* Ignore objects that aren't partition representatives. */
949	if (stack_vars[j].representative != j)
950	continue;
951
952	/* Do not mix objects of "small" (supported) alignment
953	and "large" (unsupported) alignment. */
954	if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
955	!= (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
956	break;
957
958	/* For Address Sanitizer do not mix objects with different
959	sizes, as the shorter vars wouldn't be adequately protected.
960	Don't do that for "large" (unsupported) alignment objects,
961	those aren't protected anyway. */
962	if (asan_sanitize_stack_p ()
963	&& maybe_ne (a: isize, b: jsize)
964	&& ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
965	break;
966
967	/* Ignore conflicting objects. */
968	if (stack_var_conflict_p (x: i, y: j))
969	continue;
970
971	/* UNION the objects, placing J at OFFSET. */
972	union_stack_vars (a: i, b: j);
973	}
974	}
975
976	update_alias_info_with_stack_vars ();
977	}
978
979	/* A debugging aid for expand_used_vars. Dump the generated partitions. */
980
981	static void
982	dump_stack_var_partition (void)
983	{
984	size_t si, i, j, n = stack_vars_num;
985
986	for (si = 0; si < n; ++si)
987	{
988	i = stack_vars_sorted[si];
989
990	/* Skip variables that aren't partition representatives, for now. */
991	if (stack_vars[i].representative != i)
992	continue;
993
994	fprintf (stream: dump_file, format: "Partition %lu: size ", (unsigned long) i);
995	print_dec (value: stack_vars[i].size, file: dump_file);
996	fprintf (stream: dump_file, format: " align %u\n", stack_vars[i].alignb);
997
998	for (j = i; j != EOC; j = stack_vars[j].next)
999	{
1000	fputc (c: '\t', stream: dump_file);
1001	print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1002	}
1003	fputc (c: '\n', stream: dump_file);
1004	}
1005	}
1006
1007	/* Assign rtl to DECL at BASE + OFFSET. */
1008
1009	static void
1010	expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1011	poly_int64 offset)
1012	{
1013	unsigned align;
1014	rtx x;
1015
1016	/* If this fails, we've overflowed the stack frame. Error nicely? */
1017	gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1018
1019	if (hwasan_sanitize_stack_p ())
1020	x = targetm.memtag.add_tag (base, offset,
1021	hwasan_current_frame_tag ());
1022	else
1023	x = plus_constant (Pmode, base, offset);
1024
1025	x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1026	? TYPE_MODE (TREE_TYPE (decl))
1027	: DECL_MODE (decl), x);
1028
1029	/* Set alignment we actually gave this decl if it isn't an SSA name.
1030	If it is we generate stack slots only accidentally so it isn't as
1031	important, we'll simply set the alignment directly on the MEM. */
1032
1033	if (stack_vars_base_reg_p (base))
1034	offset -= frame_phase;
1035	align = known_alignment (a: offset);
1036	align *= BITS_PER_UNIT;
1037	if (align == 0 || align > base_align)
1038	align = base_align;
1039
1040	if (TREE_CODE (decl) != SSA_NAME)
1041	{
1042	/* One would think that we could assert that we're not decreasing
1043	alignment here, but (at least) the i386 port does exactly this
1044	via the MINIMUM_ALIGNMENT hook. */
1045
1046	SET_DECL_ALIGN (decl, align);
1047	DECL_USER_ALIGN (decl) = 0;
1048	}
1049
1050	set_rtl (t: decl, x);
1051
1052	set_mem_align (x, align);
1053	}
1054
1055	class stack_vars_data
1056	{
1057	public:
1058	/* Vector of offset pairs, always end of some padding followed
1059	by start of the padding that needs Address Sanitizer protection.
1060	The vector is in reversed, highest offset pairs come first. */
1061	auto_vec<HOST_WIDE_INT> asan_vec;
1062
1063	/* Vector of partition representative decls in between the paddings. */
1064	auto_vec<tree> asan_decl_vec;
1065
1066	/* Base pseudo register for Address Sanitizer protected automatic vars. */
1067	rtx asan_base;
1068
1069	/* Alignment needed for the Address Sanitizer protected automatic vars. */
1070	unsigned int asan_alignb;
1071	};
1072
1073	/* A subroutine of expand_used_vars. Give each partition representative
1074	a unique location within the stack frame. Update each partition member
1075	with that location. */
1076	static void
1077	expand_stack_vars (bool (*pred) (size_t), class stack_vars_data *data)
1078	{
1079	size_t si, i, j, n = stack_vars_num;
1080	poly_uint64 large_size = 0, large_alloc = 0;
1081	rtx large_base = NULL;
1082	rtx large_untagged_base = NULL;
1083	unsigned large_align = 0;
1084	bool large_allocation_done = false;
1085	tree decl;
1086
1087	/* Determine if there are any variables requiring "large" alignment.
1088	Since these are dynamically allocated, we only process these if
1089	no predicate involved. */
1090	large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1091	if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1092	{
1093	/* Find the total size of these variables. */
1094	for (si = 0; si < n; ++si)
1095	{
1096	unsigned alignb;
1097
1098	i = stack_vars_sorted[si];
1099	alignb = stack_vars[i].alignb;
1100
1101	/* All "large" alignment decls come before all "small" alignment
1102	decls, but "large" alignment decls are not sorted based on
1103	their alignment. Increase large_align to track the largest
1104	required alignment. */
1105	if ((alignb * BITS_PER_UNIT) > large_align)
1106	large_align = alignb * BITS_PER_UNIT;
1107
1108	/* Stop when we get to the first decl with "small" alignment. */
1109	if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1110	break;
1111
1112	/* Skip variables that aren't partition representatives. */
1113	if (stack_vars[i].representative != i)
1114	continue;
1115
1116	/* Skip variables that have already had rtl assigned. See also
1117	add_stack_var where we perpetrate this pc_rtx hack. */
1118	decl = stack_vars[i].decl;
1119	if (TREE_CODE (decl) == SSA_NAME
1120	? SA.partition_to_pseudo[var_to_partition (map: SA.map, var: decl)] != NULL_RTX
1121	: DECL_RTL (decl) != pc_rtx)
1122	continue;
1123
1124	large_size = aligned_upper_bound (value: large_size, align: alignb);
1125	large_size += stack_vars[i].size;
1126	}
1127	}
1128
1129	for (si = 0; si < n; ++si)
1130	{
1131	rtx base;
1132	unsigned base_align, alignb;
1133	poly_int64 offset = 0;
1134
1135	i = stack_vars_sorted[si];
1136
1137	/* Skip variables that aren't partition representatives, for now. */
1138	if (stack_vars[i].representative != i)
1139	continue;
1140
1141	/* Skip variables that have already had rtl assigned. See also
1142	add_stack_var where we perpetrate this pc_rtx hack. */
1143	decl = stack_vars[i].decl;
1144	if (TREE_CODE (decl) == SSA_NAME
1145	? SA.partition_to_pseudo[var_to_partition (map: SA.map, var: decl)] != NULL_RTX
1146	: DECL_RTL (decl) != pc_rtx)
1147	continue;
1148
1149	/* Check the predicate to see whether this variable should be
1150	allocated in this pass. */
1151	if (pred && !pred (i))
1152	continue;
1153
1154	base = (hwasan_sanitize_stack_p ()
1155	? hwasan_frame_base ()
1156	: virtual_stack_vars_rtx);
1157	alignb = stack_vars[i].alignb;
1158	if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1159	{
1160	poly_int64 hwasan_orig_offset;
1161	if (hwasan_sanitize_stack_p ())
1162	{
1163	/* There must be no tag granule "shared" between different
1164	objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1165	chunk can have more than one object in it.
1166
1167	We ensure this by forcing the end of the last bit of data to
1168	be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1169	the start of each variable to be aligned to
1170	HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1171
1172	We can't align just one of the start or end, since there are
1173	untagged things stored on the stack which we do not align to
1174	HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1175	or the end of tagged objects then untagged objects could end
1176	up sharing the first granule of a tagged object or sharing the
1177	last granule of a tagged object respectively. */
1178	hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1179	gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1180	}
1181	/* ASAN description strings don't yet have a syntax for expressing
1182	polynomial offsets. */
1183	HOST_WIDE_INT prev_offset;
1184	if (asan_sanitize_stack_p ()
1185	&& pred
1186	&& frame_offset.is_constant (const_value: &prev_offset)
1187	&& stack_vars[i].size.is_constant ())
1188	{
1189	if (data->asan_vec.is_empty ())
1190	{
1191	align_frame_offset (ASAN_RED_ZONE_SIZE);
1192	prev_offset = frame_offset.to_constant ();
1193	}
1194	prev_offset = align_base (base: prev_offset,
1195	ASAN_MIN_RED_ZONE_SIZE,
1196	align_up: !FRAME_GROWS_DOWNWARD);
1197	tree repr_decl = NULL_TREE;
1198	unsigned HOST_WIDE_INT size
1199	= asan_var_and_redzone_size (size: stack_vars[i].size.to_constant ());
1200	if (data->asan_vec.is_empty ())
1201	size = MAX (size, ASAN_RED_ZONE_SIZE);
1202
1203	unsigned HOST_WIDE_INT alignment = MAX (alignb,
1204	ASAN_MIN_RED_ZONE_SIZE);
1205	offset = alloc_stack_frame_space (size, align: alignment);
1206
1207	data->asan_vec.safe_push (obj: prev_offset);
1208	/* Allocating a constant amount of space from a constant
1209	starting offset must give a constant result. */
1210	data->asan_vec.safe_push (obj: (offset + stack_vars[i].size)
1211	.to_constant ());
1212	/* Find best representative of the partition.
1213	Prefer those with DECL_NAME, even better
1214	satisfying asan_protect_stack_decl predicate. */
1215	for (j = i; j != EOC; j = stack_vars[j].next)
1216	if (asan_protect_stack_decl (decl: stack_vars[j].decl)
1217	&& DECL_NAME (stack_vars[j].decl))
1218	{
1219	repr_decl = stack_vars[j].decl;
1220	break;
1221	}
1222	else if (repr_decl == NULL_TREE
1223	&& DECL_P (stack_vars[j].decl)
1224	&& DECL_NAME (stack_vars[j].decl))
1225	repr_decl = stack_vars[j].decl;
1226	if (repr_decl == NULL_TREE)
1227	repr_decl = stack_vars[i].decl;
1228	data->asan_decl_vec.safe_push (obj: repr_decl);
1229
1230	/* Make sure a representative is unpoison if another
1231	variable in the partition is handled by
1232	use-after-scope sanitization. */
1233	if (asan_handled_variables != NULL
1234	&& !asan_handled_variables->contains (k: repr_decl))
1235	{
1236	for (j = i; j != EOC; j = stack_vars[j].next)
1237	if (asan_handled_variables->contains (k: stack_vars[j].decl))
1238	break;
1239	if (j != EOC)
1240	asan_handled_variables->add (k: repr_decl);
1241	}
1242
1243	data->asan_alignb = MAX (data->asan_alignb, alignb);
1244	if (data->asan_base == NULL)
1245	data->asan_base = gen_reg_rtx (Pmode);
1246	base = data->asan_base;
1247
1248	if (!STRICT_ALIGNMENT)
1249	base_align = crtl->max_used_stack_slot_alignment;
1250	else
1251	base_align = MAX (crtl->max_used_stack_slot_alignment,
1252	GET_MODE_ALIGNMENT (SImode)
1253	<< ASAN_SHADOW_SHIFT);
1254	}
1255	else
1256	{
1257	offset = alloc_stack_frame_space (size: stack_vars[i].size, align: alignb);
1258	base_align = crtl->max_used_stack_slot_alignment;
1259
1260	if (hwasan_sanitize_stack_p ())
1261	{
1262	/* Align again since the point of this alignment is to handle
1263	the "end" of the object (i.e. smallest address after the
1264	stack object). For FRAME_GROWS_DOWNWARD that requires
1265	aligning the stack before allocating, but for a frame that
1266	grows upwards that requires aligning the stack after
1267	allocation.
1268
1269	Use `frame_offset` to record the offset value rather than
1270	`offset` since the `frame_offset` describes the extent
1271	allocated for this particular variable while `offset`
1272	describes the address that this variable starts at. */
1273	align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1274	hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1275	hwasan_orig_offset, frame_offset);
1276	}
1277	}
1278	}
1279	else
1280	{
1281	/* Large alignment is only processed in the last pass. */
1282	if (pred)
1283	continue;
1284
1285	/* If there were any variables requiring "large" alignment, allocate
1286	space. */
1287	if (maybe_ne (a: large_size, b: 0U) && ! large_allocation_done)
1288	{
1289	poly_int64 loffset;
1290	rtx large_allocsize;
1291
1292	large_allocsize = gen_int_mode (large_size, Pmode);
1293	get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1294	loffset = alloc_stack_frame_space
1295	(size: rtx_to_poly_int64 (x: large_allocsize),
1296	PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1297	large_base = get_dynamic_stack_base (loffset, large_align, base);
1298	large_allocation_done = true;
1299	}
1300
1301	gcc_assert (large_base != NULL);
1302	large_alloc = aligned_upper_bound (value: large_alloc, align: alignb);
1303	offset = large_alloc;
1304	large_alloc += stack_vars[i].size;
1305	if (hwasan_sanitize_stack_p ())
1306	{
1307	/* An object with a large alignment requirement means that the
1308	alignment requirement is greater than the required alignment
1309	for tags. */
1310	if (!large_untagged_base)
1311	large_untagged_base
1312	= targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1313	/* Ensure the end of the variable is also aligned correctly. */
1314	poly_int64 align_again
1315	= aligned_upper_bound (value: large_alloc, HWASAN_TAG_GRANULE_SIZE);
1316	/* For large allocations we always allocate a chunk of space
1317	(which is addressed by large_untagged_base/large_base) and
1318	then use positive offsets from that. Hence the farthest
1319	offset is `align_again` and the nearest offset from the base
1320	is `offset`. */
1321	hwasan_record_stack_var (large_untagged_base, large_base,
1322	offset, align_again);
1323	}
1324
1325	base = large_base;
1326	base_align = large_align;
1327	}
1328
1329	/* Create rtl for each variable based on their location within the
1330	partition. */
1331	for (j = i; j != EOC; j = stack_vars[j].next)
1332	{
1333	expand_one_stack_var_at (decl: stack_vars[j].decl,
1334	base, base_align, offset);
1335	}
1336	if (hwasan_sanitize_stack_p ())
1337	hwasan_increment_frame_tag ();
1338	}
1339
1340	gcc_assert (known_eq (large_alloc, large_size));
1341	}
1342
1343	/* Take into account all sizes of partitions and reset DECL_RTLs. */
1344	static poly_uint64
1345	account_stack_vars (void)
1346	{
1347	size_t si, j, i, n = stack_vars_num;
1348	poly_uint64 size = 0;
1349
1350	for (si = 0; si < n; ++si)
1351	{
1352	i = stack_vars_sorted[si];
1353
1354	/* Skip variables that aren't partition representatives, for now. */
1355	if (stack_vars[i].representative != i)
1356	continue;
1357
1358	size += stack_vars[i].size;
1359	for (j = i; j != EOC; j = stack_vars[j].next)
1360	set_rtl (t: stack_vars[j].decl, NULL);
1361	}
1362	return size;
1363	}
1364
1365	/* Record the RTL assignment X for the default def of PARM. */
1366
1367	extern void
1368	set_parm_rtl (tree parm, rtx x)
1369	{
1370	gcc_assert (TREE_CODE (parm) == PARM_DECL
1371	|| TREE_CODE (parm) == RESULT_DECL);
1372
1373	if (x && !MEM_P (x))
1374	{
1375	unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1376	TYPE_MODE (TREE_TYPE (parm)),
1377	TYPE_ALIGN (TREE_TYPE (parm)));
1378
1379	/* If the variable alignment is very large we'll dynamicaly
1380	allocate it, which means that in-frame portion is just a
1381	pointer. ??? We've got a pseudo for sure here, do we
1382	actually dynamically allocate its spilling area if needed?
1383	??? Isn't it a problem when Pmode alignment also exceeds
1384	MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1385	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1386	align = GET_MODE_ALIGNMENT (Pmode);
1387
1388	record_alignment_for_reg_var (align);
1389	}
1390
1391	tree ssa = ssa_default_def (cfun, parm);
1392	if (!ssa)
1393	return set_rtl (t: parm, x);
1394
1395	int part = var_to_partition (map: SA.map, var: ssa);
1396	gcc_assert (part != NO_PARTITION);
1397
1398	bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1399	gcc_assert (changed);
1400
1401	set_rtl (t: ssa, x);
1402	gcc_assert (DECL_RTL (parm) == x);
1403	}
1404
1405	/* A subroutine of expand_one_var. Called to immediately assign rtl
1406	to a variable to be allocated in the stack frame. */
1407
1408	static void
1409	expand_one_stack_var_1 (tree var)
1410	{
1411	poly_uint64 size;
1412	poly_int64 offset;
1413	unsigned byte_align;
1414
1415	if (TREE_CODE (var) == SSA_NAME)
1416	{
1417	tree type = TREE_TYPE (var);
1418	size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1419	}
1420	else
1421	size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1422
1423	byte_align = align_local_variable (decl: var, really_expand: true);
1424
1425	/* We handle highly aligned variables in expand_stack_vars. */
1426	gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1427
1428	rtx base;
1429	if (hwasan_sanitize_stack_p ())
1430	{
1431	/* Allocate zero bytes to align the stack. */
1432	poly_int64 hwasan_orig_offset
1433	= align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1434	offset = alloc_stack_frame_space (size, align: byte_align);
1435	align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1436	base = hwasan_frame_base ();
1437	/* Use `frame_offset` to automatically account for machines where the
1438	frame grows upwards.
1439
1440	`offset` will always point to the "start" of the stack object, which
1441	will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1442	the "furthest" offset from the base delimiting the current stack
1443	object. `frame_offset` will always delimit the extent that the frame.
1444	*/
1445	hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1446	hwasan_orig_offset, frame_offset);
1447	}
1448	else
1449	{
1450	offset = alloc_stack_frame_space (size, align: byte_align);
1451	base = virtual_stack_vars_rtx;
1452	}
1453
1454	expand_one_stack_var_at (decl: var, base,
1455	crtl->max_used_stack_slot_alignment, offset);
1456
1457	if (hwasan_sanitize_stack_p ())
1458	hwasan_increment_frame_tag ();
1459	}
1460
1461	/* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1462	already assigned some MEM. */
1463
1464	static void
1465	expand_one_stack_var (tree var)
1466	{
1467	if (TREE_CODE (var) == SSA_NAME)
1468	{
1469	int part = var_to_partition (map: SA.map, var);
1470	if (part != NO_PARTITION)
1471	{
1472	rtx x = SA.partition_to_pseudo[part];
1473	gcc_assert (x);
1474	gcc_assert (MEM_P (x));
1475	return;
1476	}
1477	}
1478
1479	return expand_one_stack_var_1 (var);
1480	}
1481
1482	/* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1483	that will reside in a hard register. */
1484
1485	static void
1486	expand_one_hard_reg_var (tree var)
1487	{
1488	rest_of_decl_compilation (var, 0, 0);
1489	}
1490
1491	/* Record the alignment requirements of some variable assigned to a
1492	pseudo. */
1493
1494	static void
1495	record_alignment_for_reg_var (unsigned int align)
1496	{
1497	if (SUPPORTS_STACK_ALIGNMENT
1498	&& crtl->stack_alignment_estimated < align)
1499	{
1500	/* stack_alignment_estimated shouldn't change after stack
1501	realign decision made */
1502	gcc_assert (!crtl->stack_realign_processed);
1503	crtl->stack_alignment_estimated = align;
1504	}
1505
1506	/* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1507	So here we only make sure stack_alignment_needed >= align. */
1508	if (crtl->stack_alignment_needed < align)
1509	crtl->stack_alignment_needed = align;
1510	if (crtl->max_used_stack_slot_alignment < align)
1511	crtl->max_used_stack_slot_alignment = align;
1512	}
1513
1514	/* Create RTL for an SSA partition. */
1515
1516	static void
1517	expand_one_ssa_partition (tree var)
1518	{
1519	int part = var_to_partition (map: SA.map, var);
1520	gcc_assert (part != NO_PARTITION);
1521
1522	if (SA.partition_to_pseudo[part])
1523	return;
1524
1525	unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1526	TYPE_MODE (TREE_TYPE (var)),
1527	TYPE_ALIGN (TREE_TYPE (var)));
1528
1529	/* If the variable alignment is very large we'll dynamicaly allocate
1530	it, which means that in-frame portion is just a pointer. */
1531	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1532	align = GET_MODE_ALIGNMENT (Pmode);
1533
1534	record_alignment_for_reg_var (align);
1535
1536	if (!use_register_for_decl (var))
1537	{
1538	if (defer_stack_allocation (var, true))
1539	add_stack_var (decl: var, really_expand: true);
1540	else
1541	expand_one_stack_var_1 (var);
1542	return;
1543	}
1544
1545	machine_mode reg_mode = promote_ssa_mode (var, NULL);
1546	rtx x = gen_reg_rtx (reg_mode);
1547
1548	set_rtl (t: var, x);
1549
1550	/* For a promoted variable, X will not be used directly but wrapped in a
1551	SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1552	will assume that its upper bits can be inferred from its lower bits.
1553	Therefore, if X isn't initialized on every path from the entry, then
1554	we must do it manually in order to fulfill the above assumption. */
1555	if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1556	&& bitmap_bit_p (SA.partitions_for_undefined_values, part))
1557	emit_move_insn (x, CONST0_RTX (reg_mode));
1558	}
1559
1560	/* Record the association between the RTL generated for partition PART
1561	and the underlying variable of the SSA_NAME VAR. */
1562
1563	static void
1564	adjust_one_expanded_partition_var (tree var)
1565	{
1566	if (!var)
1567	return;
1568
1569	tree decl = SSA_NAME_VAR (var);
1570
1571	int part = var_to_partition (map: SA.map, var);
1572	if (part == NO_PARTITION)
1573	return;
1574
1575	rtx x = SA.partition_to_pseudo[part];
1576
1577	gcc_assert (x);
1578
1579	set_rtl (t: var, x);
1580
1581	if (!REG_P (x))
1582	return;
1583
1584	/* Note if the object is a user variable. */
1585	if (decl && !DECL_ARTIFICIAL (decl))
1586	mark_user_reg (x);
1587
1588	if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1589	mark_reg_pointer (x, get_pointer_alignment (var));
1590	}
1591
1592	/* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1593	that will reside in a pseudo register. */
1594
1595	static void
1596	expand_one_register_var (tree var)
1597	{
1598	if (TREE_CODE (var) == SSA_NAME)
1599	{
1600	int part = var_to_partition (map: SA.map, var);
1601	if (part != NO_PARTITION)
1602	{
1603	rtx x = SA.partition_to_pseudo[part];
1604	gcc_assert (x);
1605	gcc_assert (REG_P (x));
1606	return;
1607	}
1608	gcc_unreachable ();
1609	}
1610
1611	tree decl = var;
1612	tree type = TREE_TYPE (decl);
1613	machine_mode reg_mode = promote_decl_mode (decl, NULL);
1614	rtx x = gen_reg_rtx (reg_mode);
1615
1616	set_rtl (t: var, x);
1617
1618	/* Note if the object is a user variable. */
1619	if (!DECL_ARTIFICIAL (decl))
1620	mark_user_reg (x);
1621
1622	if (POINTER_TYPE_P (type))
1623	mark_reg_pointer (x, get_pointer_alignment (var));
1624	}
1625
1626	/* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1627	has some associated error, e.g. its type is error-mark. We just need
1628	to pick something that won't crash the rest of the compiler. */
1629
1630	static void
1631	expand_one_error_var (tree var)
1632	{
1633	machine_mode mode = DECL_MODE (var);
1634	rtx x;
1635
1636	if (mode == BLKmode)
1637	x = gen_rtx_MEM (BLKmode, const0_rtx);
1638	else if (mode == VOIDmode)
1639	x = const0_rtx;
1640	else
1641	x = gen_reg_rtx (mode);
1642
1643	SET_DECL_RTL (var, x);
1644	}
1645
1646	/* A subroutine of expand_one_var. VAR is a variable that will be
1647	allocated to the local stack frame. Return true if we wish to
1648	add VAR to STACK_VARS so that it will be coalesced with other
1649	variables. Return false to allocate VAR immediately.
1650
1651	This function is used to reduce the number of variables considered
1652	for coalescing, which reduces the size of the quadratic problem. */
1653
1654	static bool
1655	defer_stack_allocation (tree var, bool toplevel)
1656	{
1657	tree size_unit = TREE_CODE (var) == SSA_NAME
1658	? TYPE_SIZE_UNIT (TREE_TYPE (var))
1659	: DECL_SIZE_UNIT (var);
1660	poly_uint64 size;
1661
1662	/* Whether the variable is small enough for immediate allocation not to be
1663	a problem with regard to the frame size. */
1664	bool smallish
1665	= (poly_int_tree_p (t: size_unit, value: &size)
1666	&& (estimated_poly_value (x: size)
1667	< param_min_size_for_stack_sharing));
1668
1669	/* If stack protection is enabled, *all* stack variables must be deferred,
1670	so that we can re-order the strings to the top of the frame.
1671	Similarly for Address Sanitizer. */
1672	if (flag_stack_protect || asan_sanitize_stack_p ())
1673	return true;
1674
1675	unsigned int align = TREE_CODE (var) == SSA_NAME
1676	? TYPE_ALIGN (TREE_TYPE (var))
1677	: DECL_ALIGN (var);
1678
1679	/* We handle "large" alignment via dynamic allocation. We want to handle
1680	this extra complication in only one place, so defer them. */
1681	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1682	return true;
1683
1684	bool ignored = TREE_CODE (var) == SSA_NAME
1685	? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
1686	: DECL_IGNORED_P (var);
1687
1688	/* When optimization is enabled, DECL_IGNORED_P variables originally scoped
1689	might be detached from their block and appear at toplevel when we reach
1690	here. We want to coalesce them with variables from other blocks when
1691	the immediate contribution to the frame size would be noticeable. */
1692	if (toplevel && optimize > 0 && ignored && !smallish)
1693	return true;
1694
1695	/* Variables declared in the outermost scope automatically conflict
1696	with every other variable. The only reason to want to defer them
1697	at all is that, after sorting, we can more efficiently pack
1698	small variables in the stack frame. Continue to defer at -O2. */
1699	if (toplevel && optimize < 2)
1700	return false;
1701
1702	/* Without optimization, *most* variables are allocated from the
1703	stack, which makes the quadratic problem large exactly when we
1704	want compilation to proceed as quickly as possible. On the
1705	other hand, we don't want the function's stack frame size to
1706	get completely out of hand. So we avoid adding scalars and
1707	"small" aggregates to the list at all. */
1708	if (optimize == 0 && smallish)
1709	return false;
1710
1711	return true;
1712	}
1713
1714	/* A subroutine of expand_used_vars. Expand one variable according to
1715	its flavor. Variables to be placed on the stack are not actually
1716	expanded yet, merely recorded.
1717	When REALLY_EXPAND is false, only add stack values to be allocated.
1718	Return stack usage this variable is supposed to take.
1719	*/
1720
1721	static poly_uint64
1722	expand_one_var (tree var, bool toplevel, bool really_expand,
1723	bitmap forced_stack_var = NULL)
1724	{
1725	unsigned int align = BITS_PER_UNIT;
1726	tree origvar = var;
1727
1728	var = SSAVAR (var);
1729
1730	if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
1731	{
1732	if (is_global_var (t: var))
1733	return 0;
1734
1735	/* Because we don't know if VAR will be in register or on stack,
1736	we conservatively assume it will be on stack even if VAR is
1737	eventually put into register after RA pass. For non-automatic
1738	variables, which won't be on stack, we collect alignment of
1739	type and ignore user specified alignment. Similarly for
1740	SSA_NAMEs for which use_register_for_decl returns true. */
1741	if (TREE_STATIC (var)
1742	|| DECL_EXTERNAL (var)
1743	|| (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
1744	align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1745	TYPE_MODE (TREE_TYPE (var)),
1746	TYPE_ALIGN (TREE_TYPE (var)));
1747	else if (DECL_HAS_VALUE_EXPR_P (var)
1748	|| (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
1749	/* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
1750	or variables which were assigned a stack slot already by
1751	expand_one_stack_var_at - in the latter case DECL_ALIGN has been
1752	changed from the offset chosen to it. */
1753	align = crtl->stack_alignment_estimated;
1754	else
1755	align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
1756
1757	/* If the variable alignment is very large we'll dynamicaly allocate
1758	it, which means that in-frame portion is just a pointer. */
1759	if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1760	align = GET_MODE_ALIGNMENT (Pmode);
1761	}
1762
1763	record_alignment_for_reg_var (align);
1764
1765	poly_uint64 size;
1766	if (TREE_CODE (origvar) == SSA_NAME)
1767	{
1768	gcc_assert (!VAR_P (var)
1769	|| (!DECL_EXTERNAL (var)
1770	&& !DECL_HAS_VALUE_EXPR_P (var)
1771	&& !TREE_STATIC (var)
1772	&& TREE_TYPE (var) != error_mark_node
1773	&& !DECL_HARD_REGISTER (var)
1774	&& really_expand));
1775	}
1776	if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
1777	;
1778	else if (DECL_EXTERNAL (var))
1779	;
1780	else if (DECL_HAS_VALUE_EXPR_P (var))
1781	;
1782	else if (TREE_STATIC (var))
1783	;
1784	else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
1785	;
1786	else if (TREE_TYPE (var) == error_mark_node)
1787	{
1788	if (really_expand)
1789	expand_one_error_var (var);
1790	}
1791	else if (VAR_P (var) && DECL_HARD_REGISTER (var))
1792	{
1793	if (really_expand)
1794	{
1795	expand_one_hard_reg_var (var);
1796	if (!DECL_HARD_REGISTER (var))
1797	/* Invalid register specification. */
1798	expand_one_error_var (var);
1799	}
1800	}
1801	else if (use_register_for_decl (var)
1802	&& (!forced_stack_var
1803	|| !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
1804	{
1805	if (really_expand)
1806	expand_one_register_var (var: origvar);
1807	}
1808	else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), value: &size)
1809	|| !valid_constant_size_p (DECL_SIZE_UNIT (var)))
1810	{
1811	/* Reject variables which cover more than half of the address-space. */
1812	if (really_expand)
1813	{
1814	if (DECL_NONLOCAL_FRAME (var))
1815	error_at (DECL_SOURCE_LOCATION (current_function_decl),
1816	"total size of local objects is too large");
1817	else
1818	error_at (DECL_SOURCE_LOCATION (var),
1819	"size of variable %q+D is too large", var);
1820	expand_one_error_var (var);
1821	}
1822	}
1823	else if (defer_stack_allocation (var, toplevel))
1824	add_stack_var (decl: origvar, really_expand);
1825	else
1826	{
1827	if (really_expand)
1828	{
1829	if (lookup_attribute (attr_name: "naked",
1830	DECL_ATTRIBUTES (current_function_decl)))
1831	error ("cannot allocate stack for variable %q+D, naked function",
1832	var);
1833
1834	expand_one_stack_var (var: origvar);
1835	}
1836	return size;
1837	}
1838	return 0;
1839	}
1840
1841	/* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1842	expanding variables. Those variables that can be put into registers
1843	are allocated pseudos; those that can't are put on the stack.
1844
1845	TOPLEVEL is true if this is the outermost BLOCK. */
1846
1847	static void
1848	expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
1849	{
1850	tree t;
1851
1852	/* Expand all variables at this level. */
1853	for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1854	if (TREE_USED (t)
1855	&& ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1856	|| !DECL_NONSHAREABLE (t)))
1857	expand_one_var (var: t, toplevel, really_expand: true, forced_stack_var: forced_stack_vars);
1858
1859	/* Expand all variables at containing levels. */
1860	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1861	expand_used_vars_for_block (block: t, toplevel: false, forced_stack_vars);
1862	}
1863
1864	/* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1865	and clear TREE_USED on all local variables. */
1866
1867	static void
1868	clear_tree_used (tree block)
1869	{
1870	tree t;
1871
1872	for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1873	/* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
1874	if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1875	|| !DECL_NONSHAREABLE (t))
1876	TREE_USED (t) = 0;
1877
1878	for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1879	clear_tree_used (block: t);
1880	}
1881
1882	/* Examine TYPE and determine a bit mask of the following features. */
1883
1884	#define SPCT_HAS_LARGE_CHAR_ARRAY 1
1885	#define SPCT_HAS_SMALL_CHAR_ARRAY 2
1886	#define SPCT_HAS_ARRAY 4
1887	#define SPCT_HAS_AGGREGATE 8
1888
1889	static unsigned int
1890	stack_protect_classify_type (tree type)
1891	{
1892	unsigned int ret = 0;
1893	tree t;
1894
1895	switch (TREE_CODE (type))
1896	{
1897	case ARRAY_TYPE:
1898	t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
1899	if (t == char_type_node
1900	|| t == signed_char_type_node
1901	|| t == unsigned_char_type_node)
1902	{
1903	unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
1904	unsigned HOST_WIDE_INT len;
1905
1906	if (!TYPE_SIZE_UNIT (type)
1907	|| !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
1908	len = max;
1909	else
1910	len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
1911
1912	if (len < max)
1913	ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
1914	else
1915	ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
1916	}
1917	else
1918	ret = SPCT_HAS_ARRAY;
1919	break;
1920
1921	case UNION_TYPE:
1922	case QUAL_UNION_TYPE:
1923	case RECORD_TYPE:
1924	ret = SPCT_HAS_AGGREGATE;
1925	for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
1926	if (TREE_CODE (t) == FIELD_DECL)
1927	ret |= stack_protect_classify_type (TREE_TYPE (t));
1928	break;
1929
1930	default:
1931	break;
1932	}
1933
1934	return ret;
1935	}
1936
1937	/* Return nonzero if DECL should be segregated into the "vulnerable" upper
1938	part of the local stack frame. Remember if we ever return nonzero for
1939	any variable in this function. The return value is the phase number in
1940	which the variable should be allocated. */
1941
1942	static int
1943	stack_protect_decl_phase (tree decl)
1944	{
1945	unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
1946	int ret = 0;
1947
1948	if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
1949	has_short_buffer = true;
1950
1951	tree attribs = DECL_ATTRIBUTES (current_function_decl);
1952	if (!lookup_attribute (attr_name: "no_stack_protector", list: attribs)
1953	&& (flag_stack_protect == SPCT_FLAG_ALL
1954	|| flag_stack_protect == SPCT_FLAG_STRONG
1955	|| (flag_stack_protect == SPCT_FLAG_EXPLICIT
1956	&& lookup_attribute (attr_name: "stack_protect", list: attribs))))
1957	{
1958	if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
1959	&& !(bits & SPCT_HAS_AGGREGATE))
1960	ret = 1;
1961	else if (bits & SPCT_HAS_ARRAY)
1962	ret = 2;
1963	}
1964	else
1965	ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
1966
1967	if (ret)
1968	has_protected_decls = true;
1969
1970	return ret;
1971	}
1972
1973	/* Two helper routines that check for phase 1 and phase 2. These are used
1974	as callbacks for expand_stack_vars. */
1975
1976	static bool
1977	stack_protect_decl_phase_1 (size_t i)
1978	{
1979	return stack_protect_decl_phase (decl: stack_vars[i].decl) == 1;
1980	}
1981
1982	static bool
1983	stack_protect_decl_phase_2 (size_t i)
1984	{
1985	return stack_protect_decl_phase (decl: stack_vars[i].decl) == 2;
1986	}
1987
1988	/* And helper function that checks for asan phase (with stack protector
1989	it is phase 3). This is used as callback for expand_stack_vars.
1990	Returns true if any of the vars in the partition need to be protected. */
1991
1992	static bool
1993	asan_decl_phase_3 (size_t i)
1994	{
1995	while (i != EOC)
1996	{
1997	if (asan_protect_stack_decl (decl: stack_vars[i].decl))
1998	return true;
1999	i = stack_vars[i].next;
2000	}
2001	return false;
2002	}
2003
2004	/* Ensure that variables in different stack protection phases conflict
2005	so that they are not merged and share the same stack slot.
2006	Return true if there are any address taken variables. */
2007
2008	static bool
2009	add_stack_protection_conflicts (void)
2010	{
2011	size_t i, j, n = stack_vars_num;
2012	unsigned char *phase;
2013	bool ret = false;
2014
2015	phase = XNEWVEC (unsigned char, n);
2016	for (i = 0; i < n; ++i)
2017	{
2018	phase[i] = stack_protect_decl_phase (decl: stack_vars[i].decl);
2019	if (TREE_ADDRESSABLE (stack_vars[i].decl))
2020	ret = true;
2021	}
2022
2023	for (i = 0; i < n; ++i)
2024	{
2025	unsigned char ph_i = phase[i];
2026	for (j = i + 1; j < n; ++j)
2027	if (ph_i != phase[j])
2028	add_stack_var_conflict (x: i, y: j);
2029	}
2030
2031	XDELETEVEC (phase);
2032	return ret;
2033	}
2034
2035	/* Create a decl for the guard at the top of the stack frame. */
2036
2037	static void
2038	create_stack_guard (void)
2039	{
2040	tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2041	VAR_DECL, NULL, ptr_type_node);
2042	TREE_THIS_VOLATILE (guard) = 1;
2043	TREE_USED (guard) = 1;
2044	expand_one_stack_var (var: guard);
2045	crtl->stack_protect_guard = guard;
2046	}
2047
2048	/* Prepare for expanding variables. */
2049	static void
2050	init_vars_expansion (void)
2051	{
2052	/* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2053	bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2054
2055	/* A map from decl to stack partition. */
2056	decl_to_stack_part = new hash_map<tree, size_t>;
2057
2058	/* Initialize local stack smashing state. */
2059	has_protected_decls = false;
2060	has_short_buffer = false;
2061	if (hwasan_sanitize_stack_p ())
2062	hwasan_record_frame_init ();
2063	}
2064
2065	/* Free up stack variable graph data. */
2066	static void
2067	fini_vars_expansion (void)
2068	{
2069	bitmap_obstack_release (&stack_var_bitmap_obstack);
2070	if (stack_vars)
2071	XDELETEVEC (stack_vars);
2072	if (stack_vars_sorted)
2073	XDELETEVEC (stack_vars_sorted);
2074	stack_vars = NULL;
2075	stack_vars_sorted = NULL;
2076	stack_vars_alloc = stack_vars_num = 0;
2077	delete decl_to_stack_part;
2078	decl_to_stack_part = NULL;
2079	}
2080
2081	/* Make a fair guess for the size of the stack frame of the function
2082	in NODE. This doesn't have to be exact, the result is only used in
2083	the inline heuristics. So we don't want to run the full stack var
2084	packing algorithm (which is quadratic in the number of stack vars).
2085	Instead, we calculate the total size of all stack vars. This turns
2086	out to be a pretty fair estimate -- packing of stack vars doesn't
2087	happen very often. */
2088
2089	HOST_WIDE_INT
2090	estimated_stack_frame_size (struct cgraph_node *node)
2091	{
2092	poly_int64 size = 0;
2093	size_t i;
2094	tree var;
2095	struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2096
2097	push_cfun (new_cfun: fn);
2098
2099	init_vars_expansion ();
2100
2101	FOR_EACH_LOCAL_DECL (fn, i, var)
2102	if (auto_var_in_fn_p (var, fn->decl))
2103	size += expand_one_var (var, toplevel: true, really_expand: false);
2104
2105	if (stack_vars_num > 0)
2106	{
2107	/* Fake sorting the stack vars for account_stack_vars (). */
2108	stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
2109	for (i = 0; i < stack_vars_num; ++i)
2110	stack_vars_sorted[i] = i;
2111	size += account_stack_vars ();
2112	}
2113
2114	fini_vars_expansion ();
2115	pop_cfun ();
2116	return estimated_poly_value (x: size);
2117	}
2118
2119	/* Check if the current function has calls that use a return slot. */
2120
2121	static bool
2122	stack_protect_return_slot_p ()
2123	{
2124	basic_block bb;
2125
2126	FOR_ALL_BB_FN (bb, cfun)
2127	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2128	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
2129	{
2130	gimple *stmt = gsi_stmt (i: gsi);
2131	/* This assumes that calls to internal-only functions never
2132	use a return slot. */
2133	if (is_gimple_call (gs: stmt)
2134	&& !gimple_call_internal_p (gs: stmt)
2135	&& aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2136	gimple_call_fndecl (gs: stmt)))
2137	return true;
2138	}
2139	return false;
2140	}
2141
2142	/* Expand all variables used in the function. */
2143
2144	static rtx_insn *
2145	expand_used_vars (bitmap forced_stack_vars)
2146	{
2147	tree var, outer_block = DECL_INITIAL (current_function_decl);
2148	auto_vec<tree> maybe_local_decls;
2149	rtx_insn *var_end_seq = NULL;
2150	unsigned i;
2151	unsigned len;
2152	bool gen_stack_protect_signal = false;
2153
2154	/* Compute the phase of the stack frame for this function. */
2155	{
2156	int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2157	int off = targetm.starting_frame_offset () % align;
2158	frame_phase = off ? align - off : 0;
2159	}
2160
2161	/* Set TREE_USED on all variables in the local_decls. */
2162	FOR_EACH_LOCAL_DECL (cfun, i, var)
2163	TREE_USED (var) = 1;
2164	/* Clear TREE_USED on all variables associated with a block scope. */
2165	clear_tree_used (DECL_INITIAL (current_function_decl));
2166
2167	init_vars_expansion ();
2168
2169	if (targetm.use_pseudo_pic_reg ())
2170	pic_offset_table_rtx = gen_reg_rtx (Pmode);
2171
2172	for (i = 0; i < SA.map->num_partitions; i++)
2173	{
2174	if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2175	continue;
2176
2177	tree var = partition_to_var (map: SA.map, i);
2178
2179	gcc_assert (!virtual_operand_p (var));
2180
2181	expand_one_ssa_partition (var);
2182	}
2183
2184	if (flag_stack_protect == SPCT_FLAG_STRONG)
2185	gen_stack_protect_signal = stack_protect_return_slot_p ();
2186
2187	/* At this point all variables on the local_decls with TREE_USED
2188	set are not associated with any block scope. Lay them out. */
2189
2190	len = vec_safe_length (cfun->local_decls);
2191	FOR_EACH_LOCAL_DECL (cfun, i, var)
2192	{
2193	bool expand_now = false;
2194
2195	/* Expanded above already. */
2196	if (is_gimple_reg (var))
2197	{
2198	TREE_USED (var) = 0;
2199	goto next;
2200	}
2201	/* We didn't set a block for static or extern because it's hard
2202	to tell the difference between a global variable (re)declared
2203	in a local scope, and one that's really declared there to
2204	begin with. And it doesn't really matter much, since we're
2205	not giving them stack space. Expand them now. */
2206	else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2207	expand_now = true;
2208
2209	/* Expand variables not associated with any block now. Those created by
2210	the optimizers could be live anywhere in the function. Those that
2211	could possibly have been scoped originally and detached from their
2212	block will have their allocation deferred so we coalesce them with
2213	others when optimization is enabled. */
2214	else if (TREE_USED (var))
2215	expand_now = true;
2216
2217	/* Finally, mark all variables on the list as used. We'll use
2218	this in a moment when we expand those associated with scopes. */
2219	TREE_USED (var) = 1;
2220
2221	if (expand_now)
2222	expand_one_var (var, toplevel: true, really_expand: true, forced_stack_var: forced_stack_vars);
2223
2224	next:
2225	if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2226	{
2227	rtx rtl = DECL_RTL_IF_SET (var);
2228
2229	/* Keep artificial non-ignored vars in cfun->local_decls
2230	chain until instantiate_decls. */
2231	if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2232	add_local_decl (cfun, d: var);
2233	else if (rtl == NULL_RTX)
2234	/* If rtl isn't set yet, which can happen e.g. with
2235	-fstack-protector, retry before returning from this
2236	function. */
2237	maybe_local_decls.safe_push (obj: var);
2238	}
2239	}
2240
2241	/* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2242
2243	+-----------------+-----------------+
2244	| ...processed... | ...duplicates...|
2245	+-----------------+-----------------+
2246	^
2247	+-- LEN points here.
2248
2249	We just want the duplicates, as those are the artificial
2250	non-ignored vars that we want to keep until instantiate_decls.
2251	Move them down and truncate the array. */
2252	if (!vec_safe_is_empty (cfun->local_decls))
2253	cfun->local_decls->block_remove (ix: 0, len);
2254
2255	/* At this point, all variables within the block tree with TREE_USED
2256	set are actually used by the optimized function. Lay them out. */
2257	expand_used_vars_for_block (block: outer_block, toplevel: true, forced_stack_vars);
2258
2259	tree attribs = DECL_ATTRIBUTES (current_function_decl);
2260	if (stack_vars_num > 0)
2261	{
2262	bool has_addressable_vars = false;
2263
2264	add_scope_conflicts ();
2265
2266	/* If stack protection is enabled, we don't share space between
2267	vulnerable data and non-vulnerable data. */
2268	if (flag_stack_protect != 0
2269	&& !lookup_attribute (attr_name: "no_stack_protector", list: attribs)
2270	&& (flag_stack_protect != SPCT_FLAG_EXPLICIT
2271	|| (flag_stack_protect == SPCT_FLAG_EXPLICIT
2272	&& lookup_attribute (attr_name: "stack_protect", list: attribs))))
2273	has_addressable_vars = add_stack_protection_conflicts ();
2274
2275	if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2276	gen_stack_protect_signal = true;
2277
2278	/* Now that we have collected all stack variables, and have computed a
2279	minimal interference graph, attempt to save some stack space. */
2280	partition_stack_vars ();
2281	if (dump_file)
2282	dump_stack_var_partition ();
2283	}
2284
2285
2286	if (!lookup_attribute (attr_name: "no_stack_protector", list: attribs))
2287	switch (flag_stack_protect)
2288	{
2289	case SPCT_FLAG_ALL:
2290	create_stack_guard ();
2291	break;
2292
2293	case SPCT_FLAG_STRONG:
2294	if (gen_stack_protect_signal
2295	|| cfun->calls_alloca
2296	|| has_protected_decls
2297	|| lookup_attribute (attr_name: "stack_protect", list: attribs))
2298	create_stack_guard ();
2299	break;
2300
2301	case SPCT_FLAG_DEFAULT:
2302	if (cfun->calls_alloca
2303	|| has_protected_decls
2304	|| lookup_attribute (attr_name: "stack_protect", list: attribs))
2305	create_stack_guard ();
2306	break;
2307
2308	case SPCT_FLAG_EXPLICIT:
2309	if (lookup_attribute (attr_name: "stack_protect", list: attribs))
2310	create_stack_guard ();
2311	break;
2312
2313	default:
2314	break;
2315	}
2316
2317	/* Assign rtl to each variable based on these partitions. */
2318	if (stack_vars_num > 0)
2319	{
2320	class stack_vars_data data;
2321
2322	data.asan_base = NULL_RTX;
2323	data.asan_alignb = 0;
2324
2325	/* Reorder decls to be protected by iterating over the variables
2326	array multiple times, and allocating out of each phase in turn. */
2327	/* ??? We could probably integrate this into the qsort we did
2328	earlier, such that we naturally see these variables first,
2329	and thus naturally allocate things in the right order. */
2330	if (has_protected_decls)
2331	{
2332	/* Phase 1 contains only character arrays. */
2333	expand_stack_vars (pred: stack_protect_decl_phase_1, data: &data);
2334
2335	/* Phase 2 contains other kinds of arrays. */
2336	if (!lookup_attribute (attr_name: "no_stack_protector", list: attribs)
2337	&& (flag_stack_protect == SPCT_FLAG_ALL
2338	|| flag_stack_protect == SPCT_FLAG_STRONG
2339	|| (flag_stack_protect == SPCT_FLAG_EXPLICIT
2340	&& lookup_attribute (attr_name: "stack_protect", list: attribs))))
2341	expand_stack_vars (pred: stack_protect_decl_phase_2, data: &data);
2342	}
2343
2344	if (asan_sanitize_stack_p ())
2345	/* Phase 3, any partitions that need asan protection
2346	in addition to phase 1 and 2. */
2347	expand_stack_vars (pred: asan_decl_phase_3, data: &data);
2348
2349	/* ASAN description strings don't yet have a syntax for expressing
2350	polynomial offsets. */
2351	HOST_WIDE_INT prev_offset;
2352	if (!data.asan_vec.is_empty ()
2353	&& frame_offset.is_constant (const_value: &prev_offset))
2354	{
2355	HOST_WIDE_INT offset, sz, redzonesz;
2356	redzonesz = ASAN_RED_ZONE_SIZE;
2357	sz = data.asan_vec[0] - prev_offset;
2358	if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2359	&& data.asan_alignb <= 4096
2360	&& sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2361	redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2362	& ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2363	/* Allocating a constant amount of space from a constant
2364	starting offset must give a constant result. */
2365	offset = (alloc_stack_frame_space (size: redzonesz, ASAN_RED_ZONE_SIZE)
2366	.to_constant ());
2367	data.asan_vec.safe_push (obj: prev_offset);
2368	data.asan_vec.safe_push (obj: offset);
2369	/* Leave space for alignment if STRICT_ALIGNMENT. */
2370	if (STRICT_ALIGNMENT)
2371	alloc_stack_frame_space (size: (GET_MODE_ALIGNMENT (SImode)
2372	<< ASAN_SHADOW_SHIFT)
2373	/ BITS_PER_UNIT, align: 1);
2374
2375	var_end_seq
2376	= asan_emit_stack_protection (virtual_stack_vars_rtx,
2377	data.asan_base,
2378	data.asan_alignb,
2379	data.asan_vec.address (),
2380	data.asan_decl_vec.address (),
2381	data.asan_vec.length ());
2382	}
2383
2384	expand_stack_vars (NULL, data: &data);
2385	}
2386
2387	if (hwasan_sanitize_stack_p ())
2388	hwasan_emit_prologue ();
2389	if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2390	var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2391	virtual_stack_vars_rtx,
2392	var_end_seq);
2393	else if (hwasan_sanitize_allocas_p () && cfun->calls_alloca)
2394	/* When using out-of-line instrumentation we only want to emit one function
2395	call for clearing the tags in a region of shadow stack. When there are
2396	alloca calls in this frame we want to emit a call using the
2397	virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2398	rtx we created in expand_stack_vars. */
2399	var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2400	virtual_stack_vars_rtx);
2401	else if (hwasan_sanitize_stack_p ())
2402	/* If no variables were stored on the stack, `hwasan_get_frame_extent`
2403	will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2404	NULL (i.e. an empty sequence). */
2405	var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2406	virtual_stack_vars_rtx);
2407
2408	fini_vars_expansion ();
2409
2410	/* If there were any artificial non-ignored vars without rtl
2411	found earlier, see if deferred stack allocation hasn't assigned
2412	rtl to them. */
2413	FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2414	{
2415	rtx rtl = DECL_RTL_IF_SET (var);
2416
2417	/* Keep artificial non-ignored vars in cfun->local_decls
2418	chain until instantiate_decls. */
2419	if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2420	add_local_decl (cfun, d: var);
2421	}
2422
2423	/* If the target requires that FRAME_OFFSET be aligned, do it. */
2424	if (STACK_ALIGNMENT_NEEDED)
2425	{
2426	HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2427	if (FRAME_GROWS_DOWNWARD)
2428	frame_offset = aligned_lower_bound (frame_offset, align);
2429	else
2430	frame_offset = aligned_upper_bound (frame_offset, align);
2431	}
2432
2433	return var_end_seq;
2434	}
2435
2436
2437	/* If we need to produce a detailed dump, print the tree representation
2438	for STMT to the dump file. SINCE is the last RTX after which the RTL
2439	generated for STMT should have been appended. */
2440
2441	static void
2442	maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2443	{
2444	if (dump_file && (dump_flags & TDF_DETAILS))
2445	{
2446	fprintf (stream: dump_file, format: "\n;; ");
2447	print_gimple_stmt (dump_file, stmt, 0,
2448	TDF_SLIM | (dump_flags & TDF_LINENO));
2449	fprintf (stream: dump_file, format: "\n");
2450
2451	print_rtl (dump_file, since ? NEXT_INSN (insn: since) : since);
2452	}
2453	}
2454
2455	/* Maps the blocks that do not contain tree labels to rtx labels. */
2456
2457	static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2458
2459	/* Returns the label_rtx expression for a label starting basic block BB. */
2460
2461	static rtx_code_label *
2462	label_rtx_for_bb (basic_block bb ATTRIBUTE_UNUSED)
2463	{
2464	if (bb->flags & BB_RTL)
2465	return block_label (bb);
2466
2467	rtx_code_label **elt = lab_rtx_for_bb->get (k: bb);
2468	if (elt)
2469	return *elt;
2470
2471	/* Find the tree label if it is present. */
2472	gimple_stmt_iterator gsi = gsi_start_bb (bb);
2473	glabel *lab_stmt;
2474	if (!gsi_end_p (i: gsi)
2475	&& (lab_stmt = dyn_cast <glabel *> (p: gsi_stmt (i: gsi)))
2476	&& !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
2477	return jump_target_rtx (gimple_label_label (gs: lab_stmt));
2478
2479	rtx_code_label *l = gen_label_rtx ();
2480	lab_rtx_for_bb->put (k: bb, v: l);
2481	return l;
2482	}
2483
2484
2485	/* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2486	of a basic block where we just expanded the conditional at the end,
2487	possibly clean up the CFG and instruction sequence. LAST is the
2488	last instruction before the just emitted jump sequence. */
2489
2490	static void
2491	maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2492	{
2493	/* Special case: when jumpif decides that the condition is
2494	trivial it emits an unconditional jump (and the necessary
2495	barrier). But we still have two edges, the fallthru one is
2496	wrong. purge_dead_edges would clean this up later. Unfortunately
2497	we have to insert insns (and split edges) before
2498	find_many_sub_basic_blocks and hence before purge_dead_edges.
2499	But splitting edges might create new blocks which depend on the
2500	fact that if there are two edges there's no barrier. So the
2501	barrier would get lost and verify_flow_info would ICE. Instead
2502	of auditing all edge splitters to care for the barrier (which
2503	normally isn't there in a cleaned CFG), fix it here. */
2504	if (BARRIER_P (get_last_insn ()))
2505	{
2506	rtx_insn *insn;
2507	remove_edge (e);
2508	/* Now, we have a single successor block, if we have insns to
2509	insert on the remaining edge we potentially will insert
2510	it at the end of this block (if the dest block isn't feasible)
2511	in order to avoid splitting the edge. This insertion will take
2512	place in front of the last jump. But we might have emitted
2513	multiple jumps (conditional and one unconditional) to the
2514	same destination. Inserting in front of the last one then
2515	is a problem. See PR 40021. We fix this by deleting all
2516	jumps except the last unconditional one. */
2517	insn = PREV_INSN (insn: get_last_insn ());
2518	/* Make sure we have an unconditional jump. Otherwise we're
2519	confused. */
2520	gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2521	for (insn = PREV_INSN (insn); insn != last;)
2522	{
2523	insn = PREV_INSN (insn);
2524	if (JUMP_P (NEXT_INSN (insn)))
2525	{
2526	if (!any_condjump_p (NEXT_INSN (insn)))
2527	{
2528	gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2529	delete_insn (NEXT_INSN (insn: NEXT_INSN (insn)));
2530	}
2531	delete_insn (NEXT_INSN (insn));
2532	}
2533	}
2534	}
2535	}
2536
2537	/* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2538	Returns a new basic block if we've terminated the current basic
2539	block and created a new one. */
2540
2541	static basic_block
2542	expand_gimple_cond (basic_block bb, gcond *stmt)
2543	{
2544	basic_block new_bb, dest;
2545	edge true_edge;
2546	edge false_edge;
2547	rtx_insn *last2, *last;
2548	enum tree_code code;
2549	tree op0, op1;
2550
2551	code = gimple_cond_code (gs: stmt);
2552	op0 = gimple_cond_lhs (gs: stmt);
2553	op1 = gimple_cond_rhs (gs: stmt);
2554	/* We're sometimes presented with such code:
2555	D.123_1 = x < y;
2556	if (D.123_1 != 0)
2557	...
2558	This would expand to two comparisons which then later might
2559	be cleaned up by combine. But some pattern matchers like if-conversion
2560	work better when there's only one compare, so make up for this
2561	here as special exception if TER would have made the same change. */
2562	if (SA.values
2563	&& TREE_CODE (op0) == SSA_NAME
2564	&& TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2565	&& TREE_CODE (op1) == INTEGER_CST
2566	&& ((gimple_cond_code (gs: stmt) == NE_EXPR
2567	&& integer_zerop (op1))
2568	|| (gimple_cond_code (gs: stmt) == EQ_EXPR
2569	&& integer_onep (op1)))
2570	&& bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2571	{
2572	gimple *second = SSA_NAME_DEF_STMT (op0);
2573	if (gimple_code (g: second) == GIMPLE_ASSIGN)
2574	{
2575	enum tree_code code2 = gimple_assign_rhs_code (gs: second);
2576	if (TREE_CODE_CLASS (code2) == tcc_comparison)
2577	{
2578	code = code2;
2579	op0 = gimple_assign_rhs1 (gs: second);
2580	op1 = gimple_assign_rhs2 (gs: second);
2581	}
2582	/* If jumps are cheap and the target does not support conditional
2583	compare, turn some more codes into jumpy sequences. */
2584	else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2585	&& targetm.gen_ccmp_first == NULL)
2586	{
2587	if ((code2 == BIT_AND_EXPR
2588	&& TYPE_PRECISION (TREE_TYPE (op0)) == 1
2589	&& TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
2590	|| code2 == TRUTH_AND_EXPR)
2591	{
2592	code = TRUTH_ANDIF_EXPR;
2593	op0 = gimple_assign_rhs1 (gs: second);
2594	op1 = gimple_assign_rhs2 (gs: second);
2595	}
2596	else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
2597	{
2598	code = TRUTH_ORIF_EXPR;
2599	op0 = gimple_assign_rhs1 (gs: second);
2600	op1 = gimple_assign_rhs2 (gs: second);
2601	}
2602	}
2603	}
2604	}
2605
2606	/* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
2607	into (x - C2) * C3 < C4. */
2608	if ((code == EQ_EXPR || code == NE_EXPR)
2609	&& TREE_CODE (op0) == SSA_NAME
2610	&& TREE_CODE (op1) == INTEGER_CST)
2611	code = maybe_optimize_mod_cmp (code, &op0, &op1);
2612
2613	/* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
2614	if (!TYPE_UNSIGNED (TREE_TYPE (op0))
2615	&& (code == LT_EXPR || code == LE_EXPR
2616	|| code == GT_EXPR || code == GE_EXPR)
2617	&& integer_zerop (op1)
2618	&& TREE_CODE (op0) == SSA_NAME)
2619	maybe_optimize_sub_cmp_0 (code, &op0, &op1);
2620
2621	last2 = last = get_last_insn ();
2622
2623	extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
2624	set_curr_insn_location (gimple_location (g: stmt));
2625
2626	/* These flags have no purpose in RTL land. */
2627	true_edge->flags &= ~EDGE_TRUE_VALUE;
2628	false_edge->flags &= ~EDGE_FALSE_VALUE;
2629
2630	/* We can either have a pure conditional jump with one fallthru edge or
2631	two-way jump that needs to be decomposed into two basic blocks. */
2632	if (false_edge->dest == bb->next_bb)
2633	{
2634	jumpif_1 (code, op0, op1, label_rtx_for_bb (bb: true_edge->dest),
2635	true_edge->probability);
2636	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
2637	if (true_edge->goto_locus != UNKNOWN_LOCATION)
2638	set_curr_insn_location (true_edge->goto_locus);
2639	false_edge->flags |= EDGE_FALLTHRU;
2640	maybe_cleanup_end_of_block (e: false_edge, last);
2641	return NULL;
2642	}
2643	if (true_edge->dest == bb->next_bb)
2644	{
2645	jumpifnot_1 (code, op0, op1, label_rtx_for_bb (bb: false_edge->dest),
2646	false_edge->probability);
2647	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
2648	if (false_edge->goto_locus != UNKNOWN_LOCATION)
2649	set_curr_insn_location (false_edge->goto_locus);
2650	true_edge->flags |= EDGE_FALLTHRU;
2651	maybe_cleanup_end_of_block (e: true_edge, last);
2652	return NULL;
2653	}
2654
2655	jumpif_1 (code, op0, op1, label_rtx_for_bb (bb: true_edge->dest),
2656	true_edge->probability);
2657	last = get_last_insn ();
2658	if (false_edge->goto_locus != UNKNOWN_LOCATION)
2659	set_curr_insn_location (false_edge->goto_locus);
2660	emit_jump (label_rtx_for_bb (bb: false_edge->dest));
2661
2662	BB_END (bb) = last;
2663	if (BARRIER_P (BB_END (bb)))
2664	BB_END (bb) = PREV_INSN (BB_END (bb));
2665	update_bb_for_insn (bb);
2666
2667	new_bb = create_basic_block (NEXT_INSN (insn: last), get_last_insn (), bb);
2668	dest = false_edge->dest;
2669	redirect_edge_succ (false_edge, new_bb);
2670	false_edge->flags |= EDGE_FALLTHRU;
2671	new_bb->count = false_edge->count ();
2672	loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
2673	add_bb_to_loop (new_bb, loop);
2674	if (loop->latch == bb
2675	&& loop->header == dest)
2676	loop->latch = new_bb;
2677	make_single_succ_edge (new_bb, dest, 0);
2678	if (BARRIER_P (BB_END (new_bb)))
2679	BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
2680	update_bb_for_insn (new_bb);
2681
2682	maybe_dump_rtl_for_gimple_stmt (stmt, since: last2);
2683
2684	if (true_edge->goto_locus != UNKNOWN_LOCATION)
2685	{
2686	set_curr_insn_location (true_edge->goto_locus);
2687	true_edge->goto_locus = curr_insn_location ();
2688	}
2689
2690	return new_bb;
2691	}
2692
2693	/* Mark all calls that can have a transaction restart. */
2694
2695	static void
2696	mark_transaction_restart_calls (gimple *stmt)
2697	{
2698	struct tm_restart_node dummy;
2699	tm_restart_node **slot;
2700
2701	if (!cfun->gimple_df->tm_restart)
2702	return;
2703
2704	dummy.stmt = stmt;
2705	slot = cfun->gimple_df->tm_restart->find_slot (value: &dummy, insert: NO_INSERT);
2706	if (slot)
2707	{
2708	struct tm_restart_node *n = *slot;
2709	tree list = n->label_or_list;
2710	rtx_insn *insn;
2711
2712	for (insn = next_real_insn (get_last_insn ());
2713	!CALL_P (insn);
2714	insn = next_real_insn (insn))
2715	continue;
2716
2717	if (TREE_CODE (list) == LABEL_DECL)
2718	add_reg_note (insn, REG_TM, label_rtx (list));
2719	else
2720	for (; list ; list = TREE_CHAIN (list))
2721	add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
2722	}
2723	}
2724
2725	/* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
2726	statement STMT. */
2727
2728	static void
2729	expand_call_stmt (gcall *stmt)
2730	{
2731	tree exp, decl, lhs;
2732	bool builtin_p;
2733	size_t i;
2734
2735	if (gimple_call_internal_p (gs: stmt))
2736	{
2737	expand_internal_call (stmt);
2738	return;
2739	}
2740
2741	/* If this is a call to a built-in function and it has no effect other
2742	than setting the lhs, try to implement it using an internal function
2743	instead. */
2744	decl = gimple_call_fndecl (gs: stmt);
2745	if (gimple_call_lhs (gs: stmt)
2746	&& !gimple_has_side_effects (stmt)
2747	&& (optimize || (decl && called_as_built_in (decl))))
2748	{
2749	internal_fn ifn = replacement_internal_fn (stmt);
2750	if (ifn != IFN_LAST)
2751	{
2752	expand_internal_call (ifn, stmt);
2753	return;
2754	}
2755	}
2756
2757	exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (gs: stmt) + 3);
2758
2759	CALL_EXPR_FN (exp) = gimple_call_fn (gs: stmt);
2760	builtin_p = decl && fndecl_built_in_p (node: decl);
2761
2762	/* If this is not a builtin function, the function type through which the
2763	call is made may be different from the type of the function. */
2764	if (!builtin_p)
2765	CALL_EXPR_FN (exp)
2766	= fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
2767	CALL_EXPR_FN (exp));
2768
2769	TREE_TYPE (exp) = gimple_call_return_type (gs: stmt);
2770	CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (gs: stmt);
2771
2772	for (i = 0; i < gimple_call_num_args (gs: stmt); i++)
2773	{
2774	tree arg = gimple_call_arg (gs: stmt, index: i);
2775	gimple *def;
2776	/* TER addresses into arguments of builtin functions so we have a
2777	chance to infer more correct alignment information. See PR39954. */
2778	if (builtin_p
2779	&& TREE_CODE (arg) == SSA_NAME
2780	&& (def = get_gimple_for_ssa_name (exp: arg))
2781	&& gimple_assign_rhs_code (gs: def) == ADDR_EXPR)
2782	arg = gimple_assign_rhs1 (gs: def);
2783	CALL_EXPR_ARG (exp, i) = arg;
2784	}
2785
2786	if (gimple_has_side_effects (stmt)
2787	/* ??? Downstream in expand_expr_real_1 we assume that expressions
2788	w/o side-effects do not throw so work around this here. */
2789	|| stmt_could_throw_p (cfun, stmt))
2790	TREE_SIDE_EFFECTS (exp) = 1;
2791
2792	if (gimple_call_nothrow_p (s: stmt))
2793	TREE_NOTHROW (exp) = 1;
2794
2795	CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (s: stmt);
2796	CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (s: stmt);
2797	CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (s: stmt);
2798	if (decl
2799	&& fndecl_built_in_p (node: decl, klass: BUILT_IN_NORMAL)
2800	&& ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
2801	CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (s: stmt);
2802	else
2803	CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (s: stmt);
2804	CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (s: stmt);
2805	CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (s: stmt);
2806	SET_EXPR_LOCATION (exp, gimple_location (stmt));
2807
2808	/* Must come after copying location. */
2809	copy_warning (exp, stmt);
2810
2811	/* Ensure RTL is created for debug args. */
2812	if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
2813	{
2814	vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
2815	unsigned int ix;
2816	tree dtemp;
2817
2818	if (debug_args)
2819	for (ix = 1; (*debug_args)->iterate (ix, ptr: &dtemp); ix += 2)
2820	{
2821	gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
2822	expand_debug_expr (dtemp);
2823	}
2824	}
2825
2826	rtx_insn *before_call = get_last_insn ();
2827	lhs = gimple_call_lhs (gs: stmt);
2828	if (lhs)
2829	expand_assignment (lhs, exp, false);
2830	else
2831	expand_expr (exp, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
2832
2833	/* If the gimple call is an indirect call and has 'nocf_check'
2834	attribute find a generated CALL insn to mark it as no
2835	control-flow verification is needed. */
2836	if (gimple_call_nocf_check_p (gs: stmt)
2837	&& !gimple_call_fndecl (gs: stmt))
2838	{
2839	rtx_insn *last = get_last_insn ();
2840	while (!CALL_P (last)
2841	&& last != before_call)
2842	last = PREV_INSN (insn: last);
2843
2844	if (last != before_call)
2845	add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
2846	}
2847
2848	mark_transaction_restart_calls (stmt);
2849	}
2850
2851
2852	/* Generate RTL for an asm statement (explicit assembler code).
2853	STRING is a STRING_CST node containing the assembler code text,
2854	or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
2855	insn is volatile; don't optimize it. */
2856
2857	static void
2858	expand_asm_loc (tree string, int vol, location_t locus)
2859	{
2860	rtx body;
2861
2862	body = gen_rtx_ASM_INPUT_loc (VOIDmode,
2863	ggc_strdup (TREE_STRING_POINTER (string)),
2864	locus);
2865
2866	MEM_VOLATILE_P (body) = vol;
2867
2868	/* Non-empty basic ASM implicitly clobbers memory. */
2869	if (TREE_STRING_LENGTH (string) != 0)
2870	{
2871	rtx asm_op, clob;
2872	unsigned i, nclobbers;
2873	auto_vec<rtx> input_rvec, output_rvec;
2874	auto_vec<machine_mode> input_mode;
2875	auto_vec<const char *> constraints;
2876	auto_vec<rtx> clobber_rvec;
2877	HARD_REG_SET clobbered_regs;
2878	CLEAR_HARD_REG_SET (set&: clobbered_regs);
2879
2880	clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
2881	clobber_rvec.safe_push (obj: clob);
2882
2883	if (targetm.md_asm_adjust)
2884	targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
2885	constraints, clobber_rvec, clobbered_regs,
2886	locus);
2887
2888	asm_op = body;
2889	nclobbers = clobber_rvec.length ();
2890	body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nclobbers));
2891
2892	XVECEXP (body, 0, 0) = asm_op;
2893	for (i = 0; i < nclobbers; i++)
2894	XVECEXP (body, 0, i + 1) = gen_rtx_CLOBBER (VOIDmode, clobber_rvec[i]);
2895	}
2896
2897	emit_insn (body);
2898	}
2899
2900	/* Return the number of times character C occurs in string S. */
2901	static int
2902	n_occurrences (int c, const char *s)
2903	{
2904	int n = 0;
2905	while (*s)
2906	n += (*s++ == c);
2907	return n;
2908	}
2909
2910	/* A subroutine of expand_asm_operands. Check that all operands have
2911	the same number of alternatives. Return true if so. */
2912
2913	static bool
2914	check_operand_nalternatives (const vec<const char *> &constraints)
2915	{
2916	unsigned len = constraints.length();
2917	if (len > 0)
2918	{
2919	int nalternatives = n_occurrences (c: ',', s: constraints[0]);
2920
2921	if (nalternatives + 1 > MAX_RECOG_ALTERNATIVES)
2922	{
2923	error ("too many alternatives in %<asm%>");
2924	return false;
2925	}
2926
2927	for (unsigned i = 1; i < len; ++i)
2928	if (n_occurrences (c: ',', s: constraints[i]) != nalternatives)
2929	{
2930	error ("operand constraints for %<asm%> differ "
2931	"in number of alternatives");
2932	return false;
2933	}
2934	}
2935	return true;
2936	}
2937
2938	/* Check for overlap between registers marked in CLOBBERED_REGS and
2939	anything inappropriate in T. Emit error and return the register
2940	variable definition for error, NULL_TREE for ok. */
2941
2942	static bool
2943	tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
2944	location_t loc)
2945	{
2946	/* Conflicts between asm-declared register variables and the clobber
2947	list are not allowed. */
2948	tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
2949
2950	if (overlap)
2951	{
2952	error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
2953	"%<asm%> clobber list", DECL_NAME (overlap));
2954
2955	/* Reset registerness to stop multiple errors emitted for a single
2956	variable. */
2957	DECL_REGISTER (overlap) = 0;
2958	return true;
2959	}
2960
2961	return false;
2962	}
2963
2964	/* Check that the given REGNO spanning NREGS is a valid
2965	asm clobber operand. Some HW registers cannot be
2966	saved/restored, hence they should not be clobbered by
2967	asm statements. */
2968	static bool
2969	asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
2970	{
2971	bool is_valid = true;
2972	HARD_REG_SET regset;
2973
2974	CLEAR_HARD_REG_SET (set&: regset);
2975
2976	add_range_to_hard_reg_set (regs: &regset, regno, nregs);
2977
2978	/* Clobbering the PIC register is an error. */
2979	if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
2980	&& overlaps_hard_reg_set_p (regs: regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
2981	{
2982	/* ??? Diagnose during gimplification? */
2983	error ("PIC register clobbered by %qs in %<asm%>", regname);
2984	is_valid = false;
2985	}
2986	else if (!in_hard_reg_set_p
2987	(accessible_reg_set, reg_raw_mode[regno], regno))
2988	{
2989	/* ??? Diagnose during gimplification? */
2990	error ("the register %qs cannot be clobbered in %<asm%>"
2991	" for the current target", regname);
2992	is_valid = false;
2993	}
2994
2995	/* Clobbering the stack pointer register is deprecated. GCC expects
2996	the value of the stack pointer after an asm statement to be the same
2997	as it was before, so no asm can validly clobber the stack pointer in
2998	the usual sense. Adding the stack pointer to the clobber list has
2999	traditionally had some undocumented and somewhat obscure side-effects. */
3000	if (overlaps_hard_reg_set_p (regs: regset, Pmode, STACK_POINTER_REGNUM))
3001	{
3002	crtl->sp_is_clobbered_by_asm = true;
3003	if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3004	" %qs in a clobber list is deprecated", regname))
3005	inform (input_location, "the value of the stack pointer after"
3006	" an %<asm%> statement must be the same as it was before"
3007	" the statement");
3008	}
3009
3010	return is_valid;
3011	}
3012
3013	/* Generate RTL for an asm statement with arguments.
3014	STRING is the instruction template.
3015	OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3016	Each output or input has an expression in the TREE_VALUE and
3017	a tree list in TREE_PURPOSE which in turn contains a constraint
3018	name in TREE_VALUE (or NULL_TREE) and a constraint string
3019	in TREE_PURPOSE.
3020	CLOBBERS is a list of STRING_CST nodes each naming a hard register
3021	that is clobbered by this insn.
3022
3023	LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3024	should be the fallthru basic block of the asm goto.
3025
3026	Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3027	Some elements of OUTPUTS may be replaced with trees representing temporary
3028	values. The caller should copy those temporary values to the originally
3029	specified lvalues.
3030
3031	VOL nonzero means the insn is volatile; don't optimize it. */
3032
3033	static void
3034	expand_asm_stmt (gasm *stmt)
3035	{
3036	class save_input_location
3037	{
3038	location_t old;
3039
3040	public:
3041	explicit save_input_location(location_t where)
3042	{
3043	old = input_location;
3044	input_location = where;
3045	}
3046
3047	~save_input_location()
3048	{
3049	input_location = old;
3050	}
3051	};
3052
3053	location_t locus = gimple_location (g: stmt);
3054
3055	if (gimple_asm_input_p (asm_stmt: stmt))
3056	{
3057	const char *s = gimple_asm_string (asm_stmt: stmt);
3058	tree string = build_string (strlen (s: s), s);
3059	expand_asm_loc (string, vol: gimple_asm_volatile_p (asm_stmt: stmt), locus);
3060	return;
3061	}
3062
3063	/* There are some legacy diagnostics in here. */
3064	save_input_location s_i_l(locus);
3065
3066	unsigned noutputs = gimple_asm_noutputs (asm_stmt: stmt);
3067	unsigned ninputs = gimple_asm_ninputs (asm_stmt: stmt);
3068	unsigned nlabels = gimple_asm_nlabels (asm_stmt: stmt);
3069	unsigned i;
3070	bool error_seen = false;
3071
3072	/* ??? Diagnose during gimplification? */
3073	if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3074	{
3075	error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3076	return;
3077	}
3078
3079	auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3080	auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3081	auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3082
3083	/* Copy the gimple vectors into new vectors that we can manipulate. */
3084
3085	output_tvec.safe_grow (len: noutputs, exact: true);
3086	input_tvec.safe_grow (len: ninputs, exact: true);
3087	constraints.safe_grow (len: noutputs + ninputs, exact: true);
3088
3089	for (i = 0; i < noutputs; ++i)
3090	{
3091	tree t = gimple_asm_output_op (asm_stmt: stmt, index: i);
3092	output_tvec[i] = TREE_VALUE (t);
3093	constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3094	}
3095	for (i = 0; i < ninputs; i++)
3096	{
3097	tree t = gimple_asm_input_op (asm_stmt: stmt, index: i);
3098	input_tvec[i] = TREE_VALUE (t);
3099	constraints[i + noutputs]
3100	= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3101	}
3102
3103	/* ??? Diagnose during gimplification? */
3104	if (! check_operand_nalternatives (constraints))
3105	return;
3106
3107	/* Count the number of meaningful clobbered registers, ignoring what
3108	we would ignore later. */
3109	auto_vec<rtx> clobber_rvec;
3110	HARD_REG_SET clobbered_regs;
3111	CLEAR_HARD_REG_SET (set&: clobbered_regs);
3112
3113	if (unsigned n = gimple_asm_nclobbers (asm_stmt: stmt))
3114	{
3115	clobber_rvec.reserve (nelems: n);
3116	for (i = 0; i < n; i++)
3117	{
3118	tree t = gimple_asm_clobber_op (asm_stmt: stmt, index: i);
3119	const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3120	int nregs, j;
3121
3122	j = decode_reg_name_and_count (regname, &nregs);
3123	if (j < 0)
3124	{
3125	if (j == -2)
3126	{
3127	/* ??? Diagnose during gimplification? */
3128	error_at (locus, "unknown register name %qs in %<asm%>",
3129	regname);
3130	error_seen = true;
3131	}
3132	else if (j == -4)
3133	{
3134	rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3135	clobber_rvec.safe_push (obj: x);
3136	}
3137	else
3138	{
3139	/* Otherwise we should have -1 == empty string
3140	or -3 == cc, which is not a register. */
3141	gcc_assert (j == -1 || j == -3);
3142	}
3143	}
3144	else
3145	for (int reg = j; reg < j + nregs; reg++)
3146	{
3147	if (!asm_clobber_reg_is_valid (regno: reg, nregs, regname))
3148	return;
3149
3150	SET_HARD_REG_BIT (set&: clobbered_regs, bit: reg);
3151	rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3152	clobber_rvec.safe_push (obj: x);
3153	}
3154	}
3155	}
3156
3157	/* First pass over inputs and outputs checks validity and sets
3158	mark_addressable if needed. */
3159	/* ??? Diagnose during gimplification? */
3160
3161	for (i = 0; i < noutputs; ++i)
3162	{
3163	tree val = output_tvec[i];
3164	tree type = TREE_TYPE (val);
3165	const char *constraint;
3166	bool is_inout;
3167	bool allows_reg;
3168	bool allows_mem;
3169
3170	/* Try to parse the output constraint. If that fails, there's
3171	no point in going further. */
3172	constraint = constraints[i];
3173	if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3174	&allows_mem, &allows_reg, &is_inout))
3175	return;
3176
3177	/* If the output is a hard register, verify it doesn't conflict with
3178	any other operand's possible hard register use. */
3179	if (DECL_P (val)
3180	&& REG_P (DECL_RTL (val))
3181	&& HARD_REGISTER_P (DECL_RTL (val)))
3182	{
3183	unsigned j, output_hregno = REGNO (DECL_RTL (val));
3184	bool early_clobber_p = strchr (s: constraints[i], c: '&') != NULL;
3185	unsigned long match;
3186
3187	/* Verify the other outputs do not use the same hard register. */
3188	for (j = i + 1; j < noutputs; ++j)
3189	if (DECL_P (output_tvec[j])
3190	&& REG_P (DECL_RTL (output_tvec[j]))
3191	&& HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3192	&& output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3193	{
3194	error_at (locus, "invalid hard register usage between output "
3195	"operands");
3196	error_seen = true;
3197	}
3198
3199	/* Verify matching constraint operands use the same hard register
3200	and that the non-matching constraint operands do not use the same
3201	hard register if the output is an early clobber operand. */
3202	for (j = 0; j < ninputs; ++j)
3203	if (DECL_P (input_tvec[j])
3204	&& REG_P (DECL_RTL (input_tvec[j]))
3205	&& HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3206	{
3207	unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3208	switch (*constraints[j + noutputs])
3209	{
3210	case '0': case '1': case '2': case '3': case '4':
3211	case '5': case '6': case '7': case '8': case '9':
3212	match = strtoul (nptr: constraints[j + noutputs], NULL, base: 10);
3213	break;
3214	default:
3215	match = ULONG_MAX;
3216	break;
3217	}
3218	if (i == match
3219	&& output_hregno != input_hregno)
3220	{
3221	error_at (locus, "invalid hard register usage between "
3222	"output operand and matching constraint operand");
3223	error_seen = true;
3224	}
3225	else if (early_clobber_p
3226	&& i != match
3227	&& output_hregno == input_hregno)
3228	{
3229	error_at (locus, "invalid hard register usage between "
3230	"earlyclobber operand and input operand");
3231	error_seen = true;
3232	}
3233	}
3234	}
3235
3236	if (! allows_reg
3237	&& (allows_mem
3238	|| is_inout
3239	|| (DECL_P (val)
3240	&& REG_P (DECL_RTL (val))
3241	&& GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3242	mark_addressable (val);
3243	}
3244
3245	for (i = 0; i < ninputs; ++i)
3246	{
3247	bool allows_reg, allows_mem;
3248	const char *constraint;
3249
3250	constraint = constraints[i + noutputs];
3251	if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3252	constraints.address (),
3253	&allows_mem, &allows_reg))
3254	return;
3255
3256	if (! allows_reg && allows_mem)
3257	mark_addressable (input_tvec[i]);
3258	}
3259
3260	/* Second pass evaluates arguments. */
3261
3262	/* Make sure stack is consistent for asm goto. */
3263	if (nlabels > 0)
3264	do_pending_stack_adjust ();
3265	int old_generating_concat_p = generating_concat_p;
3266
3267	/* Vector of RTX's of evaluated output operands. */
3268	auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3269	auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3270	rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3271
3272	output_rvec.safe_grow (len: noutputs, exact: true);
3273
3274	for (i = 0; i < noutputs; ++i)
3275	{
3276	tree val = output_tvec[i];
3277	tree type = TREE_TYPE (val);
3278	bool is_inout, allows_reg, allows_mem, ok;
3279	rtx op;
3280
3281	ok = parse_output_constraint (&constraints[i], i, ninputs,
3282	noutputs, &allows_mem, &allows_reg,
3283	&is_inout);
3284	gcc_assert (ok);
3285
3286	/* If an output operand is not a decl or indirect ref and our constraint
3287	allows a register, make a temporary to act as an intermediate.
3288	Make the asm insn write into that, then we will copy it to
3289	the real output operand. Likewise for promoted variables. */
3290
3291	generating_concat_p = 0;
3292
3293	gcc_assert (TREE_CODE (val) != INDIRECT_REF);
3294	if (((TREE_CODE (val) == MEM_REF
3295	&& TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
3296	&& allows_mem)
3297	|| (DECL_P (val)
3298	&& (allows_mem || REG_P (DECL_RTL (val)))
3299	&& ! (REG_P (DECL_RTL (val))
3300	&& GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3301	|| ! allows_reg
3302	|| is_inout
3303	|| TREE_ADDRESSABLE (type)
3304	|| (!tree_fits_poly_int64_p (TYPE_SIZE (type))
3305	&& !known_size_p (a: max_int_size_in_bytes (type))))
3306	{
3307	op = expand_expr (exp: val, NULL_RTX, VOIDmode,
3308	modifier: !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3309	if (MEM_P (op))
3310	op = validize_mem (op);
3311
3312	if (! allows_reg && !MEM_P (op))
3313	{
3314	error_at (locus, "output number %d not directly addressable", i);
3315	error_seen = true;
3316	}
3317	if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3318	|| GET_CODE (op) == CONCAT)
3319	{
3320	rtx old_op = op;
3321	op = gen_reg_rtx (GET_MODE (op));
3322
3323	generating_concat_p = old_generating_concat_p;
3324
3325	if (is_inout)
3326	emit_move_insn (op, old_op);
3327
3328	push_to_sequence2 (after_rtl_seq, after_rtl_end);
3329	emit_move_insn (old_op, op);
3330	after_rtl_seq = get_insns ();
3331	after_rtl_end = get_last_insn ();
3332	end_sequence ();
3333	}
3334	}
3335	else
3336	{
3337	op = assign_temp (type, 0, 1);
3338	op = validize_mem (op);
3339	if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3340	set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), x: op);
3341
3342	generating_concat_p = old_generating_concat_p;
3343
3344	push_to_sequence2 (after_rtl_seq, after_rtl_end);
3345	expand_assignment (val, make_tree (type, op), false);
3346	after_rtl_seq = get_insns ();
3347	after_rtl_end = get_last_insn ();
3348	end_sequence ();
3349	}
3350	output_rvec[i] = op;
3351
3352	if (is_inout)
3353	inout_opnum.safe_push (obj: i);
3354	}
3355
3356	const char *str = gimple_asm_string (asm_stmt: stmt);
3357	if (error_seen)
3358	{
3359	ninputs = 0;
3360	noutputs = 0;
3361	inout_opnum.truncate (size: 0);
3362	output_rvec.truncate (size: 0);
3363	clobber_rvec.truncate (size: 0);
3364	constraints.truncate (size: 0);
3365	CLEAR_HARD_REG_SET (set&: clobbered_regs);
3366	str = "";
3367	}
3368
3369	auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3370	auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3371
3372	input_rvec.safe_grow (len: ninputs, exact: true);
3373	input_mode.safe_grow (len: ninputs, exact: true);
3374
3375	generating_concat_p = 0;
3376
3377	for (i = 0; i < ninputs; ++i)
3378	{
3379	tree val = input_tvec[i];
3380	tree type = TREE_TYPE (val);
3381	bool allows_reg, allows_mem, ok;
3382	const char *constraint;
3383	rtx op;
3384
3385	constraint = constraints[i + noutputs];
3386	ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3387	constraints.address (),
3388	&allows_mem, &allows_reg);
3389	gcc_assert (ok);
3390
3391	/* EXPAND_INITIALIZER will not generate code for valid initializer
3392	constants, but will still generate code for other types of operand.
3393	This is the behavior we want for constant constraints. */
3394	op = expand_expr (exp: val, NULL_RTX, VOIDmode,
3395	modifier: allows_reg ? EXPAND_NORMAL
3396	: allows_mem ? EXPAND_MEMORY
3397	: EXPAND_INITIALIZER);
3398
3399	/* Never pass a CONCAT to an ASM. */
3400	if (GET_CODE (op) == CONCAT)
3401	op = force_reg (GET_MODE (op), op);
3402	else if (MEM_P (op))
3403	op = validize_mem (op);
3404
3405	if (asm_operand_ok (op, constraint, NULL) <= 0)
3406	{
3407	if (allows_reg && TYPE_MODE (type) != BLKmode)
3408	op = force_reg (TYPE_MODE (type), op);
3409	else if (!allows_mem)
3410	warning_at (locus, 0, "%<asm%> operand %d probably does not match "
3411	"constraints", i + noutputs);
3412	else if (MEM_P (op))
3413	{
3414	/* We won't recognize either volatile memory or memory
3415	with a queued address as available a memory_operand
3416	at this point. Ignore it: clearly this *is* a memory. */
3417	}
3418	else
3419	gcc_unreachable ();
3420	}
3421	input_rvec[i] = op;
3422	input_mode[i] = TYPE_MODE (type);
3423	}
3424
3425	/* For in-out operands, copy output rtx to input rtx. */
3426	unsigned ninout = inout_opnum.length ();
3427	for (i = 0; i < ninout; i++)
3428	{
3429	int j = inout_opnum[i];
3430	rtx o = output_rvec[j];
3431
3432	input_rvec.safe_push (obj: o);
3433	input_mode.safe_push (GET_MODE (o));
3434
3435	char buffer[16];
3436	sprintf (s: buffer, format: "%d", j);
3437	constraints.safe_push (ggc_strdup (buffer));
3438	}
3439	ninputs += ninout;
3440
3441	/* Sometimes we wish to automatically clobber registers across an asm.
3442	Case in point is when the i386 backend moved from cc0 to a hard reg --
3443	maintaining source-level compatibility means automatically clobbering
3444	the flags register. */
3445	rtx_insn *after_md_seq = NULL;
3446	if (targetm.md_asm_adjust)
3447	after_md_seq
3448	= targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3449	constraints, clobber_rvec, clobbered_regs,
3450	locus);
3451
3452	/* Do not allow the hook to change the output and input count,
3453	lest it mess up the operand numbering. */
3454	gcc_assert (output_rvec.length() == noutputs);
3455	gcc_assert (input_rvec.length() == ninputs);
3456	gcc_assert (constraints.length() == noutputs + ninputs);
3457
3458	/* But it certainly can adjust the clobbers. */
3459	unsigned nclobbers = clobber_rvec.length ();
3460
3461	/* Third pass checks for easy conflicts. */
3462	/* ??? Why are we doing this on trees instead of rtx. */
3463
3464	bool clobber_conflict_found = 0;
3465	for (i = 0; i < noutputs; ++i)
3466	if (tree_conflicts_with_clobbers_p (t: output_tvec[i], clobbered_regs: &clobbered_regs, loc: locus))
3467	clobber_conflict_found = 1;
3468	for (i = 0; i < ninputs - ninout; ++i)
3469	if (tree_conflicts_with_clobbers_p (t: input_tvec[i], clobbered_regs: &clobbered_regs, loc: locus))
3470	clobber_conflict_found = 1;
3471
3472	/* Make vectors for the expression-rtx, constraint strings,
3473	and named operands. */
3474
3475	rtvec argvec = rtvec_alloc (ninputs);
3476	rtvec constraintvec = rtvec_alloc (ninputs);
3477	rtvec labelvec = rtvec_alloc (nlabels);
3478
3479	rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3480	: GET_MODE (output_rvec[0])),
3481	ggc_strdup (str),
3482	"", 0, argvec, constraintvec,
3483	labelvec, locus);
3484	MEM_VOLATILE_P (body) = gimple_asm_volatile_p (asm_stmt: stmt);
3485
3486	for (i = 0; i < ninputs; ++i)
3487	{
3488	ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3489	ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3490	= gen_rtx_ASM_INPUT_loc (input_mode[i],
3491	constraints[i + noutputs],
3492	locus);
3493	}
3494
3495	/* Copy labels to the vector. */
3496	rtx_code_label *fallthru_label = NULL;
3497	if (nlabels > 0)
3498	{
3499	basic_block fallthru_bb = NULL;
3500	edge fallthru = find_fallthru_edge (edges: gimple_bb (g: stmt)->succs);
3501	if (fallthru)
3502	fallthru_bb = fallthru->dest;
3503
3504	for (i = 0; i < nlabels; ++i)
3505	{
3506	tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3507	rtx_insn *r;
3508	/* If asm goto has any labels in the fallthru basic block, use
3509	a label that we emit immediately after the asm goto. Expansion
3510	may insert further instructions into the same basic block after
3511	asm goto and if we don't do this, insertion of instructions on
3512	the fallthru edge might misbehave. See PR58670. */
3513	if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3514	{
3515	if (fallthru_label == NULL_RTX)
3516	fallthru_label = gen_label_rtx ();
3517	r = fallthru_label;
3518	}
3519	else
3520	r = label_rtx (label);
3521	ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3522	}
3523	}
3524
3525	/* Now, for each output, construct an rtx
3526	(set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3527	ARGVEC CONSTRAINTS OPNAMES))
3528	If there is more than one, put them inside a PARALLEL. */
3529
3530	if (noutputs == 0 && nclobbers == 0)
3531	{
3532	/* No output operands: put in a raw ASM_OPERANDS rtx. */
3533	if (nlabels > 0)
3534	emit_jump_insn (body);
3535	else
3536	emit_insn (body);
3537	}
3538	else if (noutputs == 1 && nclobbers == 0)
3539	{
3540	ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3541	if (nlabels > 0)
3542	emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3543	else
3544	emit_insn (gen_rtx_SET (output_rvec[0], body));
3545	}
3546	else
3547	{
3548	rtx obody = body;
3549	int num = noutputs;
3550
3551	if (num == 0)
3552	num = 1;
3553
3554	body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (num + nclobbers));
3555
3556	/* For each output operand, store a SET. */
3557	for (i = 0; i < noutputs; ++i)
3558	{
3559	rtx src, o = output_rvec[i];
3560	if (i == 0)
3561	{
3562	ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3563	src = obody;
3564	}
3565	else
3566	{
3567	src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3568	ASM_OPERANDS_TEMPLATE (obody),
3569	constraints[i], i, argvec,
3570	constraintvec, labelvec, locus);
3571	MEM_VOLATILE_P (src) = gimple_asm_volatile_p (asm_stmt: stmt);
3572	}
3573	XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3574	}
3575
3576	/* If there are no outputs (but there are some clobbers)
3577	store the bare ASM_OPERANDS into the PARALLEL. */
3578	if (i == 0)
3579	XVECEXP (body, 0, i++) = obody;
3580
3581	/* Store (clobber REG) for each clobbered register specified. */
3582	for (unsigned j = 0; j < nclobbers; ++j)
3583	{
3584	rtx clobbered_reg = clobber_rvec[j];
3585
3586	/* Do sanity check for overlap between clobbers and respectively
3587	input and outputs that hasn't been handled. Such overlap
3588	should have been detected and reported above. */
3589	if (!clobber_conflict_found && REG_P (clobbered_reg))
3590	{
3591	/* We test the old body (obody) contents to avoid
3592	tripping over the under-construction body. */
3593	for (unsigned k = 0; k < noutputs; ++k)
3594	if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3595	internal_error ("%<asm%> clobber conflict with "
3596	"output operand");
3597
3598	for (unsigned k = 0; k < ninputs - ninout; ++k)
3599	if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
3600	internal_error ("%<asm%> clobber conflict with "
3601	"input operand");
3602	}
3603
3604	XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
3605	}
3606
3607	if (nlabels > 0)
3608	emit_jump_insn (body);
3609	else
3610	emit_insn (body);
3611	}
3612
3613	generating_concat_p = old_generating_concat_p;
3614
3615	if (fallthru_label)
3616	emit_label (fallthru_label);
3617
3618	if (after_md_seq)
3619	emit_insn (after_md_seq);
3620	if (after_rtl_seq)
3621	{
3622	if (nlabels == 0)
3623	emit_insn (after_rtl_seq);
3624	else
3625	{
3626	edge e;
3627	edge_iterator ei;
3628
3629	FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
3630	{
3631	start_sequence ();
3632	for (rtx_insn *curr = after_rtl_seq;
3633	curr != NULL_RTX;
3634	curr = NEXT_INSN (insn: curr))
3635	emit_insn (copy_insn (PATTERN (insn: curr)));
3636	rtx_insn *copy = get_insns ();
3637	end_sequence ();
3638	insert_insn_on_edge (copy, e);
3639	}
3640	}
3641	}
3642
3643	free_temp_slots ();
3644	crtl->has_asm_statement = 1;
3645	}
3646
3647	/* Emit code to jump to the address
3648	specified by the pointer expression EXP. */
3649
3650	static void
3651	expand_computed_goto (tree exp)
3652	{
3653	rtx x = expand_normal (exp);
3654
3655	do_pending_stack_adjust ();
3656	emit_indirect_jump (x);
3657	}
3658
3659	/* Generate RTL code for a `goto' statement with target label LABEL.
3660	LABEL should be a LABEL_DECL tree node that was or will later be
3661	defined with `expand_label'. */
3662
3663	static void
3664	expand_goto (tree label)
3665	{
3666	if (flag_checking)
3667	{
3668	/* Check for a nonlocal goto to a containing function. Should have
3669	gotten translated to __builtin_nonlocal_goto. */
3670	tree context = decl_function_context (label);
3671	gcc_assert (!context || context == current_function_decl);
3672	}
3673
3674	emit_jump (jump_target_rtx (label));
3675	}
3676
3677	/* Output a return with no value. */
3678
3679	static void
3680	expand_null_return_1 (void)
3681	{
3682	clear_pending_stack_adjust ();
3683	do_pending_stack_adjust ();
3684	emit_jump (return_label);
3685	}
3686
3687	/* Generate RTL to return from the current function, with no value.
3688	(That is, we do not do anything about returning any value.) */
3689
3690	void
3691	expand_null_return (void)
3692	{
3693	/* If this function was declared to return a value, but we
3694	didn't, clobber the return registers so that they are not
3695	propagated live to the rest of the function. */
3696	clobber_return_register ();
3697
3698	expand_null_return_1 ();
3699	}
3700
3701	/* Generate RTL to return from the current function, with value VAL. */
3702
3703	static void
3704	expand_value_return (rtx val)
3705	{
3706	/* Copy the value to the return location unless it's already there. */
3707
3708	tree decl = DECL_RESULT (current_function_decl);
3709	rtx return_reg = DECL_RTL (decl);
3710	if (return_reg != val)
3711	{
3712	tree funtype = TREE_TYPE (current_function_decl);
3713	tree type = TREE_TYPE (decl);
3714	int unsignedp = TYPE_UNSIGNED (type);
3715	machine_mode old_mode = DECL_MODE (decl);
3716	machine_mode mode;
3717	if (DECL_BY_REFERENCE (decl))
3718	mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
3719	else
3720	mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
3721
3722	if (mode != old_mode)
3723	{
3724	/* Some ABIs require scalar floating point modes to be returned
3725	in a wider scalar integer mode. We need to explicitly
3726	reinterpret to an integer mode of the correct precision
3727	before extending to the desired result. */
3728	if (SCALAR_INT_MODE_P (mode)
3729	&& SCALAR_FLOAT_MODE_P (old_mode)
3730	&& known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
3731	val = convert_float_to_wider_int (mode, fmode: old_mode, x: val);
3732	else
3733	val = convert_modes (mode, oldmode: old_mode, x: val, unsignedp);
3734	}
3735
3736	if (GET_CODE (return_reg) == PARALLEL)
3737	emit_group_load (return_reg, val, type, int_size_in_bytes (type));
3738	else
3739	emit_move_insn (return_reg, val);
3740	}
3741
3742	expand_null_return_1 ();
3743	}
3744
3745	/* Generate RTL to evaluate the expression RETVAL and return it
3746	from the current function. */
3747
3748	static void
3749	expand_return (tree retval)
3750	{
3751	rtx result_rtl;
3752	rtx val = 0;
3753	tree retval_rhs;
3754
3755	/* If function wants no value, give it none. */
3756	if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
3757	{
3758	expand_normal (exp: retval);
3759	expand_null_return ();
3760	return;
3761	}
3762
3763	if (retval == error_mark_node)
3764	{
3765	/* Treat this like a return of no value from a function that
3766	returns a value. */
3767	expand_null_return ();
3768	return;
3769	}
3770	else if ((TREE_CODE (retval) == MODIFY_EXPR
3771	|| TREE_CODE (retval) == INIT_EXPR)
3772	&& TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
3773	retval_rhs = TREE_OPERAND (retval, 1);
3774	else
3775	retval_rhs = retval;
3776
3777	result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
3778
3779	/* If we are returning the RESULT_DECL, then the value has already
3780	been stored into it, so we don't have to do anything special. */
3781	if (TREE_CODE (retval_rhs) == RESULT_DECL)
3782	expand_value_return (val: result_rtl);
3783
3784	/* If the result is an aggregate that is being returned in one (or more)
3785	registers, load the registers here. */
3786
3787	else if (retval_rhs != 0
3788	&& TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
3789	&& REG_P (result_rtl))
3790	{
3791	val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
3792	if (val)
3793	{
3794	/* Use the mode of the result value on the return register. */
3795	PUT_MODE (x: result_rtl, GET_MODE (val));
3796	expand_value_return (val);
3797	}
3798	else
3799	expand_null_return ();
3800	}
3801	else if (retval_rhs != 0
3802	&& !VOID_TYPE_P (TREE_TYPE (retval_rhs))
3803	&& (REG_P (result_rtl)
3804	|| (GET_CODE (result_rtl) == PARALLEL)))
3805	{
3806	/* Compute the return value into a temporary (usually a pseudo reg). */
3807	val
3808	= assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
3809	val = expand_expr (exp: retval_rhs, target: val, GET_MODE (val), modifier: EXPAND_NORMAL);
3810	val = force_not_mem (val);
3811	expand_value_return (val);
3812	}
3813	else
3814	{
3815	/* No hard reg used; calculate value into hard return reg. */
3816	expand_expr (exp: retval, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
3817	expand_value_return (val: result_rtl);
3818	}
3819	}
3820
3821	/* Expand a clobber of LHS. If LHS is stored it in a multi-part
3822	register, tell the rtl optimizers that its value is no longer
3823	needed. */
3824
3825	static void
3826	expand_clobber (tree lhs)
3827	{
3828	if (DECL_P (lhs))
3829	{
3830	rtx decl_rtl = DECL_RTL_IF_SET (lhs);
3831	if (decl_rtl && REG_P (decl_rtl))
3832	{
3833	machine_mode decl_mode = GET_MODE (decl_rtl);
3834	if (maybe_gt (GET_MODE_SIZE (decl_mode),
3835	REGMODE_NATURAL_SIZE (decl_mode)))
3836	emit_clobber (decl_rtl);
3837	}
3838	}
3839	}
3840
3841	/* A subroutine of expand_gimple_stmt, expanding one gimple statement
3842	STMT that doesn't require special handling for outgoing edges. That
3843	is no tailcalls and no GIMPLE_COND. */
3844
3845	static void
3846	expand_gimple_stmt_1 (gimple *stmt)
3847	{
3848	tree op0;
3849
3850	set_curr_insn_location (gimple_location (g: stmt));
3851
3852	switch (gimple_code (g: stmt))
3853	{
3854	case GIMPLE_GOTO:
3855	op0 = gimple_goto_dest (gs: stmt);
3856	if (TREE_CODE (op0) == LABEL_DECL)
3857	expand_goto (label: op0);
3858	else
3859	expand_computed_goto (exp: op0);
3860	break;
3861	case GIMPLE_LABEL:
3862	expand_label (gimple_label_label (gs: as_a <glabel *> (p: stmt)));
3863	break;
3864	case GIMPLE_NOP:
3865	case GIMPLE_PREDICT:
3866	break;
3867	case GIMPLE_SWITCH:
3868	{
3869	gswitch *swtch = as_a <gswitch *> (p: stmt);
3870	if (gimple_switch_num_labels (gs: swtch) == 1)
3871	expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
3872	else
3873	expand_case (swtch);
3874	}
3875	break;
3876	case GIMPLE_ASM:
3877	expand_asm_stmt (stmt: as_a <gasm *> (p: stmt));
3878	break;
3879	case GIMPLE_CALL:
3880	expand_call_stmt (stmt: as_a <gcall *> (p: stmt));
3881	break;
3882
3883	case GIMPLE_RETURN:
3884	{
3885	op0 = gimple_return_retval (gs: as_a <greturn *> (p: stmt));
3886
3887	/* If a return doesn't have a location, it very likely represents
3888	multiple user returns so we cannot let it inherit the location
3889	of the last statement of the previous basic block in RTL. */
3890	if (!gimple_has_location (g: stmt))
3891	set_curr_insn_location (cfun->function_end_locus);
3892
3893	if (op0 && op0 != error_mark_node)
3894	{
3895	tree result = DECL_RESULT (current_function_decl);
3896
3897	/* If we are not returning the current function's RESULT_DECL,
3898	build an assignment to it. */
3899	if (op0 != result)
3900	{
3901	/* I believe that a function's RESULT_DECL is unique. */
3902	gcc_assert (TREE_CODE (op0) != RESULT_DECL);
3903
3904	/* ??? We'd like to use simply expand_assignment here,
3905	but this fails if the value is of BLKmode but the return
3906	decl is a register. expand_return has special handling
3907	for this combination, which eventually should move
3908	to common code. See comments there. Until then, let's
3909	build a modify expression :-/ */
3910	op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
3911	result, op0);
3912	}
3913	}
3914
3915	if (!op0)
3916	expand_null_return ();
3917	else
3918	expand_return (retval: op0);
3919	}
3920	break;
3921
3922	case GIMPLE_ASSIGN:
3923	{
3924	gassign *assign_stmt = as_a <gassign *> (p: stmt);
3925	tree lhs = gimple_assign_lhs (gs: assign_stmt);
3926
3927	/* Tree expand used to fiddle with |= and &= of two bitfield
3928	COMPONENT_REFs here. This can't happen with gimple, the LHS
3929	of binary assigns must be a gimple reg. */
3930
3931	if (TREE_CODE (lhs) != SSA_NAME
3932	|| gimple_assign_rhs_class (gs: assign_stmt) == GIMPLE_SINGLE_RHS)
3933	{
3934	tree rhs = gimple_assign_rhs1 (gs: assign_stmt);
3935	gcc_assert (gimple_assign_rhs_class (assign_stmt)
3936	== GIMPLE_SINGLE_RHS);
3937	if (gimple_has_location (g: stmt) && CAN_HAVE_LOCATION_P (rhs)
3938	/* Do not put locations on possibly shared trees. */
3939	&& !is_gimple_min_invariant (rhs))
3940	SET_EXPR_LOCATION (rhs, gimple_location (stmt));
3941	if (TREE_CLOBBER_P (rhs))
3942	/* This is a clobber to mark the going out of scope for
3943	this LHS. */
3944	expand_clobber (lhs);
3945	else
3946	expand_assignment (lhs, rhs,
3947	gimple_assign_nontemporal_move_p (
3948	gs: assign_stmt));
3949	}
3950	else
3951	{
3952	rtx target, temp;
3953	bool nontemporal = gimple_assign_nontemporal_move_p (gs: assign_stmt);
3954	struct separate_ops ops;
3955	bool promoted = false;
3956
3957	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3958	if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
3959	promoted = true;
3960
3961	ops.code = gimple_assign_rhs_code (gs: assign_stmt);
3962	ops.type = TREE_TYPE (lhs);
3963	switch (get_gimple_rhs_class (code: ops.code))
3964	{
3965	case GIMPLE_TERNARY_RHS:
3966	ops.op2 = gimple_assign_rhs3 (gs: assign_stmt);
3967	/* Fallthru */
3968	case GIMPLE_BINARY_RHS:
3969	ops.op1 = gimple_assign_rhs2 (gs: assign_stmt);
3970	/* Fallthru */
3971	case GIMPLE_UNARY_RHS:
3972	ops.op0 = gimple_assign_rhs1 (gs: assign_stmt);
3973	break;
3974	default:
3975	gcc_unreachable ();
3976	}
3977	ops.location = gimple_location (g: stmt);
3978
3979	/* If we want to use a nontemporal store, force the value to
3980	register first. If we store into a promoted register,
3981	don't directly expand to target. */
3982	temp = nontemporal || promoted ? NULL_RTX : target;
3983	temp = expand_expr_real_2 (&ops, temp, GET_MODE (target),
3984	EXPAND_NORMAL);
3985
3986	if (temp == target)
3987	;
3988	else if (promoted)
3989	{
3990	int unsignedp = SUBREG_PROMOTED_SIGN (target);
3991	/* If TEMP is a VOIDmode constant, use convert_modes to make
3992	sure that we properly convert it. */
3993	if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
3994	{
3995	temp = convert_modes (GET_MODE (target),
3996	TYPE_MODE (ops.type),
3997	x: temp, unsignedp);
3998	temp = convert_modes (GET_MODE (SUBREG_REG (target)),
3999	GET_MODE (target), x: temp, unsignedp);
4000	}
4001
4002	convert_move (SUBREG_REG (target), temp, unsignedp);
4003	}
4004	else if (nontemporal && emit_storent_insn (to: target, from: temp))
4005	;
4006	else
4007	{
4008	temp = force_operand (temp, target);
4009	if (temp != target)
4010	emit_move_insn (target, temp);
4011	}
4012	}
4013	}
4014	break;
4015
4016	default:
4017	gcc_unreachable ();
4018	}
4019	}
4020
4021	/* Expand one gimple statement STMT and return the last RTL instruction
4022	before any of the newly generated ones.
4023
4024	In addition to generating the necessary RTL instructions this also
4025	sets REG_EH_REGION notes if necessary and sets the current source
4026	location for diagnostics. */
4027
4028	static rtx_insn *
4029	expand_gimple_stmt (gimple *stmt)
4030	{
4031	location_t saved_location = input_location;
4032	rtx_insn *last = get_last_insn ();
4033	int lp_nr;
4034
4035	gcc_assert (cfun);
4036
4037	/* We need to save and restore the current source location so that errors
4038	discovered during expansion are emitted with the right location. But
4039	it would be better if the diagnostic routines used the source location
4040	embedded in the tree nodes rather than globals. */
4041	if (gimple_has_location (g: stmt))
4042	input_location = gimple_location (g: stmt);
4043
4044	expand_gimple_stmt_1 (stmt);
4045
4046	/* Free any temporaries used to evaluate this statement. */
4047	free_temp_slots ();
4048
4049	input_location = saved_location;
4050
4051	/* Mark all insns that may trap. */
4052	lp_nr = lookup_stmt_eh_lp (stmt);
4053	if (lp_nr)
4054	{
4055	rtx_insn *insn;
4056	for (insn = next_real_insn (last); insn;
4057	insn = next_real_insn (insn))
4058	{
4059	if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4060	/* If we want exceptions for non-call insns, any
4061	may_trap_p instruction may throw. */
4062	&& GET_CODE (PATTERN (insn)) != CLOBBER
4063	&& GET_CODE (PATTERN (insn)) != USE
4064	&& insn_could_throw_p (insn))
4065	make_reg_eh_region_note (insn, ecf_flags: 0, lp_nr);
4066	}
4067	}
4068
4069	return last;
4070	}
4071
4072	/* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4073	that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4074	generated a tail call (something that might be denied by the ABI
4075	rules governing the call; see calls.cc).
4076
4077	Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4078	can still reach the rest of BB. The case here is __builtin_sqrt,
4079	where the NaN result goes through the external function (with a
4080	tailcall) and the normal result happens via a sqrt instruction. */
4081
4082	static basic_block
4083	expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru)
4084	{
4085	rtx_insn *last2, *last;
4086	edge e;
4087	edge_iterator ei;
4088	profile_probability probability;
4089
4090	last2 = last = expand_gimple_stmt (stmt);
4091
4092	for (last = NEXT_INSN (insn: last); last; last = NEXT_INSN (insn: last))
4093	if (CALL_P (last) && SIBLING_CALL_P (last))
4094	goto found;
4095
4096	maybe_dump_rtl_for_gimple_stmt (stmt, since: last2);
4097
4098	*can_fallthru = true;
4099	return NULL;
4100
4101	found:
4102	/* ??? Wouldn't it be better to just reset any pending stack adjust?
4103	Any instructions emitted here are about to be deleted. */
4104	do_pending_stack_adjust ();
4105
4106	/* Remove any non-eh, non-abnormal edges that don't go to exit. */
4107	/* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4108	EH or abnormal edges, we shouldn't have created a tail call in
4109	the first place. So it seems to me we should just be removing
4110	all edges here, or redirecting the existing fallthru edge to
4111	the exit block. */
4112
4113	probability = profile_probability::never ();
4114
4115	for (ei = ei_start (bb->succs); (e = ei_safe_edge (i: ei)); )
4116	{
4117	if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4118	{
4119	if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4120	e->dest->count -= e->count ();
4121	probability += e->probability;
4122	remove_edge (e);
4123	}
4124	else
4125	ei_next (i: &ei);
4126	}
4127
4128	/* This is somewhat ugly: the call_expr expander often emits instructions
4129	after the sibcall (to perform the function return). These confuse the
4130	find_many_sub_basic_blocks code, so we need to get rid of these. */
4131	last = NEXT_INSN (insn: last);
4132	gcc_assert (BARRIER_P (last));
4133
4134	*can_fallthru = false;
4135	while (NEXT_INSN (insn: last))
4136	{
4137	/* For instance an sqrt builtin expander expands if with
4138	sibcall in the then and label for `else`. */
4139	if (LABEL_P (NEXT_INSN (last)))
4140	{
4141	*can_fallthru = true;
4142	break;
4143	}
4144	delete_insn (NEXT_INSN (insn: last));
4145	}
4146
4147	e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4148	| EDGE_SIBCALL);
4149	e->probability = probability;
4150	BB_END (bb) = last;
4151	update_bb_for_insn (bb);
4152
4153	if (NEXT_INSN (insn: last))
4154	{
4155	bb = create_basic_block (NEXT_INSN (insn: last), get_last_insn (), bb);
4156
4157	last = BB_END (bb);
4158	if (BARRIER_P (last))
4159	BB_END (bb) = PREV_INSN (insn: last);
4160	}
4161
4162	maybe_dump_rtl_for_gimple_stmt (stmt, since: last2);
4163
4164	return bb;
4165	}
4166
4167	/* Return the difference between the floor and the truncated result of
4168	a signed division by OP1 with remainder MOD. */
4169	static rtx
4170	floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4171	{
4172	/* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4173	return gen_rtx_IF_THEN_ELSE
4174	(mode, gen_rtx_NE (BImode, mod, const0_rtx),
4175	gen_rtx_IF_THEN_ELSE
4176	(mode, gen_rtx_LT (BImode,
4177	gen_rtx_DIV (mode, op1, mod),
4178	const0_rtx),
4179	constm1_rtx, const0_rtx),
4180	const0_rtx);
4181	}
4182
4183	/* Return the difference between the ceil and the truncated result of
4184	a signed division by OP1 with remainder MOD. */
4185	static rtx
4186	ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4187	{
4188	/* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4189	return gen_rtx_IF_THEN_ELSE
4190	(mode, gen_rtx_NE (BImode, mod, const0_rtx),
4191	gen_rtx_IF_THEN_ELSE
4192	(mode, gen_rtx_GT (BImode,
4193	gen_rtx_DIV (mode, op1, mod),
4194	const0_rtx),
4195	const1_rtx, const0_rtx),
4196	const0_rtx);
4197	}
4198
4199	/* Return the difference between the ceil and the truncated result of
4200	an unsigned division by OP1 with remainder MOD. */
4201	static rtx
4202	ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4203	{
4204	/* (mod != 0 ? 1 : 0) */
4205	return gen_rtx_IF_THEN_ELSE
4206	(mode, gen_rtx_NE (BImode, mod, const0_rtx),
4207	const1_rtx, const0_rtx);
4208	}
4209
4210	/* Return the difference between the rounded and the truncated result
4211	of a signed division by OP1 with remainder MOD. Halfway cases are
4212	rounded away from zero, rather than to the nearest even number. */
4213	static rtx
4214	round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4215	{
4216	/* (abs (mod) >= abs (op1) - abs (mod)
4217	? (op1 / mod > 0 ? 1 : -1)
4218	: 0) */
4219	return gen_rtx_IF_THEN_ELSE
4220	(mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4221	gen_rtx_MINUS (mode,
4222	gen_rtx_ABS (mode, op1),
4223	gen_rtx_ABS (mode, mod))),
4224	gen_rtx_IF_THEN_ELSE
4225	(mode, gen_rtx_GT (BImode,
4226	gen_rtx_DIV (mode, op1, mod),
4227	const0_rtx),
4228	const1_rtx, constm1_rtx),
4229	const0_rtx);
4230	}
4231
4232	/* Return the difference between the rounded and the truncated result
4233	of a unsigned division by OP1 with remainder MOD. Halfway cases
4234	are rounded away from zero, rather than to the nearest even
4235	number. */
4236	static rtx
4237	round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4238	{
4239	/* (mod >= op1 - mod ? 1 : 0) */
4240	return gen_rtx_IF_THEN_ELSE
4241	(mode, gen_rtx_GE (BImode, mod,
4242	gen_rtx_MINUS (mode, op1, mod)),
4243	const1_rtx, const0_rtx);
4244	}
4245
4246	/* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4247	any rtl. */
4248
4249	static rtx
4250	convert_debug_memory_address (scalar_int_mode mode, rtx x,
4251	addr_space_t as)
4252	{
4253	#ifndef POINTERS_EXTEND_UNSIGNED
4254	gcc_assert (mode == Pmode
4255	|| mode == targetm.addr_space.address_mode (as));
4256	gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4257	#else
4258	rtx temp;
4259
4260	gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4261
4262	if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4263	return x;
4264
4265	/* X must have some form of address mode already. */
4266	scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4267	if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (mode: xmode))
4268	x = lowpart_subreg (outermode: mode, op: x, innermode: xmode);
4269	else if (POINTERS_EXTEND_UNSIGNED > 0)
4270	x = gen_rtx_ZERO_EXTEND (mode, x);
4271	else if (!POINTERS_EXTEND_UNSIGNED)
4272	x = gen_rtx_SIGN_EXTEND (mode, x);
4273	else
4274	{
4275	switch (GET_CODE (x))
4276	{
4277	case SUBREG:
4278	if ((SUBREG_PROMOTED_VAR_P (x)
4279	|| (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4280	|| (GET_CODE (SUBREG_REG (x)) == PLUS
4281	&& REG_P (XEXP (SUBREG_REG (x), 0))
4282	&& REG_POINTER (XEXP (SUBREG_REG (x), 0))
4283	&& CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4284	&& GET_MODE (SUBREG_REG (x)) == mode)
4285	return SUBREG_REG (x);
4286	break;
4287	case LABEL_REF:
4288	temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4289	LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4290	return temp;
4291	case SYMBOL_REF:
4292	temp = shallow_copy_rtx (x);
4293	PUT_MODE (x: temp, mode);
4294	return temp;
4295	case CONST:
4296	temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4297	if (temp)
4298	temp = gen_rtx_CONST (mode, temp);
4299	return temp;
4300	case PLUS:
4301	case MINUS:
4302	if (CONST_INT_P (XEXP (x, 1)))
4303	{
4304	temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4305	if (temp)
4306	return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4307	}
4308	break;
4309	default:
4310	break;
4311	}
4312	/* Don't know how to express ptr_extend as operation in debug info. */
4313	return NULL;
4314	}
4315	#endif /* POINTERS_EXTEND_UNSIGNED */
4316
4317	return x;
4318	}
4319
4320	/* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4321	by avoid_deep_ter_for_debug. */
4322
4323	static hash_map<tree, tree> *deep_ter_debug_map;
4324
4325	/* Split too deep TER chains for debug stmts using debug temporaries. */
4326
4327	static void
4328	avoid_deep_ter_for_debug (gimple *stmt, int depth)
4329	{
4330	use_operand_p use_p;
4331	ssa_op_iter iter;
4332	FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4333	{
4334	tree use = USE_FROM_PTR (use_p);
4335	if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4336	continue;
4337	gimple *g = get_gimple_for_ssa_name (exp: use);
4338	if (g == NULL)
4339	continue;
4340	if (depth > 6 && !stmt_ends_bb_p (g))
4341	{
4342	if (deep_ter_debug_map == NULL)
4343	deep_ter_debug_map = new hash_map<tree, tree>;
4344
4345	tree &vexpr = deep_ter_debug_map->get_or_insert (k: use);
4346	if (vexpr != NULL)
4347	continue;
4348	vexpr = build_debug_expr_decl (TREE_TYPE (use));
4349	gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4350	gimple_stmt_iterator gsi = gsi_for_stmt (g);
4351	gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4352	avoid_deep_ter_for_debug (stmt: def_temp, depth: 0);
4353	}
4354	else
4355	avoid_deep_ter_for_debug (stmt: g, depth: depth + 1);
4356	}
4357	}
4358
4359	/* Return an RTX equivalent to the value of the parameter DECL. */
4360
4361	static rtx
4362	expand_debug_parm_decl (tree decl)
4363	{
4364	rtx incoming = DECL_INCOMING_RTL (decl);
4365
4366	if (incoming
4367	&& GET_MODE (incoming) != BLKmode
4368	&& ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4369	|| (MEM_P (incoming)
4370	&& REG_P (XEXP (incoming, 0))
4371	&& HARD_REGISTER_P (XEXP (incoming, 0)))))
4372	{
4373	rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4374
4375	#ifdef HAVE_window_save
4376	/* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4377	If the target machine has an explicit window save instruction, the
4378	actual entry value is the corresponding OUTGOING_REGNO instead. */
4379	if (REG_P (incoming)
4380	&& OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4381	incoming
4382	= gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4383	OUTGOING_REGNO (REGNO (incoming)), 0);
4384	else if (MEM_P (incoming))
4385	{
4386	rtx reg = XEXP (incoming, 0);
4387	if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4388	{
4389	reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4390	incoming = replace_equiv_address_nv (incoming, reg);
4391	}
4392	else
4393	incoming = copy_rtx (incoming);
4394	}
4395	#endif
4396
4397	ENTRY_VALUE_EXP (rtl) = incoming;
4398	return rtl;
4399	}
4400
4401	if (incoming
4402	&& GET_MODE (incoming) != BLKmode
4403	&& !TREE_ADDRESSABLE (decl)
4404	&& MEM_P (incoming)
4405	&& (XEXP (incoming, 0) == virtual_incoming_args_rtx
4406	|| (GET_CODE (XEXP (incoming, 0)) == PLUS
4407	&& XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4408	&& CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4409	return copy_rtx (incoming);
4410
4411	return NULL_RTX;
4412	}
4413
4414	/* Return an RTX equivalent to the value of the tree expression EXP. */
4415
4416	static rtx
4417	expand_debug_expr (tree exp)
4418	{
4419	rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4420	machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4421	machine_mode inner_mode = VOIDmode;
4422	int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4423	addr_space_t as;
4424	scalar_int_mode op0_mode, op1_mode, addr_mode;
4425
4426	switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4427	{
4428	case tcc_expression:
4429	switch (TREE_CODE (exp))
4430	{
4431	case COND_EXPR:
4432	case DOT_PROD_EXPR:
4433	case SAD_EXPR:
4434	case WIDEN_MULT_PLUS_EXPR:
4435	case WIDEN_MULT_MINUS_EXPR:
4436	goto ternary;
4437
4438	case TRUTH_ANDIF_EXPR:
4439	case TRUTH_ORIF_EXPR:
4440	case TRUTH_AND_EXPR:
4441	case TRUTH_OR_EXPR:
4442	case TRUTH_XOR_EXPR:
4443	goto binary;
4444
4445	case TRUTH_NOT_EXPR:
4446	goto unary;
4447
4448	default:
4449	break;
4450	}
4451	break;
4452
4453	ternary:
4454	op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4455	if (!op2)
4456	return NULL_RTX;
4457	/* Fall through. */
4458
4459	binary:
4460	case tcc_binary:
4461	if (mode == BLKmode)
4462	return NULL_RTX;
4463	op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4464	if (!op1)
4465	return NULL_RTX;
4466	switch (TREE_CODE (exp))
4467	{
4468	case LSHIFT_EXPR:
4469	case RSHIFT_EXPR:
4470	case LROTATE_EXPR:
4471	case RROTATE_EXPR:
4472	case WIDEN_LSHIFT_EXPR:
4473	/* Ensure second operand isn't wider than the first one. */
4474	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4475	if (is_a <scalar_int_mode> (m: inner_mode, result: &op1_mode)
4476	&& (GET_MODE_UNIT_PRECISION (mode)
4477	< GET_MODE_PRECISION (mode: op1_mode)))
4478	op1 = lowpart_subreg (GET_MODE_INNER (mode), op: op1, innermode: op1_mode);
4479	break;
4480	default:
4481	break;
4482	}
4483	/* Fall through. */
4484
4485	unary:
4486	case tcc_unary:
4487	if (mode == BLKmode)
4488	return NULL_RTX;
4489	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4490	op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4491	if (!op0)
4492	return NULL_RTX;
4493	break;
4494
4495	case tcc_comparison:
4496	unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4497	goto binary;
4498
4499	case tcc_type:
4500	case tcc_statement:
4501	gcc_unreachable ();
4502
4503	case tcc_constant:
4504	case tcc_exceptional:
4505	case tcc_declaration:
4506	case tcc_reference:
4507	case tcc_vl_exp:
4508	break;
4509	}
4510
4511	switch (TREE_CODE (exp))
4512	{
4513	case STRING_CST:
4514	if (!lookup_constant_def (exp))
4515	{
4516	if (strlen (TREE_STRING_POINTER (exp)) + 1
4517	!= (size_t) TREE_STRING_LENGTH (exp))
4518	return NULL_RTX;
4519	op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4520	op0 = gen_rtx_MEM (BLKmode, op0);
4521	set_mem_attributes (op0, exp, 0);
4522	return op0;
4523	}
4524	/* Fall through. */
4525
4526	case INTEGER_CST:
4527	if (TREE_CODE (TREE_TYPE (exp)) == BITINT_TYPE
4528	&& TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
4529	return NULL;
4530	/* FALLTHRU */
4531	case REAL_CST:
4532	case FIXED_CST:
4533	op0 = expand_expr (exp, NULL_RTX, mode, modifier: EXPAND_INITIALIZER);
4534	return op0;
4535
4536	case POLY_INT_CST:
4537	return immed_wide_int_const (poly_int_cst_value (x: exp), mode);
4538
4539	case COMPLEX_CST:
4540	gcc_assert (COMPLEX_MODE_P (mode));
4541	op0 = expand_debug_expr (TREE_REALPART (exp));
4542	op1 = expand_debug_expr (TREE_IMAGPART (exp));
4543	return gen_rtx_CONCAT (mode, op0, op1);
4544
4545	case DEBUG_EXPR_DECL:
4546	op0 = DECL_RTL_IF_SET (exp);
4547
4548	if (op0)
4549	{
4550	if (GET_MODE (op0) != mode)
4551	gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4552	else
4553	return op0;
4554	}
4555
4556	op0 = gen_rtx_DEBUG_EXPR (mode);
4557	DEBUG_EXPR_TREE_DECL (op0) = exp;
4558	SET_DECL_RTL (exp, op0);
4559
4560	return op0;
4561
4562	case VAR_DECL:
4563	case PARM_DECL:
4564	case FUNCTION_DECL:
4565	case LABEL_DECL:
4566	case CONST_DECL:
4567	case RESULT_DECL:
4568	op0 = DECL_RTL_IF_SET (exp);
4569
4570	/* This decl was probably optimized away. */
4571	if (!op0
4572	/* At least label RTXen are sometimes replaced by
4573	NOTE_INSN_DELETED_LABEL. Any notes here are not
4574	handled by copy_rtx. */
4575	|| NOTE_P (op0))
4576	{
4577	if (!VAR_P (exp)
4578	|| DECL_EXTERNAL (exp)
4579	|| !TREE_STATIC (exp)
4580	|| !DECL_NAME (exp)
4581	|| DECL_HARD_REGISTER (exp)
4582	|| DECL_IN_CONSTANT_POOL (exp)
4583	|| mode == VOIDmode
4584	|| symtab_node::get (decl: exp) == NULL)
4585	return NULL;
4586
4587	op0 = make_decl_rtl_for_debug (exp);
4588	if (!MEM_P (op0)
4589	|| GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
4590	|| SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
4591	return NULL;
4592	}
4593	else if (VAR_P (exp)
4594	&& is_global_var (t: exp)
4595	&& symtab_node::get (decl: exp) == NULL)
4596	return NULL;
4597	else
4598	op0 = copy_rtx (op0);
4599
4600	if (GET_MODE (op0) == BLKmode
4601	/* If op0 is not BLKmode, but mode is, adjust_mode
4602	below would ICE. While it is likely a FE bug,
4603	try to be robust here. See PR43166. */
4604	|| mode == BLKmode
4605	|| (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
4606	{
4607	gcc_assert (MEM_P (op0));
4608	op0 = adjust_address_nv (op0, mode, 0);
4609	return op0;
4610	}
4611
4612	/* Fall through. */
4613
4614	adjust_mode:
4615	case PAREN_EXPR:
4616	CASE_CONVERT:
4617	{
4618	inner_mode = GET_MODE (op0);
4619
4620	if (mode == inner_mode)
4621	return op0;
4622
4623	if (inner_mode == VOIDmode)
4624	{
4625	if (TREE_CODE (exp) == SSA_NAME)
4626	inner_mode = TYPE_MODE (TREE_TYPE (exp));
4627	else
4628	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4629	if (mode == inner_mode)
4630	return op0;
4631	}
4632
4633	if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
4634	{
4635	if (GET_MODE_UNIT_BITSIZE (mode)
4636	== GET_MODE_UNIT_BITSIZE (inner_mode))
4637	op0 = simplify_gen_subreg (outermode: mode, op: op0, innermode: inner_mode, byte: 0);
4638	else if (GET_MODE_UNIT_BITSIZE (mode)
4639	< GET_MODE_UNIT_BITSIZE (inner_mode))
4640	op0 = simplify_gen_unary (code: FLOAT_TRUNCATE, mode, op: op0, op_mode: inner_mode);
4641	else
4642	op0 = simplify_gen_unary (code: FLOAT_EXTEND, mode, op: op0, op_mode: inner_mode);
4643	}
4644	else if (FLOAT_MODE_P (mode))
4645	{
4646	gcc_assert (TREE_CODE (exp) != SSA_NAME);
4647	if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
4648	op0 = simplify_gen_unary (code: UNSIGNED_FLOAT, mode, op: op0, op_mode: inner_mode);
4649	else
4650	op0 = simplify_gen_unary (code: FLOAT, mode, op: op0, op_mode: inner_mode);
4651	}
4652	else if (FLOAT_MODE_P (inner_mode))
4653	{
4654	if (unsignedp)
4655	op0 = simplify_gen_unary (code: UNSIGNED_FIX, mode, op: op0, op_mode: inner_mode);
4656	else
4657	op0 = simplify_gen_unary (code: FIX, mode, op: op0, op_mode: inner_mode);
4658	}
4659	else if (GET_MODE_UNIT_PRECISION (mode)
4660	== GET_MODE_UNIT_PRECISION (inner_mode))
4661	op0 = lowpart_subreg (outermode: mode, op: op0, innermode: inner_mode);
4662	else if (GET_MODE_UNIT_PRECISION (mode)
4663	< GET_MODE_UNIT_PRECISION (inner_mode))
4664	op0 = simplify_gen_unary (code: TRUNCATE, mode, op: op0, op_mode: inner_mode);
4665	else if (UNARY_CLASS_P (exp)
4666	? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
4667	: unsignedp)
4668	op0 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op0, op_mode: inner_mode);
4669	else
4670	op0 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op0, op_mode: inner_mode);
4671
4672	return op0;
4673	}
4674
4675	case MEM_REF:
4676	if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4677	{
4678	tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
4679	TREE_OPERAND (exp, 0),
4680	TREE_OPERAND (exp, 1));
4681	if (newexp)
4682	return expand_debug_expr (exp: newexp);
4683	}
4684	/* FALLTHROUGH */
4685	case INDIRECT_REF:
4686	inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4687	op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4688	if (!op0)
4689	return NULL;
4690
4691	if (TREE_CODE (exp) == MEM_REF)
4692	{
4693	if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
4694	|| (GET_CODE (op0) == PLUS
4695	&& GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
4696	/* (mem (debug_implicit_ptr)) might confuse aliasing.
4697	Instead just use get_inner_reference. */
4698	goto component_ref;
4699
4700	op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4701	poly_int64 offset;
4702	if (!op1 || !poly_int_rtx_p (x: op1, res: &offset))
4703	return NULL;
4704
4705	op0 = plus_constant (inner_mode, op0, offset);
4706	}
4707
4708	as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4709
4710	op0 = convert_debug_memory_address (mode: targetm.addr_space.address_mode (as),
4711	x: op0, as);
4712	if (op0 == NULL_RTX)
4713	return NULL;
4714
4715	op0 = gen_rtx_MEM (mode, op0);
4716	set_mem_attributes (op0, exp, 0);
4717	if (TREE_CODE (exp) == MEM_REF
4718	&& !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4719	set_mem_expr (op0, NULL_TREE);
4720	set_mem_addr_space (op0, as);
4721
4722	return op0;
4723
4724	case TARGET_MEM_REF:
4725	if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
4726	&& !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
4727	return NULL;
4728
4729	op0 = expand_debug_expr
4730	(exp: tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
4731	if (!op0)
4732	return NULL;
4733
4734	as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4735	op0 = convert_debug_memory_address (mode: targetm.addr_space.address_mode (as),
4736	x: op0, as);
4737	if (op0 == NULL_RTX)
4738	return NULL;
4739
4740	op0 = gen_rtx_MEM (mode, op0);
4741
4742	set_mem_attributes (op0, exp, 0);
4743	set_mem_addr_space (op0, as);
4744
4745	return op0;
4746
4747	component_ref:
4748	case ARRAY_REF:
4749	case ARRAY_RANGE_REF:
4750	case COMPONENT_REF:
4751	case BIT_FIELD_REF:
4752	case REALPART_EXPR:
4753	case IMAGPART_EXPR:
4754	case VIEW_CONVERT_EXPR:
4755	{
4756	machine_mode mode1;
4757	poly_int64 bitsize, bitpos;
4758	tree offset;
4759	int reversep, volatilep = 0;
4760	tree tem
4761	= get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
4762	&unsignedp, &reversep, &volatilep);
4763	rtx orig_op0;
4764
4765	if (known_eq (bitsize, 0))
4766	return NULL;
4767
4768	orig_op0 = op0 = expand_debug_expr (exp: tem);
4769
4770	if (!op0)
4771	return NULL;
4772
4773	if (offset)
4774	{
4775	machine_mode addrmode, offmode;
4776
4777	if (!MEM_P (op0))
4778	return NULL;
4779
4780	op0 = XEXP (op0, 0);
4781	addrmode = GET_MODE (op0);
4782	if (addrmode == VOIDmode)
4783	addrmode = Pmode;
4784
4785	op1 = expand_debug_expr (exp: offset);
4786	if (!op1)
4787	return NULL;
4788
4789	offmode = GET_MODE (op1);
4790	if (offmode == VOIDmode)
4791	offmode = TYPE_MODE (TREE_TYPE (offset));
4792
4793	if (addrmode != offmode)
4794	op1 = lowpart_subreg (outermode: addrmode, op: op1, innermode: offmode);
4795
4796	/* Don't use offset_address here, we don't need a
4797	recognizable address, and we don't want to generate
4798	code. */
4799	op0 = gen_rtx_MEM (mode, simplify_gen_binary (code: PLUS, mode: addrmode,
4800	op0, op1));
4801	}
4802
4803	if (MEM_P (op0))
4804	{
4805	if (mode1 == VOIDmode)
4806	{
4807	if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
4808	return NULL;
4809	/* Bitfield. */
4810	mode1 = smallest_int_mode_for_size (size: bitsize);
4811	}
4812	poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
4813	if (maybe_ne (a: bytepos, b: 0))
4814	{
4815	op0 = adjust_address_nv (op0, mode1, bytepos);
4816	bitpos = num_trailing_bits (bitpos);
4817	}
4818	else if (known_eq (bitpos, 0)
4819	&& known_eq (bitsize, GET_MODE_BITSIZE (mode)))
4820	op0 = adjust_address_nv (op0, mode, 0);
4821	else if (GET_MODE (op0) != mode1)
4822	op0 = adjust_address_nv (op0, mode1, 0);
4823	else
4824	op0 = copy_rtx (op0);
4825	if (op0 == orig_op0)
4826	op0 = shallow_copy_rtx (op0);
4827	if (TREE_CODE (tem) != SSA_NAME)
4828	set_mem_attributes (op0, exp, 0);
4829	}
4830
4831	if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
4832	return op0;
4833
4834	if (maybe_lt (a: bitpos, b: 0))
4835	return NULL;
4836
4837	if (GET_MODE (op0) == BLKmode || mode == BLKmode)
4838	return NULL;
4839
4840	poly_int64 bytepos;
4841	if (multiple_p (a: bitpos, BITS_PER_UNIT, multiple: &bytepos)
4842	&& known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
4843	{
4844	machine_mode opmode = GET_MODE (op0);
4845
4846	if (opmode == VOIDmode)
4847	opmode = TYPE_MODE (TREE_TYPE (tem));
4848
4849	/* This condition may hold if we're expanding the address
4850	right past the end of an array that turned out not to
4851	be addressable (i.e., the address was only computed in
4852	debug stmts). The gen_subreg below would rightfully
4853	crash, and the address doesn't really exist, so just
4854	drop it. */
4855	if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
4856	return NULL;
4857
4858	if (multiple_p (a: bitpos, b: GET_MODE_BITSIZE (mode)))
4859	return simplify_gen_subreg (outermode: mode, op: op0, innermode: opmode, byte: bytepos);
4860	}
4861
4862	return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
4863	&& TYPE_UNSIGNED (TREE_TYPE (exp))
4864	? SIGN_EXTRACT
4865	: ZERO_EXTRACT, mode,
4866	GET_MODE (op0) != VOIDmode
4867	? GET_MODE (op0)
4868	: TYPE_MODE (TREE_TYPE (tem)),
4869	op0, op1: gen_int_mode (bitsize, word_mode),
4870	op2: gen_int_mode (bitpos, word_mode));
4871	}
4872
4873	case ABS_EXPR:
4874	case ABSU_EXPR:
4875	return simplify_gen_unary (code: ABS, mode, op: op0, op_mode: mode);
4876
4877	case NEGATE_EXPR:
4878	return simplify_gen_unary (code: NEG, mode, op: op0, op_mode: mode);
4879
4880	case BIT_NOT_EXPR:
4881	return simplify_gen_unary (code: NOT, mode, op: op0, op_mode: mode);
4882
4883	case FLOAT_EXPR:
4884	return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
4885	0)))
4886	? UNSIGNED_FLOAT : FLOAT, mode, op: op0,
4887	op_mode: inner_mode);
4888
4889	case FIX_TRUNC_EXPR:
4890	return simplify_gen_unary (code: unsignedp ? UNSIGNED_FIX : FIX, mode, op: op0,
4891	op_mode: inner_mode);
4892
4893	case POINTER_PLUS_EXPR:
4894	/* For the rare target where pointers are not the same size as
4895	size_t, we need to check for mis-matched modes and correct
4896	the addend. */
4897	if (op0 && op1
4898	&& is_a <scalar_int_mode> (GET_MODE (op0), result: &op0_mode)
4899	&& is_a <scalar_int_mode> (GET_MODE (op1), result: &op1_mode)
4900	&& op0_mode != op1_mode)
4901	{
4902	if (GET_MODE_BITSIZE (mode: op0_mode) < GET_MODE_BITSIZE (mode: op1_mode)
4903	/* If OP0 is a partial mode, then we must truncate, even
4904	if it has the same bitsize as OP1 as GCC's
4905	representation of partial modes is opaque. */
4906	|| (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
4907	&& (GET_MODE_BITSIZE (mode: op0_mode)
4908	== GET_MODE_BITSIZE (mode: op1_mode))))
4909	op1 = simplify_gen_unary (code: TRUNCATE, mode: op0_mode, op: op1, op_mode: op1_mode);
4910	else
4911	/* We always sign-extend, regardless of the signedness of
4912	the operand, because the operand is always unsigned
4913	here even if the original C expression is signed. */
4914	op1 = simplify_gen_unary (code: SIGN_EXTEND, mode: op0_mode, op: op1, op_mode: op1_mode);
4915	}
4916	/* Fall through. */
4917	case PLUS_EXPR:
4918	return simplify_gen_binary (code: PLUS, mode, op0, op1);
4919
4920	case MINUS_EXPR:
4921	case POINTER_DIFF_EXPR:
4922	return simplify_gen_binary (code: MINUS, mode, op0, op1);
4923
4924	case MULT_EXPR:
4925	return simplify_gen_binary (code: MULT, mode, op0, op1);
4926
4927	case RDIV_EXPR:
4928	case TRUNC_DIV_EXPR:
4929	case EXACT_DIV_EXPR:
4930	if (unsignedp)
4931	return simplify_gen_binary (code: UDIV, mode, op0, op1);
4932	else
4933	return simplify_gen_binary (code: DIV, mode, op0, op1);
4934
4935	case TRUNC_MOD_EXPR:
4936	return simplify_gen_binary (code: unsignedp ? UMOD : MOD, mode, op0, op1);
4937
4938	case FLOOR_DIV_EXPR:
4939	if (unsignedp)
4940	return simplify_gen_binary (code: UDIV, mode, op0, op1);
4941	else
4942	{
4943	rtx div = simplify_gen_binary (code: DIV, mode, op0, op1);
4944	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
4945	rtx adj = floor_sdiv_adjust (mode, mod, op1);
4946	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
4947	}
4948
4949	case FLOOR_MOD_EXPR:
4950	if (unsignedp)
4951	return simplify_gen_binary (code: UMOD, mode, op0, op1);
4952	else
4953	{
4954	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
4955	rtx adj = floor_sdiv_adjust (mode, mod, op1);
4956	adj = simplify_gen_unary (code: NEG, mode,
4957	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
4958	op_mode: mode);
4959	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
4960	}
4961
4962	case CEIL_DIV_EXPR:
4963	if (unsignedp)
4964	{
4965	rtx div = simplify_gen_binary (code: UDIV, mode, op0, op1);
4966	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
4967	rtx adj = ceil_udiv_adjust (mode, mod, op1);
4968	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
4969	}
4970	else
4971	{
4972	rtx div = simplify_gen_binary (code: DIV, mode, op0, op1);
4973	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
4974	rtx adj = ceil_sdiv_adjust (mode, mod, op1);
4975	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
4976	}
4977
4978	case CEIL_MOD_EXPR:
4979	if (unsignedp)
4980	{
4981	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
4982	rtx adj = ceil_udiv_adjust (mode, mod, op1);
4983	adj = simplify_gen_unary (code: NEG, mode,
4984	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
4985	op_mode: mode);
4986	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
4987	}
4988	else
4989	{
4990	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
4991	rtx adj = ceil_sdiv_adjust (mode, mod, op1);
4992	adj = simplify_gen_unary (code: NEG, mode,
4993	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
4994	op_mode: mode);
4995	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
4996	}
4997
4998	case ROUND_DIV_EXPR:
4999	if (unsignedp)
5000	{
5001	rtx div = simplify_gen_binary (code: UDIV, mode, op0, op1);
5002	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
5003	rtx adj = round_udiv_adjust (mode, mod, op1);
5004	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
5005	}
5006	else
5007	{
5008	rtx div = simplify_gen_binary (code: DIV, mode, op0, op1);
5009	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
5010	rtx adj = round_sdiv_adjust (mode, mod, op1);
5011	return simplify_gen_binary (code: PLUS, mode, op0: div, op1: adj);
5012	}
5013
5014	case ROUND_MOD_EXPR:
5015	if (unsignedp)
5016	{
5017	rtx mod = simplify_gen_binary (code: UMOD, mode, op0, op1);
5018	rtx adj = round_udiv_adjust (mode, mod, op1);
5019	adj = simplify_gen_unary (code: NEG, mode,
5020	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
5021	op_mode: mode);
5022	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
5023	}
5024	else
5025	{
5026	rtx mod = simplify_gen_binary (code: MOD, mode, op0, op1);
5027	rtx adj = round_sdiv_adjust (mode, mod, op1);
5028	adj = simplify_gen_unary (code: NEG, mode,
5029	op: simplify_gen_binary (code: MULT, mode, op0: adj, op1),
5030	op_mode: mode);
5031	return simplify_gen_binary (code: PLUS, mode, op0: mod, op1: adj);
5032	}
5033
5034	case LSHIFT_EXPR:
5035	return simplify_gen_binary (code: ASHIFT, mode, op0, op1);
5036
5037	case RSHIFT_EXPR:
5038	if (unsignedp)
5039	return simplify_gen_binary (code: LSHIFTRT, mode, op0, op1);
5040	else
5041	return simplify_gen_binary (code: ASHIFTRT, mode, op0, op1);
5042
5043	case LROTATE_EXPR:
5044	return simplify_gen_binary (code: ROTATE, mode, op0, op1);
5045
5046	case RROTATE_EXPR:
5047	return simplify_gen_binary (code: ROTATERT, mode, op0, op1);
5048
5049	case MIN_EXPR:
5050	return simplify_gen_binary (code: unsignedp ? UMIN : SMIN, mode, op0, op1);
5051
5052	case MAX_EXPR:
5053	return simplify_gen_binary (code: unsignedp ? UMAX : SMAX, mode, op0, op1);
5054
5055	case BIT_AND_EXPR:
5056	case TRUTH_AND_EXPR:
5057	return simplify_gen_binary (code: AND, mode, op0, op1);
5058
5059	case BIT_IOR_EXPR:
5060	case TRUTH_OR_EXPR:
5061	return simplify_gen_binary (code: IOR, mode, op0, op1);
5062
5063	case BIT_XOR_EXPR:
5064	case TRUTH_XOR_EXPR:
5065	return simplify_gen_binary (code: XOR, mode, op0, op1);
5066
5067	case TRUTH_ANDIF_EXPR:
5068	return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5069
5070	case TRUTH_ORIF_EXPR:
5071	return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5072
5073	case TRUTH_NOT_EXPR:
5074	return simplify_gen_relational (code: EQ, mode, op_mode: inner_mode, op0, const0_rtx);
5075
5076	case LT_EXPR:
5077	return simplify_gen_relational (code: unsignedp ? LTU : LT, mode, op_mode: inner_mode,
5078	op0, op1);
5079
5080	case LE_EXPR:
5081	return simplify_gen_relational (code: unsignedp ? LEU : LE, mode, op_mode: inner_mode,
5082	op0, op1);
5083
5084	case GT_EXPR:
5085	return simplify_gen_relational (code: unsignedp ? GTU : GT, mode, op_mode: inner_mode,
5086	op0, op1);
5087
5088	case GE_EXPR:
5089	return simplify_gen_relational (code: unsignedp ? GEU : GE, mode, op_mode: inner_mode,
5090	op0, op1);
5091
5092	case EQ_EXPR:
5093	return simplify_gen_relational (code: EQ, mode, op_mode: inner_mode, op0, op1);
5094
5095	case NE_EXPR:
5096	return simplify_gen_relational (code: NE, mode, op_mode: inner_mode, op0, op1);
5097
5098	case UNORDERED_EXPR:
5099	return simplify_gen_relational (code: UNORDERED, mode, op_mode: inner_mode, op0, op1);
5100
5101	case ORDERED_EXPR:
5102	return simplify_gen_relational (code: ORDERED, mode, op_mode: inner_mode, op0, op1);
5103
5104	case UNLT_EXPR:
5105	return simplify_gen_relational (code: UNLT, mode, op_mode: inner_mode, op0, op1);
5106
5107	case UNLE_EXPR:
5108	return simplify_gen_relational (code: UNLE, mode, op_mode: inner_mode, op0, op1);
5109
5110	case UNGT_EXPR:
5111	return simplify_gen_relational (code: UNGT, mode, op_mode: inner_mode, op0, op1);
5112
5113	case UNGE_EXPR:
5114	return simplify_gen_relational (code: UNGE, mode, op_mode: inner_mode, op0, op1);
5115
5116	case UNEQ_EXPR:
5117	return simplify_gen_relational (code: UNEQ, mode, op_mode: inner_mode, op0, op1);
5118
5119	case LTGT_EXPR:
5120	return simplify_gen_relational (code: LTGT, mode, op_mode: inner_mode, op0, op1);
5121
5122	case COND_EXPR:
5123	return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5124
5125	case COMPLEX_EXPR:
5126	gcc_assert (COMPLEX_MODE_P (mode));
5127	if (GET_MODE (op0) == VOIDmode)
5128	op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5129	if (GET_MODE (op1) == VOIDmode)
5130	op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5131	return gen_rtx_CONCAT (mode, op0, op1);
5132
5133	case CONJ_EXPR:
5134	if (GET_CODE (op0) == CONCAT)
5135	return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5136	simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5137	XEXP (op0, 1),
5138	GET_MODE_INNER (mode)));
5139	else
5140	{
5141	scalar_mode imode = GET_MODE_INNER (mode);
5142	rtx re, im;
5143
5144	if (MEM_P (op0))
5145	{
5146	re = adjust_address_nv (op0, imode, 0);
5147	im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5148	}
5149	else
5150	{
5151	scalar_int_mode ifmode;
5152	scalar_int_mode ihmode;
5153	rtx halfsize;
5154	if (!int_mode_for_mode (mode).exists (mode: &ifmode)
5155	|| !int_mode_for_mode (imode).exists (mode: &ihmode))
5156	return NULL;
5157	halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5158	re = op0;
5159	if (mode != ifmode)
5160	re = gen_rtx_SUBREG (ifmode, re, 0);
5161	re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5162	if (imode != ihmode)
5163	re = gen_rtx_SUBREG (imode, re, 0);
5164	im = copy_rtx (op0);
5165	if (mode != ifmode)
5166	im = gen_rtx_SUBREG (ifmode, im, 0);
5167	im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5168	if (imode != ihmode)
5169	im = gen_rtx_SUBREG (imode, im, 0);
5170	}
5171	im = gen_rtx_NEG (imode, im);
5172	return gen_rtx_CONCAT (mode, re, im);
5173	}
5174
5175	case ADDR_EXPR:
5176	op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5177	if (!op0 || !MEM_P (op0))
5178	{
5179	if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5180	|| TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5181	|| TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5182	&& (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5183	|| target_for_debug_bind (TREE_OPERAND (exp, 0))))
5184	return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5185
5186	if (handled_component_p (TREE_OPERAND (exp, 0)))
5187	{
5188	poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5189	bool reverse;
5190	tree decl
5191	= get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5192	&bitsize, &maxsize, &reverse);
5193	if ((VAR_P (decl)
5194	|| TREE_CODE (decl) == PARM_DECL
5195	|| TREE_CODE (decl) == RESULT_DECL)
5196	&& (!TREE_ADDRESSABLE (decl)
5197	|| target_for_debug_bind (decl))
5198	&& multiple_p (a: bitoffset, BITS_PER_UNIT, multiple: &byteoffset)
5199	&& known_gt (bitsize, 0)
5200	&& known_eq (bitsize, maxsize))
5201	{
5202	rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5203	return plus_constant (mode, base, byteoffset);
5204	}
5205	}
5206
5207	if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5208	&& TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5209	== ADDR_EXPR)
5210	{
5211	op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5212	0));
5213	if (op0 != NULL
5214	&& (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5215	|| (GET_CODE (op0) == PLUS
5216	&& GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5217	&& CONST_INT_P (XEXP (op0, 1)))))
5218	{
5219	op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5220	1));
5221	poly_int64 offset;
5222	if (!op1 || !poly_int_rtx_p (x: op1, res: &offset))
5223	return NULL;
5224
5225	return plus_constant (mode, op0, offset);
5226	}
5227	}
5228
5229	return NULL;
5230	}
5231
5232	as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5233	addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5234	op0 = convert_debug_memory_address (mode: addr_mode, XEXP (op0, 0), as);
5235
5236	return op0;
5237
5238	case VECTOR_CST:
5239	{
5240	unsigned HOST_WIDE_INT i, nelts;
5241
5242	if (!VECTOR_CST_NELTS (exp).is_constant (const_value: &nelts))
5243	return NULL;
5244
5245	op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5246
5247	for (i = 0; i < nelts; ++i)
5248	{
5249	op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5250	if (!op1)
5251	return NULL;
5252	XVECEXP (op0, 0, i) = op1;
5253	}
5254
5255	return op0;
5256	}
5257
5258	case CONSTRUCTOR:
5259	if (TREE_CLOBBER_P (exp))
5260	return NULL;
5261	else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5262	{
5263	unsigned i;
5264	unsigned HOST_WIDE_INT nelts;
5265	tree val;
5266
5267	if (!TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp)).is_constant (const_value: &nelts))
5268	goto flag_unsupported;
5269
5270	op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5271
5272	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5273	{
5274	op1 = expand_debug_expr (exp: val);
5275	if (!op1)
5276	return NULL;
5277	XVECEXP (op0, 0, i) = op1;
5278	}
5279
5280	if (i < nelts)
5281	{
5282	op1 = expand_debug_expr
5283	(exp: build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5284
5285	if (!op1)
5286	return NULL;
5287
5288	for (; i < nelts; i++)
5289	XVECEXP (op0, 0, i) = op1;
5290	}
5291
5292	return op0;
5293	}
5294	else
5295	goto flag_unsupported;
5296
5297	case CALL_EXPR:
5298	/* ??? Maybe handle some builtins? */
5299	return NULL;
5300
5301	case SSA_NAME:
5302	{
5303	gimple *g = get_gimple_for_ssa_name (exp);
5304	if (g)
5305	{
5306	tree t = NULL_TREE;
5307	if (deep_ter_debug_map)
5308	{
5309	tree *slot = deep_ter_debug_map->get (k: exp);
5310	if (slot)
5311	t = *slot;
5312	}
5313	if (t == NULL_TREE)
5314	t = gimple_assign_rhs_to_tree (stmt: g);
5315	op0 = expand_debug_expr (exp: t);
5316	if (!op0)
5317	return NULL;
5318	}
5319	else
5320	{
5321	/* If this is a reference to an incoming value of
5322	parameter that is never used in the code or where the
5323	incoming value is never used in the code, use
5324	PARM_DECL's DECL_RTL if set. */
5325	if (SSA_NAME_IS_DEFAULT_DEF (exp)
5326	&& SSA_NAME_VAR (exp)
5327	&& TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5328	&& has_zero_uses (var: exp))
5329	{
5330	op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5331	if (op0)
5332	goto adjust_mode;
5333	op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5334	if (op0)
5335	goto adjust_mode;
5336	}
5337
5338	int part = var_to_partition (map: SA.map, var: exp);
5339
5340	if (part == NO_PARTITION)
5341	return NULL;
5342
5343	gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5344
5345	op0 = copy_rtx (SA.partition_to_pseudo[part]);
5346	}
5347	goto adjust_mode;
5348	}
5349
5350	case ERROR_MARK:
5351	return NULL;
5352
5353	/* Vector stuff. For most of the codes we don't have rtl codes. */
5354	case REALIGN_LOAD_EXPR:
5355	case VEC_COND_EXPR:
5356	case VEC_PACK_FIX_TRUNC_EXPR:
5357	case VEC_PACK_FLOAT_EXPR:
5358	case VEC_PACK_SAT_EXPR:
5359	case VEC_PACK_TRUNC_EXPR:
5360	case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5361	case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5362	case VEC_UNPACK_FLOAT_HI_EXPR:
5363	case VEC_UNPACK_FLOAT_LO_EXPR:
5364	case VEC_UNPACK_HI_EXPR:
5365	case VEC_UNPACK_LO_EXPR:
5366	case VEC_WIDEN_MULT_HI_EXPR:
5367	case VEC_WIDEN_MULT_LO_EXPR:
5368	case VEC_WIDEN_MULT_EVEN_EXPR:
5369	case VEC_WIDEN_MULT_ODD_EXPR:
5370	case VEC_WIDEN_LSHIFT_HI_EXPR:
5371	case VEC_WIDEN_LSHIFT_LO_EXPR:
5372	case VEC_PERM_EXPR:
5373	case VEC_DUPLICATE_EXPR:
5374	case VEC_SERIES_EXPR:
5375	case SAD_EXPR:
5376	return NULL;
5377
5378	/* Misc codes. */
5379	case ADDR_SPACE_CONVERT_EXPR:
5380	case FIXED_CONVERT_EXPR:
5381	case OBJ_TYPE_REF:
5382	case WITH_SIZE_EXPR:
5383	case BIT_INSERT_EXPR:
5384	return NULL;
5385
5386	case DOT_PROD_EXPR:
5387	if (SCALAR_INT_MODE_P (GET_MODE (op0))
5388	&& SCALAR_INT_MODE_P (mode))
5389	{
5390	op0
5391	= simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5392	0)))
5393	? ZERO_EXTEND : SIGN_EXTEND, mode, op: op0,
5394	op_mode: inner_mode);
5395	op1
5396	= simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5397	1)))
5398	? ZERO_EXTEND : SIGN_EXTEND, mode, op: op1,
5399	op_mode: inner_mode);
5400	op0 = simplify_gen_binary (code: MULT, mode, op0, op1);
5401	return simplify_gen_binary (code: PLUS, mode, op0, op1: op2);
5402	}
5403	return NULL;
5404
5405	case WIDEN_MULT_EXPR:
5406	case WIDEN_MULT_PLUS_EXPR:
5407	case WIDEN_MULT_MINUS_EXPR:
5408	if (SCALAR_INT_MODE_P (GET_MODE (op0))
5409	&& SCALAR_INT_MODE_P (mode))
5410	{
5411	inner_mode = GET_MODE (op0);
5412	if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5413	op0 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op0, op_mode: inner_mode);
5414	else
5415	op0 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op0, op_mode: inner_mode);
5416	if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5417	op1 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op1, op_mode: inner_mode);
5418	else
5419	op1 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op1, op_mode: inner_mode);
5420	op0 = simplify_gen_binary (code: MULT, mode, op0, op1);
5421	if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5422	return op0;
5423	else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5424	return simplify_gen_binary (code: PLUS, mode, op0, op1: op2);
5425	else
5426	return simplify_gen_binary (code: MINUS, mode, op0: op2, op1: op0);
5427	}
5428	return NULL;
5429
5430	case MULT_HIGHPART_EXPR:
5431	/* ??? Similar to the above. */
5432	return NULL;
5433
5434	case WIDEN_SUM_EXPR:
5435	case WIDEN_LSHIFT_EXPR:
5436	if (SCALAR_INT_MODE_P (GET_MODE (op0))
5437	&& SCALAR_INT_MODE_P (mode))
5438	{
5439	op0
5440	= simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5441	0)))
5442	? ZERO_EXTEND : SIGN_EXTEND, mode, op: op0,
5443	op_mode: inner_mode);
5444	return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5445	? ASHIFT : PLUS, mode, op0, op1);
5446	}
5447	return NULL;
5448
5449	default:
5450	flag_unsupported:
5451	if (flag_checking)
5452	{
5453	debug_tree (exp);
5454	gcc_unreachable ();
5455	}
5456	return NULL;
5457	}
5458	}
5459
5460	/* Return an RTX equivalent to the source bind value of the tree expression
5461	EXP. */
5462
5463	static rtx
5464	expand_debug_source_expr (tree exp)
5465	{
5466	rtx op0 = NULL_RTX;
5467	machine_mode mode = VOIDmode, inner_mode;
5468
5469	switch (TREE_CODE (exp))
5470	{
5471	case VAR_DECL:
5472	if (DECL_ABSTRACT_ORIGIN (exp))
5473	return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5474	break;
5475	case PARM_DECL:
5476	{
5477	mode = DECL_MODE (exp);
5478	op0 = expand_debug_parm_decl (decl: exp);
5479	if (op0)
5480	break;
5481	/* See if this isn't an argument that has been completely
5482	optimized out. */
5483	if (!DECL_RTL_SET_P (exp)
5484	&& !DECL_INCOMING_RTL (exp)
5485	&& DECL_ABSTRACT_ORIGIN (current_function_decl))
5486	{
5487	tree aexp = DECL_ORIGIN (exp);
5488	if (DECL_CONTEXT (aexp)
5489	== DECL_ABSTRACT_ORIGIN (current_function_decl))
5490	{
5491	vec<tree, va_gc> **debug_args;
5492	unsigned int ix;
5493	tree ddecl;
5494	debug_args = decl_debug_args_lookup (current_function_decl);
5495	if (debug_args != NULL)
5496	{
5497	for (ix = 0; vec_safe_iterate (v: *debug_args, ix, ptr: &ddecl);
5498	ix += 2)
5499	if (ddecl == aexp)
5500	return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5501	}
5502	}
5503	}
5504	break;
5505	}
5506	default:
5507	break;
5508	}
5509
5510	if (op0 == NULL_RTX)
5511	return NULL_RTX;
5512
5513	inner_mode = GET_MODE (op0);
5514	if (mode == inner_mode)
5515	return op0;
5516
5517	if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5518	{
5519	if (GET_MODE_UNIT_BITSIZE (mode)
5520	== GET_MODE_UNIT_BITSIZE (inner_mode))
5521	op0 = simplify_gen_subreg (outermode: mode, op: op0, innermode: inner_mode, byte: 0);
5522	else if (GET_MODE_UNIT_BITSIZE (mode)
5523	< GET_MODE_UNIT_BITSIZE (inner_mode))
5524	op0 = simplify_gen_unary (code: FLOAT_TRUNCATE, mode, op: op0, op_mode: inner_mode);
5525	else
5526	op0 = simplify_gen_unary (code: FLOAT_EXTEND, mode, op: op0, op_mode: inner_mode);
5527	}
5528	else if (FLOAT_MODE_P (mode))
5529	gcc_unreachable ();
5530	else if (FLOAT_MODE_P (inner_mode))
5531	{
5532	if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5533	op0 = simplify_gen_unary (code: UNSIGNED_FIX, mode, op: op0, op_mode: inner_mode);
5534	else
5535	op0 = simplify_gen_unary (code: FIX, mode, op: op0, op_mode: inner_mode);
5536	}
5537	else if (GET_MODE_UNIT_PRECISION (mode)
5538	== GET_MODE_UNIT_PRECISION (inner_mode))
5539	op0 = lowpart_subreg (outermode: mode, op: op0, innermode: inner_mode);
5540	else if (GET_MODE_UNIT_PRECISION (mode)
5541	< GET_MODE_UNIT_PRECISION (inner_mode))
5542	op0 = simplify_gen_unary (code: TRUNCATE, mode, op: op0, op_mode: inner_mode);
5543	else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5544	op0 = simplify_gen_unary (code: ZERO_EXTEND, mode, op: op0, op_mode: inner_mode);
5545	else
5546	op0 = simplify_gen_unary (code: SIGN_EXTEND, mode, op: op0, op_mode: inner_mode);
5547
5548	return op0;
5549	}
5550
5551	/* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5552	Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
5553	deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
5554
5555	static void
5556	avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
5557	{
5558	rtx exp = *exp_p;
5559
5560	if (exp == NULL_RTX)
5561	return;
5562
5563	if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
5564	return;
5565
5566	if (depth == 4)
5567	{
5568	/* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
5569	rtx dval = make_debug_expr_from_rtl (exp);
5570
5571	/* Emit a debug bind insn before INSN. */
5572	rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
5573	DEBUG_EXPR_TREE_DECL (dval), exp,
5574	VAR_INIT_STATUS_INITIALIZED);
5575
5576	emit_debug_insn_before (bind, insn);
5577	*exp_p = dval;
5578	return;
5579	}
5580
5581	const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
5582	int i, j;
5583	for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
5584	switch (*format_ptr++)
5585	{
5586	case 'e':
5587	avoid_complex_debug_insns (insn, exp_p: &XEXP (exp, i), depth: depth + 1);
5588	break;
5589
5590	case 'E':
5591	case 'V':
5592	for (j = 0; j < XVECLEN (exp, i); j++)
5593	avoid_complex_debug_insns (insn, exp_p: &XVECEXP (exp, i, j), depth: depth + 1);
5594	break;
5595
5596	default:
5597	break;
5598	}
5599	}
5600
5601	/* Expand the _LOCs in debug insns. We run this after expanding all
5602	regular insns, so that any variables referenced in the function
5603	will have their DECL_RTLs set. */
5604
5605	static void
5606	expand_debug_locations (void)
5607	{
5608	rtx_insn *insn;
5609	rtx_insn *last = get_last_insn ();
5610	int save_strict_alias = flag_strict_aliasing;
5611
5612	/* New alias sets while setting up memory attributes cause
5613	-fcompare-debug failures, even though it doesn't bring about any
5614	codegen changes. */
5615	flag_strict_aliasing = 0;
5616
5617	for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
5618	if (DEBUG_BIND_INSN_P (insn))
5619	{
5620	tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
5621	rtx val;
5622	rtx_insn *prev_insn, *insn2;
5623	machine_mode mode;
5624
5625	if (value == NULL_TREE)
5626	val = NULL_RTX;
5627	else
5628	{
5629	if (INSN_VAR_LOCATION_STATUS (insn)
5630	== VAR_INIT_STATUS_UNINITIALIZED)
5631	val = expand_debug_source_expr (exp: value);
5632	/* The avoid_deep_ter_for_debug function inserts
5633	debug bind stmts after SSA_NAME definition, with the
5634	SSA_NAME as the whole bind location. Disable temporarily
5635	expansion of that SSA_NAME into the DEBUG_EXPR_DECL
5636	being defined in this DEBUG_INSN. */
5637	else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
5638	{
5639	tree *slot = deep_ter_debug_map->get (k: value);
5640	if (slot)
5641	{
5642	if (*slot == INSN_VAR_LOCATION_DECL (insn))
5643	*slot = NULL_TREE;
5644	else
5645	slot = NULL;
5646	}
5647	val = expand_debug_expr (exp: value);
5648	if (slot)
5649	*slot = INSN_VAR_LOCATION_DECL (insn);
5650	}
5651	else
5652	val = expand_debug_expr (exp: value);
5653	gcc_assert (last == get_last_insn ());
5654	}
5655
5656	if (!val)
5657	val = gen_rtx_UNKNOWN_VAR_LOC ();
5658	else
5659	{
5660	mode = GET_MODE (INSN_VAR_LOCATION (insn));
5661
5662	gcc_assert (mode == GET_MODE (val)
5663	|| (GET_MODE (val) == VOIDmode
5664	&& (CONST_SCALAR_INT_P (val)
5665	|| GET_CODE (val) == CONST_FIXED
5666	|| GET_CODE (val) == LABEL_REF)));
5667	}
5668
5669	INSN_VAR_LOCATION_LOC (insn) = val;
5670	prev_insn = PREV_INSN (insn);
5671	for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn: insn2))
5672	avoid_complex_debug_insns (insn: insn2, exp_p: &INSN_VAR_LOCATION_LOC (insn2), depth: 0);
5673	}
5674
5675	flag_strict_aliasing = save_strict_alias;
5676	}
5677
5678	/* Performs swapping operands of commutative operations to expand
5679	the expensive one first. */
5680
5681	static void
5682	reorder_operands (basic_block bb)
5683	{
5684	unsigned int *lattice; /* Hold cost of each statement. */
5685	unsigned int i = 0, n = 0;
5686	gimple_stmt_iterator gsi;
5687	gimple_seq stmts;
5688	gimple *stmt;
5689	bool swap;
5690	tree op0, op1;
5691	ssa_op_iter iter;
5692	use_operand_p use_p;
5693	gimple *def0, *def1;
5694
5695	/* Compute cost of each statement using estimate_num_insns. */
5696	stmts = bb_seq (bb);
5697	for (gsi = gsi_start (seq&: stmts); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5698	{
5699	stmt = gsi_stmt (i: gsi);
5700	if (!is_gimple_debug (gs: stmt))
5701	gimple_set_uid (g: stmt, uid: n++);
5702	}
5703	lattice = XNEWVEC (unsigned int, n);
5704	for (gsi = gsi_start (seq&: stmts); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5705	{
5706	unsigned cost;
5707	stmt = gsi_stmt (i: gsi);
5708	if (is_gimple_debug (gs: stmt))
5709	continue;
5710	cost = estimate_num_insns (stmt, &eni_size_weights);
5711	lattice[i] = cost;
5712	FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
5713	{
5714	tree use = USE_FROM_PTR (use_p);
5715	gimple *def_stmt;
5716	if (TREE_CODE (use) != SSA_NAME)
5717	continue;
5718	def_stmt = get_gimple_for_ssa_name (exp: use);
5719	if (!def_stmt)
5720	continue;
5721	lattice[i] += lattice[gimple_uid (g: def_stmt)];
5722	}
5723	i++;
5724	if (!is_gimple_assign (gs: stmt)
5725	|| !commutative_tree_code (gimple_assign_rhs_code (gs: stmt)))
5726	continue;
5727	op0 = gimple_op (gs: stmt, i: 1);
5728	op1 = gimple_op (gs: stmt, i: 2);
5729	if (TREE_CODE (op0) != SSA_NAME
5730	|| TREE_CODE (op1) != SSA_NAME)
5731	continue;
5732	/* Swap operands if the second one is more expensive. */
5733	def0 = get_gimple_for_ssa_name (exp: op0);
5734	def1 = get_gimple_for_ssa_name (exp: op1);
5735	if (!def1)
5736	continue;
5737	swap = false;
5738	if (!def0 || lattice[gimple_uid (g: def1)] > lattice[gimple_uid (g: def0)])
5739	swap = true;
5740	if (swap)
5741	{
5742	if (dump_file && (dump_flags & TDF_DETAILS))
5743	{
5744	fprintf (stream: dump_file, format: "Swap operands in stmt:\n");
5745	print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
5746	fprintf (stream: dump_file, format: "Cost left opnd=%d, right opnd=%d\n",
5747	def0 ? lattice[gimple_uid (g: def0)] : 0,
5748	lattice[gimple_uid (g: def1)]);
5749	}
5750	swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (gs: stmt),
5751	gimple_assign_rhs2_ptr (gs: stmt));
5752	}
5753	}
5754	XDELETE (lattice);
5755	}
5756
5757	/* Expand basic block BB from GIMPLE trees to RTL. */
5758
5759	static basic_block
5760	expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
5761	{
5762	gimple_stmt_iterator gsi;
5763	gimple_seq stmts;
5764	gimple *stmt = NULL;
5765	rtx_note *note = NULL;
5766	rtx_insn *last;
5767	edge e;
5768	edge_iterator ei;
5769	bool nondebug_stmt_seen = false;
5770
5771	if (dump_file)
5772	fprintf (stream: dump_file, format: "\n;; Generating RTL for gimple basic block %d\n",
5773	bb->index);
5774
5775	/* Note that since we are now transitioning from GIMPLE to RTL, we
5776	cannot use the gsi_*_bb() routines because they expect the basic
5777	block to be in GIMPLE, instead of RTL. Therefore, we need to
5778	access the BB sequence directly. */
5779	if (optimize)
5780	reorder_operands (bb);
5781	stmts = bb_seq (bb);
5782	bb->il.gimple.seq = NULL;
5783	bb->il.gimple.phi_nodes = NULL;
5784	rtl_profile_for_bb (bb);
5785	init_rtl_bb_info (bb);
5786	bb->flags |= BB_RTL;
5787
5788	/* Remove the RETURN_EXPR if we may fall though to the exit
5789	instead. */
5790	gsi = gsi_last (seq&: stmts);
5791	if (!gsi_end_p (i: gsi)
5792	&& gimple_code (g: gsi_stmt (i: gsi)) == GIMPLE_RETURN)
5793	{
5794	greturn *ret_stmt = as_a <greturn *> (p: gsi_stmt (i: gsi));
5795
5796	gcc_assert (single_succ_p (bb));
5797	gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
5798
5799	if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
5800	&& !gimple_return_retval (gs: ret_stmt))
5801	{
5802	gsi_remove (&gsi, false);
5803	single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
5804	}
5805	}
5806
5807	gsi = gsi_start (seq&: stmts);
5808	if (!gsi_end_p (i: gsi))
5809	{
5810	stmt = gsi_stmt (i: gsi);
5811	if (gimple_code (g: stmt) != GIMPLE_LABEL)
5812	stmt = NULL;
5813	}
5814
5815	rtx_code_label **elt = lab_rtx_for_bb->get (k: bb);
5816
5817	if (stmt || elt)
5818	{
5819	gcc_checking_assert (!note);
5820	last = get_last_insn ();
5821
5822	if (stmt)
5823	{
5824	expand_gimple_stmt (stmt);
5825	gsi_next (i: &gsi);
5826	}
5827
5828	if (elt)
5829	emit_label (*elt);
5830
5831	BB_HEAD (bb) = NEXT_INSN (insn: last);
5832	if (NOTE_P (BB_HEAD (bb)))
5833	BB_HEAD (bb) = NEXT_INSN (BB_HEAD (bb));
5834	gcc_assert (LABEL_P (BB_HEAD (bb)));
5835	note = emit_note_after (NOTE_INSN_BASIC_BLOCK, BB_HEAD (bb));
5836
5837	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
5838	}
5839	else
5840	BB_HEAD (bb) = note = emit_note (NOTE_INSN_BASIC_BLOCK);
5841
5842	if (note)
5843	NOTE_BASIC_BLOCK (note) = bb;
5844
5845	for (; !gsi_end_p (i: gsi); gsi_next (i: &gsi))
5846	{
5847	basic_block new_bb;
5848
5849	stmt = gsi_stmt (i: gsi);
5850	if (!is_gimple_debug (gs: stmt))
5851	nondebug_stmt_seen = true;
5852
5853	/* If this statement is a non-debug one, and we generate debug
5854	insns, then this one might be the last real use of a TERed
5855	SSA_NAME, but where there are still some debug uses further
5856	down. Expanding the current SSA name in such further debug
5857	uses by their RHS might lead to wrong debug info, as coalescing
5858	might make the operands of such RHS be placed into the same
5859	pseudo as something else. Like so:
5860	a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
5861	use(a_1);
5862	a_2 = ...
5863	#DEBUG ... => a_1
5864	As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
5865	If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
5866	the write to a_2 would actually have clobbered the place which
5867	formerly held a_0.
5868
5869	So, instead of that, we recognize the situation, and generate
5870	debug temporaries at the last real use of TERed SSA names:
5871	a_1 = a_0 + 1;
5872	#DEBUG #D1 => a_1
5873	use(a_1);
5874	a_2 = ...
5875	#DEBUG ... => #D1
5876	*/
5877	if (MAY_HAVE_DEBUG_BIND_INSNS
5878	&& SA.values
5879	&& !is_gimple_debug (gs: stmt))
5880	{
5881	ssa_op_iter iter;
5882	tree op;
5883	gimple *def;
5884
5885	location_t sloc = curr_insn_location ();
5886
5887	/* Look for SSA names that have their last use here (TERed
5888	names always have only one real use). */
5889	FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
5890	if ((def = get_gimple_for_ssa_name (exp: op)))
5891	{
5892	imm_use_iterator imm_iter;
5893	use_operand_p use_p;
5894	bool have_debug_uses = false;
5895
5896	FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
5897	{
5898	if (gimple_debug_bind_p (USE_STMT (use_p)))
5899	{
5900	have_debug_uses = true;
5901	break;
5902	}
5903	}
5904
5905	if (have_debug_uses)
5906	{
5907	/* OP is a TERed SSA name, with DEF its defining
5908	statement, and where OP is used in further debug
5909	instructions. Generate a debug temporary, and
5910	replace all uses of OP in debug insns with that
5911	temporary. */
5912	gimple *debugstmt;
5913	tree value = gimple_assign_rhs_to_tree (stmt: def);
5914	tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
5915	rtx val;
5916	machine_mode mode;
5917
5918	set_curr_insn_location (gimple_location (g: def));
5919
5920	if (DECL_P (value))
5921	mode = DECL_MODE (value);
5922	else
5923	mode = TYPE_MODE (TREE_TYPE (value));
5924	/* FIXME: Is setting the mode really necessary? */
5925	SET_DECL_MODE (vexpr, mode);
5926
5927	val = gen_rtx_VAR_LOCATION
5928	(mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5929
5930	emit_debug_insn (val);
5931
5932	FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
5933	{
5934	if (!gimple_debug_bind_p (s: debugstmt))
5935	continue;
5936
5937	FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
5938	SET_USE (use_p, vexpr);
5939
5940	update_stmt (s: debugstmt);
5941	}
5942	}
5943	}
5944	set_curr_insn_location (sloc);
5945	}
5946
5947	currently_expanding_gimple_stmt = stmt;
5948
5949	/* Expand this statement, then evaluate the resulting RTL and
5950	fixup the CFG accordingly. */
5951	if (gimple_code (g: stmt) == GIMPLE_COND)
5952	{
5953	new_bb = expand_gimple_cond (bb, stmt: as_a <gcond *> (p: stmt));
5954	if (new_bb)
5955	{
5956	currently_expanding_gimple_stmt = NULL;
5957	return new_bb;
5958	}
5959	}
5960	else if (is_gimple_debug (gs: stmt))
5961	{
5962	location_t sloc = curr_insn_location ();
5963	gimple_stmt_iterator nsi = gsi;
5964
5965	for (;;)
5966	{
5967	tree var;
5968	tree value = NULL_TREE;
5969	rtx val = NULL_RTX;
5970	machine_mode mode;
5971
5972	if (!gimple_debug_nonbind_marker_p (s: stmt))
5973	{
5974	if (gimple_debug_bind_p (s: stmt))
5975	{
5976	var = gimple_debug_bind_get_var (dbg: stmt);
5977
5978	if (TREE_CODE (var) != DEBUG_EXPR_DECL
5979	&& TREE_CODE (var) != LABEL_DECL
5980	&& !target_for_debug_bind (var))
5981	goto delink_debug_stmt;
5982
5983	if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
5984	mode = DECL_MODE (var);
5985	else
5986	mode = TYPE_MODE (TREE_TYPE (var));
5987
5988	if (gimple_debug_bind_has_value_p (dbg: stmt))
5989	value = gimple_debug_bind_get_value (dbg: stmt);
5990
5991	val = gen_rtx_VAR_LOCATION
5992	(mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5993	}
5994	else if (gimple_debug_source_bind_p (s: stmt))
5995	{
5996	var = gimple_debug_source_bind_get_var (dbg: stmt);
5997
5998	value = gimple_debug_source_bind_get_value (dbg: stmt);
5999
6000	if (!VECTOR_TYPE_P (TREE_TYPE (var)))
6001	mode = DECL_MODE (var);
6002	else
6003	mode = TYPE_MODE (TREE_TYPE (var));
6004
6005	val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
6006	VAR_INIT_STATUS_UNINITIALIZED);
6007	}
6008	else
6009	gcc_unreachable ();
6010	}
6011	/* If this function was first compiled with markers
6012	enabled, but they're now disable (e.g. LTO), drop
6013	them on the floor. */
6014	else if (gimple_debug_nonbind_marker_p (s: stmt)
6015	&& !MAY_HAVE_DEBUG_MARKER_INSNS)
6016	goto delink_debug_stmt;
6017	else if (gimple_debug_begin_stmt_p (s: stmt))
6018	val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
6019	else if (gimple_debug_inline_entry_p (s: stmt))
6020	val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6021	else
6022	gcc_unreachable ();
6023
6024	last = get_last_insn ();
6025
6026	set_curr_insn_location (gimple_location (g: stmt));
6027
6028	emit_debug_insn (val);
6029
6030	if (dump_file && (dump_flags & TDF_DETAILS))
6031	{
6032	/* We can't dump the insn with a TREE where an RTX
6033	is expected. */
6034	if (GET_CODE (val) == VAR_LOCATION)
6035	{
6036	gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6037	PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6038	}
6039	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
6040	if (GET_CODE (val) == VAR_LOCATION)
6041	PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6042	}
6043
6044	delink_debug_stmt:
6045	/* In order not to generate too many debug temporaries,
6046	we delink all uses of debug statements we already expanded.
6047	Therefore debug statements between definition and real
6048	use of TERed SSA names will continue to use the SSA name,
6049	and not be replaced with debug temps. */
6050	delink_stmt_imm_use (stmt);
6051
6052	gsi = nsi;
6053	gsi_next (i: &nsi);
6054	if (gsi_end_p (i: nsi))
6055	break;
6056	stmt = gsi_stmt (i: nsi);
6057	if (!is_gimple_debug (gs: stmt))
6058	break;
6059	}
6060
6061	set_curr_insn_location (sloc);
6062	}
6063	else
6064	{
6065	gcall *call_stmt = dyn_cast <gcall *> (p: stmt);
6066	if (call_stmt
6067	&& gimple_call_tail_p (s: call_stmt)
6068	&& disable_tail_calls)
6069	gimple_call_set_tail (s: call_stmt, tail_p: false);
6070
6071	if (call_stmt && gimple_call_tail_p (s: call_stmt))
6072	{
6073	bool can_fallthru;
6074	new_bb = expand_gimple_tailcall (bb, stmt: call_stmt, can_fallthru: &can_fallthru);
6075	if (new_bb)
6076	{
6077	if (can_fallthru)
6078	bb = new_bb;
6079	else
6080	{
6081	currently_expanding_gimple_stmt = NULL;
6082	return new_bb;
6083	}
6084	}
6085	}
6086	else
6087	{
6088	def_operand_p def_p;
6089	def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6090
6091	if (def_p != NULL)
6092	{
6093	/* Ignore this stmt if it is in the list of
6094	replaceable expressions. */
6095	if (SA.values
6096	&& bitmap_bit_p (SA.values,
6097	SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6098	continue;
6099	}
6100	last = expand_gimple_stmt (stmt);
6101	maybe_dump_rtl_for_gimple_stmt (stmt, since: last);
6102	}
6103	}
6104	}
6105
6106	currently_expanding_gimple_stmt = NULL;
6107
6108	/* Expand implicit goto and convert goto_locus. */
6109	FOR_EACH_EDGE (e, ei, bb->succs)
6110	{
6111	if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
6112	set_curr_insn_location (e->goto_locus);
6113	if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6114	{
6115	emit_jump (label_rtx_for_bb (bb: e->dest));
6116	e->flags &= ~EDGE_FALLTHRU;
6117	}
6118	}
6119
6120	/* Expanded RTL can create a jump in the last instruction of block.
6121	This later might be assumed to be a jump to successor and break edge insertion.
6122	We need to insert dummy move to prevent this. PR41440. */
6123	if (single_succ_p (bb)
6124	&& (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6125	&& (last = get_last_insn ())
6126	&& (JUMP_P (last)
6127	|| (DEBUG_INSN_P (last)
6128	&& JUMP_P (prev_nondebug_insn (last)))))
6129	{
6130	rtx dummy = gen_reg_rtx (SImode);
6131	emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6132	}
6133
6134	do_pending_stack_adjust ();
6135
6136	/* Find the block tail. The last insn in the block is the insn
6137	before a barrier and/or table jump insn. */
6138	last = get_last_insn ();
6139	if (BARRIER_P (last))
6140	last = PREV_INSN (insn: last);
6141	if (JUMP_TABLE_DATA_P (last))
6142	last = PREV_INSN (insn: PREV_INSN (insn: last));
6143	if (BARRIER_P (last))
6144	last = PREV_INSN (insn: last);
6145	BB_END (bb) = last;
6146
6147	update_bb_for_insn (bb);
6148
6149	return bb;
6150	}
6151
6152
6153	/* Create a basic block for initialization code. */
6154
6155	static basic_block
6156	construct_init_block (void)
6157	{
6158	basic_block init_block, first_block;
6159	edge e = NULL;
6160	int flags;
6161
6162	/* Multiple entry points not supported yet. */
6163	gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6164	init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6165	init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6166	ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6167	EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6168
6169	e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6170
6171	/* When entry edge points to first basic block, we don't need jump,
6172	otherwise we have to jump into proper target. */
6173	if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6174	{
6175	tree label = gimple_block_label (e->dest);
6176
6177	emit_jump (jump_target_rtx (label));
6178	flags = 0;
6179	}
6180	else
6181	flags = EDGE_FALLTHRU;
6182
6183	init_block = create_basic_block (NEXT_INSN (insn: get_insns ()),
6184	get_last_insn (),
6185	ENTRY_BLOCK_PTR_FOR_FN (cfun));
6186	init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6187	add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6188	if (e)
6189	{
6190	first_block = e->dest;
6191	redirect_edge_succ (e, init_block);
6192	make_single_succ_edge (init_block, first_block, flags);
6193	}
6194	else
6195	make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6196	EDGE_FALLTHRU);
6197
6198	update_bb_for_insn (init_block);
6199	return init_block;
6200	}
6201
6202	/* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6203	found in the block tree. */
6204
6205	static void
6206	set_block_levels (tree block, int level)
6207	{
6208	while (block)
6209	{
6210	BLOCK_NUMBER (block) = level;
6211	set_block_levels (BLOCK_SUBBLOCKS (block), level: level + 1);
6212	block = BLOCK_CHAIN (block);
6213	}
6214	}
6215
6216	/* Create a block containing landing pads and similar stuff. */
6217
6218	static void
6219	construct_exit_block (void)
6220	{
6221	rtx_insn *head = get_last_insn ();
6222	rtx_insn *end;
6223	basic_block exit_block;
6224	edge e, e2;
6225	unsigned ix;
6226	edge_iterator ei;
6227	basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6228	rtx_insn *orig_end = BB_END (prev_bb);
6229
6230	rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6231
6232	/* Make sure the locus is set to the end of the function, so that
6233	epilogue line numbers and warnings are set properly. */
6234	if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6235	input_location = cfun->function_end_locus;
6236
6237	/* Generate rtl for function exit. */
6238	expand_function_end ();
6239
6240	end = get_last_insn ();
6241	if (head == end)
6242	return;
6243	/* While emitting the function end we could move end of the last basic
6244	block. */
6245	BB_END (prev_bb) = orig_end;
6246	while (NEXT_INSN (insn: head) && NOTE_P (NEXT_INSN (head)))
6247	head = NEXT_INSN (insn: head);
6248	/* But make sure exit_block starts with RETURN_LABEL, otherwise the
6249	bb count counting will be confused. Any instructions before that
6250	label are emitted for the case where PREV_BB falls through into the
6251	exit block, so append those instructions to prev_bb in that case. */
6252	if (NEXT_INSN (insn: head) != return_label)
6253	{
6254	while (NEXT_INSN (insn: head) != return_label)
6255	{
6256	if (!NOTE_P (NEXT_INSN (head)))
6257	BB_END (prev_bb) = NEXT_INSN (insn: head);
6258	head = NEXT_INSN (insn: head);
6259	}
6260	}
6261	exit_block = create_basic_block (NEXT_INSN (insn: head), end, prev_bb);
6262	exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6263	add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6264
6265	ix = 0;
6266	while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6267	{
6268	e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6269	if (!(e->flags & EDGE_ABNORMAL))
6270	redirect_edge_succ (e, exit_block);
6271	else
6272	ix++;
6273	}
6274
6275	e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6276	EDGE_FALLTHRU);
6277	FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6278	if (e2 != e)
6279	{
6280	exit_block->count -= e2->count ();
6281	}
6282	update_bb_for_insn (exit_block);
6283	}
6284
6285	/* Helper function for discover_nonconstant_array_refs.
6286	Look for ARRAY_REF nodes with non-constant indexes and mark them
6287	addressable. */
6288
6289	static tree
6290	discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6291	void *data)
6292	{
6293	tree t = *tp;
6294	bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
6295
6296	if (IS_TYPE_OR_DECL_P (t))
6297	*walk_subtrees = 0;
6298	else if (REFERENCE_CLASS_P (t) && TREE_THIS_VOLATILE (t))
6299	{
6300	t = get_base_address (t);
6301	if (t && DECL_P (t)
6302	&& DECL_MODE (t) != BLKmode
6303	&& !TREE_ADDRESSABLE (t))
6304	bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6305	*walk_subtrees = 0;
6306	}
6307	else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6308	{
6309	while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6310	&& is_gimple_min_invariant (TREE_OPERAND (t, 1))
6311	&& (!TREE_OPERAND (t, 2)
6312	|| is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6313	|| (TREE_CODE (t) == COMPONENT_REF
6314	&& (!TREE_OPERAND (t,2)
6315	|| is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6316	|| TREE_CODE (t) == BIT_FIELD_REF
6317	|| TREE_CODE (t) == REALPART_EXPR
6318	|| TREE_CODE (t) == IMAGPART_EXPR
6319	|| TREE_CODE (t) == VIEW_CONVERT_EXPR
6320	|| CONVERT_EXPR_P (t))
6321	t = TREE_OPERAND (t, 0);
6322
6323	if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6324	{
6325	t = get_base_address (t);
6326	if (t && DECL_P (t)
6327	&& DECL_MODE (t) != BLKmode
6328	&& !TREE_ADDRESSABLE (t))
6329	bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6330	}
6331
6332	*walk_subtrees = 0;
6333	}
6334	/* References of size POLY_INT_CST to a fixed-size object must go
6335	through memory. It's more efficient to force that here than
6336	to create temporary slots on the fly.
6337	RTL expansion expectes TARGET_MEM_REF to always address actual memory. */
6338	else if (TREE_CODE (t) == TARGET_MEM_REF
6339	|| (TREE_CODE (t) == MEM_REF
6340	&& TYPE_SIZE (TREE_TYPE (t))
6341	&& POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t)))))
6342	{
6343	tree base = get_base_address (t);
6344	if (base
6345	&& DECL_P (base)
6346	&& !TREE_ADDRESSABLE (base)
6347	&& DECL_MODE (base) != BLKmode
6348	&& GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6349	bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6350	*walk_subtrees = 0;
6351	}
6352
6353	return NULL_TREE;
6354	}
6355
6356	/* If there's a chance to get a pseudo for t then if it would be of float mode
6357	and the actual access is via an integer mode (lowered memcpy or similar
6358	access) then avoid the register expansion if the mode likely is not storage
6359	suitable for raw bits processing (like XFmode on i?86). */
6360
6361	static void
6362	avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
6363	{
6364	machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6365	if (access_mode != BLKmode
6366	&& !SCALAR_INT_MODE_P (access_mode))
6367	return;
6368	tree base = get_base_address (t);
6369	if (DECL_P (base)
6370	&& !TREE_ADDRESSABLE (base)
6371	&& FLOAT_MODE_P (DECL_MODE (base))
6372	&& maybe_lt (a: GET_MODE_PRECISION (DECL_MODE (base)),
6373	b: GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6374	/* Double check in the expensive way we really would get a pseudo. */
6375	&& use_register_for_decl (base))
6376	bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6377	}
6378
6379	/* RTL expansion is not able to compile array references with variable
6380	offsets for arrays stored in single register. Discover such
6381	expressions and mark variables as addressable to avoid this
6382	scenario. */
6383
6384	static void
6385	discover_nonconstant_array_refs (bitmap forced_stack_vars)
6386	{
6387	basic_block bb;
6388	gimple_stmt_iterator gsi;
6389
6390	walk_stmt_info wi = {};
6391	wi.info = forced_stack_vars;
6392	FOR_EACH_BB_FN (bb, cfun)
6393	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
6394	{
6395	gimple *stmt = gsi_stmt (i: gsi);
6396	if (!is_gimple_debug (gs: stmt))
6397	{
6398	walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
6399	gcall *call = dyn_cast <gcall *> (p: stmt);
6400	if (call && gimple_call_internal_p (gs: call))
6401	{
6402	tree cand = NULL_TREE;
6403	switch (gimple_call_internal_fn (gs: call))
6404	{
6405	case IFN_LOAD_LANES:
6406	/* The source must be a MEM. */
6407	cand = gimple_call_arg (gs: call, index: 0);
6408	break;
6409	case IFN_STORE_LANES:
6410	/* The destination must be a MEM. */
6411	cand = gimple_call_lhs (gs: call);
6412	break;
6413	default:
6414	break;
6415	}
6416	if (cand)
6417	cand = get_base_address (t: cand);
6418	if (cand
6419	&& DECL_P (cand)
6420	&& use_register_for_decl (cand))
6421	bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
6422	}
6423	if (gimple_vdef (g: stmt))
6424	{
6425	tree t = gimple_get_lhs (stmt);
6426	if (t && REFERENCE_CLASS_P (t))
6427	avoid_type_punning_on_regs (t, forced_stack_vars);
6428	}
6429	}
6430	}
6431	}
6432
6433	/* This function sets crtl->args.internal_arg_pointer to a virtual
6434	register if DRAP is needed. Local register allocator will replace
6435	virtual_incoming_args_rtx with the virtual register. */
6436
6437	static void
6438	expand_stack_alignment (void)
6439	{
6440	rtx drap_rtx;
6441	unsigned int preferred_stack_boundary;
6442
6443	if (! SUPPORTS_STACK_ALIGNMENT)
6444	return;
6445
6446	if (cfun->calls_alloca
6447	|| cfun->has_nonlocal_label
6448	|| crtl->has_nonlocal_goto)
6449	crtl->need_drap = true;
6450
6451	/* Call update_stack_boundary here again to update incoming stack
6452	boundary. It may set incoming stack alignment to a different
6453	value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6454	use the minimum incoming stack alignment to check if it is OK
6455	to perform sibcall optimization since sibcall optimization will
6456	only align the outgoing stack to incoming stack boundary. */
6457	if (targetm.calls.update_stack_boundary)
6458	targetm.calls.update_stack_boundary ();
6459
6460	/* The incoming stack frame has to be aligned at least at
6461	parm_stack_boundary. */
6462	gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6463
6464	/* Update crtl->stack_alignment_estimated and use it later to align
6465	stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6466	exceptions since callgraph doesn't collect incoming stack alignment
6467	in this case. */
6468	if (cfun->can_throw_non_call_exceptions
6469	&& PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6470	preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6471	else
6472	preferred_stack_boundary = crtl->preferred_stack_boundary;
6473	if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6474	crtl->stack_alignment_estimated = preferred_stack_boundary;
6475	if (preferred_stack_boundary > crtl->stack_alignment_needed)
6476	crtl->stack_alignment_needed = preferred_stack_boundary;
6477
6478	gcc_assert (crtl->stack_alignment_needed
6479	<= crtl->stack_alignment_estimated);
6480
6481	crtl->stack_realign_needed
6482	= INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6483	crtl->stack_realign_tried = crtl->stack_realign_needed;
6484
6485	crtl->stack_realign_processed = true;
6486
6487	/* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6488	alignment. */
6489	gcc_assert (targetm.calls.get_drap_rtx != NULL);
6490	drap_rtx = targetm.calls.get_drap_rtx ();
6491
6492	/* stack_realign_drap and drap_rtx must match. */
6493	gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6494
6495	/* Do nothing if NULL is returned, which means DRAP is not needed. */
6496	if (drap_rtx != NULL)
6497	{
6498	crtl->args.internal_arg_pointer = drap_rtx;
6499
6500	/* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6501	needed. */
6502	fixup_tail_calls ();
6503	}
6504	}
6505
6506
6507	static void
6508	expand_main_function (void)
6509	{
6510	#if (defined(INVOKE__main) \
6511	|| (!defined(HAS_INIT_SECTION) \
6512	&& !defined(INIT_SECTION_ASM_OP) \
6513	&& !defined(INIT_ARRAY_SECTION_ASM_OP)))
6514	emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6515	#endif
6516	}
6517
6518
6519	/* Expand code to initialize the stack_protect_guard. This is invoked at
6520	the beginning of a function to be protected. */
6521
6522	static void
6523	stack_protect_prologue (void)
6524	{
6525	tree guard_decl = targetm.stack_protect_guard ();
6526	rtx x, y;
6527
6528	crtl->stack_protect_guard_decl = guard_decl;
6529	x = expand_normal (crtl->stack_protect_guard);
6530
6531	if (targetm.have_stack_protect_combined_set () && guard_decl)
6532	{
6533	gcc_assert (DECL_P (guard_decl));
6534	y = DECL_RTL (guard_decl);
6535
6536	/* Allow the target to compute address of Y and copy it to X without
6537	leaking Y into a register. This combined address + copy pattern
6538	allows the target to prevent spilling of any intermediate results by
6539	splitting it after register allocator. */
6540	if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6541	{
6542	emit_insn (insn);
6543	return;
6544	}
6545	}
6546
6547	if (guard_decl)
6548	y = expand_normal (exp: guard_decl);
6549	else
6550	y = const0_rtx;
6551
6552	/* Allow the target to copy from Y to X without leaking Y into a
6553	register. */
6554	if (targetm.have_stack_protect_set ())
6555	if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
6556	{
6557	emit_insn (insn);
6558	return;
6559	}
6560
6561	/* Otherwise do a straight move. */
6562	emit_move_insn (x, y);
6563	}
6564
6565	/* Translate the intermediate representation contained in the CFG
6566	from GIMPLE trees to RTL.
6567
6568	We do conversion per basic block and preserve/update the tree CFG.
6569	This implies we have to do some magic as the CFG can simultaneously
6570	consist of basic blocks containing RTL and GIMPLE trees. This can
6571	confuse the CFG hooks, so be careful to not manipulate CFG during
6572	the expansion. */
6573
6574	namespace {
6575
6576	const pass_data pass_data_expand =
6577	{
6578	.type: RTL_PASS, /* type */
6579	.name: "expand", /* name */
6580	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
6581	.tv_id: TV_EXPAND, /* tv_id */
6582	.properties_required: ( PROP_ssa | PROP_gimple_leh | PROP_cfg
6583	| PROP_gimple_lcx
6584	| PROP_gimple_lvec
6585	| PROP_gimple_lva), /* properties_required */
6586	PROP_rtl, /* properties_provided */
6587	.properties_destroyed: ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
6588	.todo_flags_start: 0, /* todo_flags_start */
6589	.todo_flags_finish: 0, /* todo_flags_finish */
6590	};
6591
6592	class pass_expand : public rtl_opt_pass
6593	{
6594	public:
6595	pass_expand (gcc::context *ctxt)
6596	: rtl_opt_pass (pass_data_expand, ctxt)
6597	{}
6598
6599	/* opt_pass methods: */
6600	unsigned int execute (function *) final override;
6601
6602	}; // class pass_expand
6603
6604	unsigned int
6605	pass_expand::execute (function *fun)
6606	{
6607	basic_block bb, init_block;
6608	edge_iterator ei;
6609	edge e;
6610	rtx_insn *var_seq, *var_ret_seq;
6611	unsigned i;
6612
6613	timevar_push (tv: TV_OUT_OF_SSA);
6614	rewrite_out_of_ssa (sa: &SA);
6615	timevar_pop (tv: TV_OUT_OF_SSA);
6616	SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
6617
6618	if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
6619	{
6620	gimple_stmt_iterator gsi;
6621	FOR_EACH_BB_FN (bb, cfun)
6622	for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
6623	if (gimple_debug_bind_p (s: gsi_stmt (i: gsi)))
6624	avoid_deep_ter_for_debug (stmt: gsi_stmt (i: gsi), depth: 0);
6625	}
6626
6627	/* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
6628	auto_bitmap forced_stack_vars;
6629	discover_nonconstant_array_refs (forced_stack_vars);
6630
6631	/* Make sure all values used by the optimization passes have sane
6632	defaults. */
6633	reg_renumber = 0;
6634
6635	/* Some backends want to know that we are expanding to RTL. */
6636	currently_expanding_to_rtl = 1;
6637	/* Dominators are not kept up-to-date as we may create new basic-blocks. */
6638	free_dominance_info (CDI_DOMINATORS);
6639
6640	rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
6641
6642	insn_locations_init ();
6643	if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
6644	{
6645	/* Eventually, all FEs should explicitly set function_start_locus. */
6646	if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
6647	set_curr_insn_location
6648	(DECL_SOURCE_LOCATION (current_function_decl));
6649	else
6650	set_curr_insn_location (fun->function_start_locus);
6651	}
6652	else
6653	set_curr_insn_location (UNKNOWN_LOCATION);
6654	prologue_location = curr_insn_location ();
6655
6656	#ifdef INSN_SCHEDULING
6657	init_sched_attrs ();
6658	#endif
6659
6660	/* Make sure first insn is a note even if we don't want linenums.
6661	This makes sure the first insn will never be deleted.
6662	Also, final expects a note to appear there. */
6663	emit_note (NOTE_INSN_DELETED);
6664
6665	targetm.expand_to_rtl_hook ();
6666	crtl->init_stack_alignment ();
6667	fun->cfg->max_jumptable_ents = 0;
6668
6669	/* Resovle the function section. Some targets, like ARM EABI rely on knowledge
6670	of the function section at exapnsion time to predict distance of calls. */
6671	resolve_unique_section (current_function_decl, 0, flag_function_sections);
6672
6673	/* Expand the variables recorded during gimple lowering. */
6674	timevar_push (tv: TV_VAR_EXPAND);
6675	start_sequence ();
6676
6677	var_ret_seq = expand_used_vars (forced_stack_vars);
6678
6679	var_seq = get_insns ();
6680	end_sequence ();
6681	timevar_pop (tv: TV_VAR_EXPAND);
6682
6683	/* Honor stack protection warnings. */
6684	if (warn_stack_protect)
6685	{
6686	if (fun->calls_alloca)
6687	warning (OPT_Wstack_protector,
6688	"stack protector not protecting local variables: "
6689	"variable length buffer");
6690	if (has_short_buffer && !crtl->stack_protect_guard)
6691	warning (OPT_Wstack_protector,
6692	"stack protector not protecting function: "
6693	"all local arrays are less than %d bytes long",
6694	(int) param_ssp_buffer_size);
6695	}
6696
6697	/* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
6698	memory addressable so expand_function_start can emit the required
6699	copies. */
6700	auto_vec<tree, 16> marked_parms;
6701	for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
6702	parm = DECL_CHAIN (parm))
6703	if (!TREE_ADDRESSABLE (parm)
6704	&& bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
6705	{
6706	TREE_ADDRESSABLE (parm) = 1;
6707	marked_parms.safe_push (obj: parm);
6708	}
6709	if (DECL_RESULT (current_function_decl)
6710	&& !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
6711	&& bitmap_bit_p (forced_stack_vars,
6712	DECL_UID (DECL_RESULT (current_function_decl))))
6713	{
6714	TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
6715	marked_parms.safe_push (DECL_RESULT (current_function_decl));
6716	}
6717
6718	/* Set up parameters and prepare for return, for the function. */
6719	expand_function_start (current_function_decl);
6720
6721	/* Clear TREE_ADDRESSABLE again. */
6722	while (!marked_parms.is_empty ())
6723	TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
6724
6725	/* If we emitted any instructions for setting up the variables,
6726	emit them before the FUNCTION_START note. */
6727	if (var_seq)
6728	{
6729	emit_insn_before (var_seq, parm_birth_insn);
6730
6731	/* In expand_function_end we'll insert the alloca save/restore
6732	before parm_birth_insn. We've just insertted an alloca call.
6733	Adjust the pointer to match. */
6734	parm_birth_insn = var_seq;
6735	}
6736
6737	/* Now propagate the RTL assignment of each partition to the
6738	underlying var of each SSA_NAME. */
6739	tree name;
6740
6741	FOR_EACH_SSA_NAME (i, name, cfun)
6742	{
6743	/* We might have generated new SSA names in
6744	update_alias_info_with_stack_vars. They will have a NULL
6745	defining statements, and won't be part of the partitioning,
6746	so ignore those. */
6747	if (!SSA_NAME_DEF_STMT (name))
6748	continue;
6749
6750	adjust_one_expanded_partition_var (var: name);
6751	}
6752
6753	/* Clean up RTL of variables that straddle across multiple
6754	partitions, and check that the rtl of any PARM_DECLs that are not
6755	cleaned up is that of their default defs. */
6756	FOR_EACH_SSA_NAME (i, name, cfun)
6757	{
6758	int part;
6759
6760	/* We might have generated new SSA names in
6761	update_alias_info_with_stack_vars. They will have a NULL
6762	defining statements, and won't be part of the partitioning,
6763	so ignore those. */
6764	if (!SSA_NAME_DEF_STMT (name))
6765	continue;
6766	part = var_to_partition (map: SA.map, var: name);
6767	if (part == NO_PARTITION)
6768	continue;
6769
6770	/* If this decl was marked as living in multiple places, reset
6771	this now to NULL. */
6772	tree var = SSA_NAME_VAR (name);
6773	if (var && DECL_RTL_IF_SET (var) == pc_rtx)
6774	SET_DECL_RTL (var, NULL);
6775	/* Check that the pseudos chosen by assign_parms are those of
6776	the corresponding default defs. */
6777	else if (SSA_NAME_IS_DEFAULT_DEF (name)
6778	&& (TREE_CODE (var) == PARM_DECL
6779	|| TREE_CODE (var) == RESULT_DECL))
6780	{
6781	rtx in = DECL_RTL_IF_SET (var);
6782	gcc_assert (in);
6783	rtx out = SA.partition_to_pseudo[part];
6784	gcc_assert (in == out);
6785
6786	/* Now reset VAR's RTL to IN, so that the _EXPR attrs match
6787	those expected by debug backends for each parm and for
6788	the result. This is particularly important for stabs,
6789	whose register elimination from parm's DECL_RTL may cause
6790	-fcompare-debug differences as SET_DECL_RTL changes reg's
6791	attrs. So, make sure the RTL already has the parm as the
6792	EXPR, so that it won't change. */
6793	SET_DECL_RTL (var, NULL_RTX);
6794	if (MEM_P (in))
6795	set_mem_attributes (in, var, true);
6796	SET_DECL_RTL (var, in);
6797	}
6798	}
6799
6800	/* If this function is `main', emit a call to `__main'
6801	to run global initializers, etc. */
6802	if (DECL_NAME (current_function_decl)
6803	&& MAIN_NAME_P (DECL_NAME (current_function_decl))
6804	&& DECL_FILE_SCOPE_P (current_function_decl))
6805	expand_main_function ();
6806
6807	/* Initialize the stack_protect_guard field. This must happen after the
6808	call to __main (if any) so that the external decl is initialized. */
6809	if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
6810	stack_protect_prologue ();
6811
6812	expand_phi_nodes (sa: &SA);
6813
6814	/* Release any stale SSA redirection data. */
6815	redirect_edge_var_map_empty ();
6816
6817	/* Register rtl specific functions for cfg. */
6818	rtl_register_cfg_hooks ();
6819
6820	init_block = construct_init_block ();
6821
6822	/* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
6823	remaining edges later. */
6824	FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
6825	e->flags &= ~EDGE_EXECUTABLE;
6826
6827	/* If the function has too many markers, drop them while expanding. */
6828	if (cfun->debug_marker_count
6829	>= param_max_debug_marker_count)
6830	cfun->debug_nonbind_markers = false;
6831
6832	lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
6833	FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
6834	next_bb)
6835	bb = expand_gimple_basic_block (bb, disable_tail_calls: var_ret_seq != NULL_RTX);
6836
6837	if (MAY_HAVE_DEBUG_BIND_INSNS)
6838	expand_debug_locations ();
6839
6840	if (deep_ter_debug_map)
6841	{
6842	delete deep_ter_debug_map;
6843	deep_ter_debug_map = NULL;
6844	}
6845
6846	/* Free stuff we no longer need after GIMPLE optimizations. */
6847	free_dominance_info (CDI_DOMINATORS);
6848	free_dominance_info (CDI_POST_DOMINATORS);
6849	delete_tree_cfg_annotations (fun);
6850
6851	timevar_push (tv: TV_OUT_OF_SSA);
6852	finish_out_of_ssa (sa: &SA);
6853	timevar_pop (tv: TV_OUT_OF_SSA);
6854
6855	timevar_push (tv: TV_POST_EXPAND);
6856	/* We are no longer in SSA form. */
6857	fun->gimple_df->in_ssa_p = false;
6858	loops_state_clear (flags: LOOP_CLOSED_SSA);
6859
6860	/* Expansion is used by optimization passes too, set maybe_hot_insn_p
6861	conservatively to true until they are all profile aware. */
6862	delete lab_rtx_for_bb;
6863	free_histograms (fun);
6864
6865	construct_exit_block ();
6866	insn_locations_finalize ();
6867
6868	if (var_ret_seq)
6869	{
6870	rtx_insn *after = return_label;
6871	rtx_insn *next = NEXT_INSN (insn: after);
6872	if (next && NOTE_INSN_BASIC_BLOCK_P (next))
6873	after = next;
6874	emit_insn_after (var_ret_seq, after);
6875	}
6876
6877	if (hwasan_sanitize_stack_p ())
6878	hwasan_maybe_emit_frame_base_init ();
6879
6880	/* Zap the tree EH table. */
6881	set_eh_throw_stmt_table (fun, NULL);
6882
6883	/* We need JUMP_LABEL be set in order to redirect jumps, and hence
6884	split edges which edge insertions might do. */
6885	rebuild_jump_labels (get_insns ());
6886
6887	/* If we have a single successor to the entry block, put the pending insns
6888	after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
6889	if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
6890	{
6891	edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
6892	if (e->insns.r)
6893	{
6894	rtx_insn *insns = e->insns.r;
6895	e->insns.r = NULL;
6896	rebuild_jump_labels_chain (insns);
6897	if (NOTE_P (parm_birth_insn)
6898	&& NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
6899	emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
6900	else
6901	emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
6902	}
6903	}
6904
6905	/* Otherwise, as well as for other edges, take the usual way. */
6906	commit_edge_insertions ();
6907
6908	/* We're done expanding trees to RTL. */
6909	currently_expanding_to_rtl = 0;
6910
6911	flush_mark_addressable_queue ();
6912
6913	FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
6914	EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
6915	{
6916	edge e;
6917	edge_iterator ei;
6918	for (ei = ei_start (bb->succs); (e = ei_safe_edge (i: ei)); )
6919	{
6920	/* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
6921	e->flags &= ~EDGE_EXECUTABLE;
6922
6923	/* At the moment not all abnormal edges match the RTL
6924	representation. It is safe to remove them here as
6925	find_many_sub_basic_blocks will rediscover them.
6926	In the future we should get this fixed properly. */
6927	if ((e->flags & EDGE_ABNORMAL)
6928	&& !(e->flags & EDGE_SIBCALL))
6929	remove_edge (e);
6930	else
6931	ei_next (i: &ei);
6932	}
6933	}
6934
6935	auto_sbitmap blocks (last_basic_block_for_fn (fun));
6936	bitmap_ones (blocks);
6937	find_many_sub_basic_blocks (blocks);
6938	purge_all_dead_edges ();
6939
6940	/* After initial rtl generation, call back to finish generating
6941	exception support code. We need to do this before cleaning up
6942	the CFG as the code does not expect dead landing pads. */
6943	if (fun->eh->region_tree != NULL)
6944	finish_eh_generation ();
6945
6946	/* Call expand_stack_alignment after finishing all
6947	updates to crtl->preferred_stack_boundary. */
6948	expand_stack_alignment ();
6949
6950	/* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
6951	function. */
6952	if (crtl->tail_call_emit)
6953	fixup_tail_calls ();
6954
6955	HOST_WIDE_INT patch_area_size, patch_area_entry;
6956	parse_and_check_patch_area (flag_patchable_function_entry, report_error: false,
6957	patch_area_size: &patch_area_size, patch_area_start: &patch_area_entry);
6958
6959	tree patchable_function_entry_attr
6960	= lookup_attribute (attr_name: "patchable_function_entry",
6961	DECL_ATTRIBUTES (cfun->decl));
6962	if (patchable_function_entry_attr)
6963	{
6964	tree pp_val = TREE_VALUE (patchable_function_entry_attr);
6965	tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
6966
6967	patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
6968	patch_area_entry = 0;
6969	if (TREE_CHAIN (pp_val) != NULL_TREE)
6970	{
6971	tree patchable_function_entry_value2
6972	= TREE_VALUE (TREE_CHAIN (pp_val));
6973	patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
6974	}
6975	}
6976
6977	if (patch_area_entry > patch_area_size)
6978	{
6979	if (patch_area_size > 0)
6980	warning (OPT_Wattributes,
6981	"patchable function entry %wu exceeds size %wu",
6982	patch_area_entry, patch_area_size);
6983	patch_area_entry = 0;
6984	}
6985
6986	crtl->patch_area_size = patch_area_size;
6987	crtl->patch_area_entry = patch_area_entry;
6988
6989	/* BB subdivision may have created basic blocks that are only reachable
6990	from unlikely bbs but not marked as such in the profile. */
6991	if (optimize)
6992	propagate_unlikely_bbs_forward ();
6993
6994	/* Remove unreachable blocks, otherwise we cannot compute dominators
6995	which are needed for loop state verification. As a side-effect
6996	this also compacts blocks.
6997	??? We cannot remove trivially dead insns here as for example
6998	the DRAP reg on i?86 is not magically live at this point.
6999	gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
7000	cleanup_cfg (CLEANUP_NO_INSN_DEL);
7001
7002	checking_verify_flow_info ();
7003
7004	/* Initialize pseudos allocated for hard registers. */
7005	emit_initial_value_sets ();
7006
7007	/* And finally unshare all RTL. */
7008	unshare_all_rtl ();
7009
7010	/* There's no need to defer outputting this function any more; we
7011	know we want to output it. */
7012	DECL_DEFER_OUTPUT (current_function_decl) = 0;
7013
7014	/* Now that we're done expanding trees to RTL, we shouldn't have any
7015	more CONCATs anywhere. */
7016	generating_concat_p = 0;
7017
7018	if (dump_file)
7019	{
7020	fprintf (stream: dump_file,
7021	format: "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
7022	/* And the pass manager will dump RTL for us. */
7023	}
7024
7025	/* If we're emitting a nested function, make sure its parent gets
7026	emitted as well. Doing otherwise confuses debug info. */
7027	{
7028	tree parent;
7029	for (parent = DECL_CONTEXT (current_function_decl);
7030	parent != NULL_TREE;
7031	parent = get_containing_scope (parent))
7032	if (TREE_CODE (parent) == FUNCTION_DECL)
7033	TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
7034	}
7035
7036	TREE_ASM_WRITTEN (current_function_decl) = 1;
7037
7038	/* After expanding, the return labels are no longer needed. */
7039	return_label = NULL;
7040	naked_return_label = NULL;
7041
7042	/* After expanding, the tm_restart map is no longer needed. */
7043	if (fun->gimple_df->tm_restart)
7044	fun->gimple_df->tm_restart = NULL;
7045
7046	/* Tag the blocks with a depth number so that change_scope can find
7047	the common parent easily. */
7048	set_block_levels (DECL_INITIAL (fun->decl), level: 0);
7049	default_rtl_profile ();
7050
7051	/* For -dx discard loops now, otherwise IL verify in clean_state will
7052	ICE. */
7053	if (rtl_dump_and_exit)
7054	{
7055	cfun->curr_properties &= ~PROP_loops;
7056	loop_optimizer_finalize ();
7057	}
7058
7059	timevar_pop (tv: TV_POST_EXPAND);
7060
7061	return 0;
7062	}
7063
7064	} // anon namespace
7065
7066	rtl_opt_pass *
7067	make_pass_expand (gcc::context *ctxt)
7068	{
7069	return new pass_expand (ctxt);
7070	}
7071