1	/* Register to Stack convert for GNU compiler.
2	Copyright (C) 1992-2023 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it
7	under the terms of the GNU General Public License as published by
8	the Free Software Foundation; either version 3, or (at your option)
9	any later version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT
12	ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13	or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
14	License for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	/* This pass converts stack-like registers from the "flat register
21	file" model that gcc uses, to a stack convention that the 387 uses.
22
23	* The form of the input:
24
25	On input, the function consists of insn that have had their
26	registers fully allocated to a set of "virtual" registers. Note that
27	the word "virtual" is used differently here than elsewhere in gcc: for
28	each virtual stack reg, there is a hard reg, but the mapping between
29	them is not known until this pass is run. On output, hard register
30	numbers have been substituted, and various pop and exchange insns have
31	been emitted. The hard register numbers and the virtual register
32	numbers completely overlap - before this pass, all stack register
33	numbers are virtual, and afterward they are all hard.
34
35	The virtual registers can be manipulated normally by gcc, and their
36	semantics are the same as for normal registers. After the hard
37	register numbers are substituted, the semantics of an insn containing
38	stack-like regs are not the same as for an insn with normal regs: for
39	instance, it is not safe to delete an insn that appears to be a no-op
40	move. In general, no insn containing hard regs should be changed
41	after this pass is done.
42
43	* The form of the output:
44
45	After this pass, hard register numbers represent the distance from
46	the current top of stack to the desired register. A reference to
47	FIRST_STACK_REG references the top of stack, FIRST_STACK_REG + 1,
48	represents the register just below that, and so forth. Also, REG_DEAD
49	notes indicate whether or not a stack register should be popped.
50
51	A "swap" insn looks like a parallel of two patterns, where each
52	pattern is a SET: one sets A to B, the other B to A.
53
54	A "push" or "load" insn is a SET whose SET_DEST is FIRST_STACK_REG
55	and whose SET_DEST is REG or MEM. Any other SET_DEST, such as PLUS,
56	will replace the existing stack top, not push a new value.
57
58	A store insn is a SET whose SET_DEST is FIRST_STACK_REG, and whose
59	SET_SRC is REG or MEM.
60
61	The case where the SET_SRC and SET_DEST are both FIRST_STACK_REG
62	appears ambiguous. As a special case, the presence of a REG_DEAD note
63	for FIRST_STACK_REG differentiates between a load insn and a pop.
64
65	If a REG_DEAD is present, the insn represents a "pop" that discards
66	the top of the register stack. If there is no REG_DEAD note, then the
67	insn represents a "dup" or a push of the current top of stack onto the
68	stack.
69
70	* Methodology:
71
72	Existing REG_DEAD and REG_UNUSED notes for stack registers are
73	deleted and recreated from scratch. REG_DEAD is never created for a
74	SET_DEST, only REG_UNUSED.
75
76	* asm_operands:
77
78	There are several rules on the usage of stack-like regs in
79	asm_operands insns. These rules apply only to the operands that are
80	stack-like regs:
81
82	1. Given a set of input regs that die in an asm_operands, it is
83	necessary to know which are implicitly popped by the asm, and
84	which must be explicitly popped by gcc.
85
86	An input reg that is implicitly popped by the asm must be
87	explicitly clobbered, unless it is constrained to match an
88	output operand.
89
90	2. For any input reg that is implicitly popped by an asm, it is
91	necessary to know how to adjust the stack to compensate for the pop.
92	If any non-popped input is closer to the top of the reg-stack than
93	the implicitly popped reg, it would not be possible to know what the
94	stack looked like - it's not clear how the rest of the stack "slides
95	up".
96
97	All implicitly popped input regs must be closer to the top of
98	the reg-stack than any input that is not implicitly popped.
99
100	All explicitly referenced input operands may not "skip" a reg.
101	Otherwise we can have holes in the stack.
102
103	3. It is possible that if an input dies in an insn, reload might
104	use the input reg for an output reload. Consider this example:
105
106	asm ("foo" : "=t" (a) : "f" (b));
107
108	This asm says that input B is not popped by the asm, and that
109	the asm pushes a result onto the reg-stack, i.e., the stack is one
110	deeper after the asm than it was before. But, it is possible that
111	reload will think that it can use the same reg for both the input and
112	the output, if input B dies in this insn.
113
114	If any input operand uses the "f" constraint, all output reg
115	constraints must use the "&" earlyclobber.
116
117	The asm above would be written as
118
119	asm ("foo" : "=&t" (a) : "f" (b));
120
121	4. Some operands need to be in particular places on the stack. All
122	output operands fall in this category - there is no other way to
123	know which regs the outputs appear in unless the user indicates
124	this in the constraints.
125
126	Output operands must specifically indicate which reg an output
127	appears in after an asm. "=f" is not allowed: the operand
128	constraints must select a class with a single reg.
129
130	5. Output operands may not be "inserted" between existing stack regs.
131	Since no 387 opcode uses a read/write operand, all output operands
132	are dead before the asm_operands, and are pushed by the asm_operands.
133	It makes no sense to push anywhere but the top of the reg-stack.
134
135	Output operands must start at the top of the reg-stack: output
136	operands may not "skip" a reg.
137
138	6. Some asm statements may need extra stack space for internal
139	calculations. This can be guaranteed by clobbering stack registers
140	unrelated to the inputs and outputs.
141
142	Here are a couple of reasonable asms to want to write. This asm
143	takes one input, which is internally popped, and produces two outputs.
144
145	asm ("fsincos" : "=t" (cos), "=u" (sin) : "0" (inp));
146
147	This asm takes two inputs, which are popped by the fyl2xp1 opcode,
148	and replaces them with one output. The user must code the "st(1)"
149	clobber for reg-stack.cc to know that fyl2xp1 pops both inputs.
150
151	asm ("fyl2xp1" : "=t" (result) : "0" (x), "u" (y) : "st(1)");
152
153	*/
154
155	#include "config.h"
156	#include "system.h"
157	#include "coretypes.h"
158	#include "backend.h"
159	#include "target.h"
160	#include "rtl.h"
161	#include "tree.h"
162	#include "df.h"
163	#include "insn-config.h"
164	#include "memmodel.h"
165	#include "regs.h"
166	#include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
167	#include "recog.h"
168	#include "varasm.h"
169	#include "rtl-error.h"
170	#include "cfgrtl.h"
171	#include "cfganal.h"
172	#include "cfgbuild.h"
173	#include "cfgcleanup.h"
174	#include "reload.h"
175	#include "tree-pass.h"
176	#include "rtl-iter.h"
177	#include "function-abi.h"
178
179	#ifdef STACK_REGS
180
181	/* We use this array to cache info about insns, because otherwise we
182	spend too much time in stack_regs_mentioned_p.
183
184	Indexed by insn UIDs. A value of zero is uninitialized, one indicates
185	the insn uses stack registers, two indicates the insn does not use
186	stack registers. */
187	static vec<char> stack_regs_mentioned_data;
188
189	#define REG_STACK_SIZE (LAST_STACK_REG - FIRST_STACK_REG + 1)
190
191	int regstack_completed = 0;
192
193	/* This is the basic stack record. TOP is an index into REG[] such
194	that REG[TOP] is the top of stack. If TOP is -1 the stack is empty.
195
196	If TOP is -2, REG[] is not yet initialized. Stack initialization
197	consists of placing each live reg in array `reg' and setting `top'
198	appropriately.
199
200	REG_SET indicates which registers are live. */
201
202	typedef struct stack_def
203	{
204	int top; /* index to top stack element */
205	HARD_REG_SET reg_set; /* set of live registers */
206	unsigned char reg[REG_STACK_SIZE];/* register - stack mapping */
207	} *stack_ptr;
208
209	/* This is used to carry information about basic blocks. It is
210	attached to the AUX field of the standard CFG block. */
211
212	typedef struct block_info_def
213	{
214	struct stack_def stack_in; /* Input stack configuration. */
215	struct stack_def stack_out; /* Output stack configuration. */
216	HARD_REG_SET out_reg_set; /* Stack regs live on output. */
217	bool done; /* True if block already converted. */
218	int predecessors; /* Number of predecessors that need
219	to be visited. */
220	} *block_info;
221
222	#define BLOCK_INFO(B) ((block_info) (B)->aux)
223
224	/* Passed to change_stack to indicate where to emit insns. */
225	enum emit_where
226	{
227	EMIT_AFTER,
228	EMIT_BEFORE
229	};
230
231	/* The block we're currently working on. */
232	static basic_block current_block;
233
234	/* In the current_block, whether we're processing the first register
235	stack or call instruction, i.e. the regstack is currently the
236	same as BLOCK_INFO(current_block)->stack_in. */
237	static bool starting_stack_p;
238
239	/* This is the register file for all register after conversion. */
240	static rtx
241	FP_mode_reg[LAST_STACK_REG+1-FIRST_STACK_REG][(int) MAX_MACHINE_MODE];
242
243	#define FP_MODE_REG(regno,mode) \
244	(FP_mode_reg[(regno)-FIRST_STACK_REG][(int) (mode)])
245
246	/* Used to initialize uninitialized registers. */
247	static rtx not_a_num;
248
249	/* Forward declarations */
250
251	static bool stack_regs_mentioned_p (const_rtx pat);
252	static void pop_stack (stack_ptr, int);
253	static rtx *get_true_reg (rtx *);
254
255	static bool check_asm_stack_operands (rtx_insn *);
256	static void get_asm_operands_in_out (rtx, int *, int *);
257	static rtx stack_result (tree);
258	static void replace_reg (rtx *, int);
259	static void remove_regno_note (rtx_insn *, enum reg_note, unsigned int);
260	static int get_hard_regnum (stack_ptr, rtx);
261	static rtx_insn *emit_pop_insn (rtx_insn *, stack_ptr, rtx, enum emit_where);
262	static void swap_to_top (rtx_insn *, stack_ptr, rtx, rtx);
263	static bool move_for_stack_reg (rtx_insn *, stack_ptr, rtx);
264	static bool move_nan_for_stack_reg (rtx_insn *, stack_ptr, rtx);
265	static bool swap_rtx_condition_1 (rtx);
266	static bool swap_rtx_condition (rtx_insn *, int &);
267	static void compare_for_stack_reg (rtx_insn *, stack_ptr, rtx, bool);
268	static bool subst_stack_regs_pat (rtx_insn *, stack_ptr, rtx);
269	static void subst_asm_stack_regs (rtx_insn *, stack_ptr);
270	static bool subst_stack_regs (rtx_insn *, stack_ptr);
271	static void change_stack (rtx_insn *, stack_ptr, stack_ptr, enum emit_where);
272	static void print_stack (FILE *, stack_ptr);
273	static rtx_insn *next_flags_user (rtx_insn *, int &);
274
275	/* Return true if any stack register is mentioned somewhere within PAT. */
276
277	static bool
278	stack_regs_mentioned_p (const_rtx pat)
279	{
280	const char *fmt;
281	int i;
282
283	if (STACK_REG_P (pat))
284	return true;
285
286	fmt = GET_RTX_FORMAT (GET_CODE (pat));
287	for (i = GET_RTX_LENGTH (GET_CODE (pat)) - 1; i >= 0; i--)
288	{
289	if (fmt[i] == 'E')
290	{
291	int j;
292
293	for (j = XVECLEN (pat, i) - 1; j >= 0; j--)
294	if (stack_regs_mentioned_p (XVECEXP (pat, i, j)))
295	return true;
296	}
297	else if (fmt[i] == 'e' && stack_regs_mentioned_p (XEXP (pat, i)))
298	return true;
299	}
300
301	return false;
302	}
303
304	/* Return true if INSN mentions stacked registers, else return zero. */
305
306	bool
307	stack_regs_mentioned (const_rtx insn)
308	{
309	unsigned int uid, max;
310	int test;
311
312	if (! INSN_P (insn) || !stack_regs_mentioned_data.exists ())
313	return false;
314
315	uid = INSN_UID (insn);
316	max = stack_regs_mentioned_data.length ();
317	if (uid >= max)
318	{
319	/* Allocate some extra size to avoid too many reallocs, but
320	do not grow too quickly. */
321	max = uid + uid / 20 + 1;
322	stack_regs_mentioned_data.safe_grow_cleared (len: max, exact: true);
323	}
324
325	test = stack_regs_mentioned_data[uid];
326	if (test == 0)
327	{
328	/* This insn has yet to be examined. Do so now. */
329	test = stack_regs_mentioned_p (pat: PATTERN (insn)) ? 1 : 2;
330	stack_regs_mentioned_data[uid] = test;
331	}
332
333	return test == 1;
334	}
335
336	static rtx ix86_flags_rtx;
337
338	static rtx_insn *
339	next_flags_user (rtx_insn *insn, int &debug_seen)
340	{
341	/* Search forward looking for the first use of this value.
342	Stop at block boundaries. */
343
344	while (insn != BB_END (current_block))
345	{
346	insn = NEXT_INSN (insn);
347
348	if (INSN_P (insn) && reg_mentioned_p (ix86_flags_rtx, PATTERN (insn)))
349	{
350	if (DEBUG_INSN_P (insn) && debug_seen >= 0)
351	{
352	debug_seen = 1;
353	continue;
354	}
355	return insn;
356	}
357
358	if (CALL_P (insn))
359	return NULL;
360	}
361	return NULL;
362	}
363
364	/* Reorganize the stack into ascending numbers, before this insn. */
365
366	static void
367	straighten_stack (rtx_insn *insn, stack_ptr regstack)
368	{
369	struct stack_def temp_stack;
370	int top;
371
372	/* If there is only a single register on the stack, then the stack is
373	already in increasing order and no reorganization is needed.
374
375	Similarly if the stack is empty. */
376	if (regstack->top <= 0)
377	return;
378
379	temp_stack.reg_set = regstack->reg_set;
380
381	for (top = temp_stack.top = regstack->top; top >= 0; top--)
382	temp_stack.reg[top] = FIRST_STACK_REG + temp_stack.top - top;
383
384	change_stack (insn, regstack, &temp_stack, EMIT_BEFORE);
385	}
386
387	/* Pop a register from the stack. */
388
389	static void
390	pop_stack (stack_ptr regstack, int regno)
391	{
392	int top = regstack->top;
393
394	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, bit: regno);
395	regstack->top--;
396	/* If regno was not at the top of stack then adjust stack. */
397	if (regstack->reg [top] != regno)
398	{
399	int i;
400	for (i = regstack->top; i >= 0; i--)
401	if (regstack->reg [i] == regno)
402	{
403	int j;
404	for (j = i; j < top; j++)
405	regstack->reg [j] = regstack->reg [j + 1];
406	break;
407	}
408	}
409	}
410
411	/* Return a pointer to the REG expression within PAT. If PAT is not a
412	REG, possible enclosed by a conversion rtx, return the inner part of
413	PAT that stopped the search. */
414
415	static rtx *
416	get_true_reg (rtx *pat)
417	{
418	for (;;)
419	switch (GET_CODE (*pat))
420	{
421	case SUBREG:
422	/* Eliminate FP subregister accesses in favor of the
423	actual FP register in use. */
424	{
425	rtx subreg = SUBREG_REG (*pat);
426
427	if (STACK_REG_P (subreg))
428	{
429	int regno_off = subreg_regno_offset (REGNO (subreg),
430	GET_MODE (subreg),
431	SUBREG_BYTE (*pat),
432	GET_MODE (*pat));
433	*pat = FP_MODE_REG (REGNO (subreg) + regno_off,
434	GET_MODE (subreg));
435	return pat;
436	}
437	pat = &XEXP (*pat, 0);
438	break;
439	}
440
441	case FLOAT_TRUNCATE:
442	if (!flag_unsafe_math_optimizations)
443	return pat;
444	/* FALLTHRU */
445
446	case FLOAT:
447	case FIX:
448	case FLOAT_EXTEND:
449	pat = &XEXP (*pat, 0);
450	break;
451
452	case UNSPEC:
453	if (XINT (*pat, 1) == UNSPEC_TRUNC_NOOP
454	|| XINT (*pat, 1) == UNSPEC_FILD_ATOMIC)
455	pat = &XVECEXP (*pat, 0, 0);
456	return pat;
457
458	default:
459	return pat;
460	}
461	}
462
463	/* Set if we find any malformed asms in a function. */
464	static bool any_malformed_asm;
465
466	/* There are many rules that an asm statement for stack-like regs must
467	follow. Those rules are explained at the top of this file: the rule
468	numbers below refer to that explanation. */
469
470	static bool
471	check_asm_stack_operands (rtx_insn *insn)
472	{
473	int i;
474	int n_clobbers;
475	bool malformed_asm = false;
476	rtx body = PATTERN (insn);
477
478	char reg_used_as_output[FIRST_PSEUDO_REGISTER];
479	char implicitly_dies[FIRST_PSEUDO_REGISTER];
480	char explicitly_used[FIRST_PSEUDO_REGISTER];
481
482	rtx *clobber_reg = 0;
483	int n_inputs, n_outputs;
484
485	/* Find out what the constraints require. If no constraint
486	alternative matches, this asm is malformed. */
487	extract_constrain_insn (insn);
488
489	preprocess_constraints (insn);
490
491	get_asm_operands_in_out (body, &n_outputs, &n_inputs);
492
493	if (which_alternative < 0)
494	{
495	/* Avoid further trouble with this insn. */
496	PATTERN (insn) = gen_rtx_USE (VOIDmode, const0_rtx);
497	return 0;
498	}
499	const operand_alternative *op_alt = which_op_alt ();
500
501	/* Strip SUBREGs here to make the following code simpler. */
502	for (i = 0; i < recog_data.n_operands; i++)
503	if (GET_CODE (recog_data.operand[i]) == SUBREG
504	&& REG_P (SUBREG_REG (recog_data.operand[i])))
505	recog_data.operand[i] = SUBREG_REG (recog_data.operand[i]);
506
507	/* Set up CLOBBER_REG. */
508
509	n_clobbers = 0;
510
511	if (GET_CODE (body) == PARALLEL)
512	{
513	clobber_reg = XALLOCAVEC (rtx, XVECLEN (body, 0));
514
515	for (i = 0; i < XVECLEN (body, 0); i++)
516	if (GET_CODE (XVECEXP (body, 0, i)) == CLOBBER)
517	{
518	rtx clobber = XVECEXP (body, 0, i);
519	rtx reg = XEXP (clobber, 0);
520
521	if (GET_CODE (reg) == SUBREG && REG_P (SUBREG_REG (reg)))
522	reg = SUBREG_REG (reg);
523
524	if (STACK_REG_P (reg))
525	{
526	clobber_reg[n_clobbers] = reg;
527	n_clobbers++;
528	}
529	}
530	}
531
532	/* Enforce rule #4: Output operands must specifically indicate which
533	reg an output appears in after an asm. "=f" is not allowed: the
534	operand constraints must select a class with a single reg.
535
536	Also enforce rule #5: Output operands must start at the top of
537	the reg-stack: output operands may not "skip" a reg. */
538
539	memset (s: reg_used_as_output, c: 0, n: sizeof (reg_used_as_output));
540	for (i = 0; i < n_outputs; i++)
541	if (STACK_REG_P (recog_data.operand[i]))
542	{
543	if (reg_class_size[(int) op_alt[i].cl] != 1)
544	{
545	error_for_asm (insn, "output constraint %d must specify a single register", i);
546	malformed_asm = true;
547	}
548	else
549	{
550	int j;
551
552	for (j = 0; j < n_clobbers; j++)
553	if (REGNO (recog_data.operand[i]) == REGNO (clobber_reg[j]))
554	{
555	error_for_asm (insn, "output constraint %d cannot be "
556	"specified together with %qs clobber",
557	i, reg_names [REGNO (clobber_reg[j])]);
558	malformed_asm = true;
559	break;
560	}
561	if (j == n_clobbers)
562	reg_used_as_output[REGNO (recog_data.operand[i])] = 1;
563	}
564	}
565
566
567	/* Search for first non-popped reg. */
568	for (i = FIRST_STACK_REG; i < LAST_STACK_REG + 1; i++)
569	if (! reg_used_as_output[i])
570	break;
571
572	/* If there are any other popped regs, that's an error. */
573	for (; i < LAST_STACK_REG + 1; i++)
574	if (reg_used_as_output[i])
575	break;
576
577	if (i != LAST_STACK_REG + 1)
578	{
579	error_for_asm (insn, "output registers must be grouped at top of stack");
580	malformed_asm = true;
581	}
582
583	/* Enforce rule #2: All implicitly popped input regs must be closer
584	to the top of the reg-stack than any input that is not implicitly
585	popped. */
586
587	memset (s: implicitly_dies, c: 0, n: sizeof (implicitly_dies));
588	memset (s: explicitly_used, c: 0, n: sizeof (explicitly_used));
589	for (i = n_outputs; i < n_outputs + n_inputs; i++)
590	if (STACK_REG_P (recog_data.operand[i]))
591	{
592	/* An input reg is implicitly popped if it is tied to an
593	output, or if there is a CLOBBER for it. */
594	int j;
595
596	for (j = 0; j < n_clobbers; j++)
597	if (operands_match_p (clobber_reg[j], recog_data.operand[i]))
598	break;
599
600	if (j < n_clobbers || op_alt[i].matches >= 0)
601	implicitly_dies[REGNO (recog_data.operand[i])] = 1;
602	else if (reg_class_size[(int) op_alt[i].cl] == 1)
603	explicitly_used[REGNO (recog_data.operand[i])] = 1;
604	}
605
606	/* Search for first non-popped reg. */
607	for (i = FIRST_STACK_REG; i < LAST_STACK_REG + 1; i++)
608	if (! implicitly_dies[i])
609	break;
610
611	/* If there are any other popped regs, that's an error. */
612	for (; i < LAST_STACK_REG + 1; i++)
613	if (implicitly_dies[i])
614	break;
615
616	if (i != LAST_STACK_REG + 1)
617	{
618	error_for_asm (insn,
619	"implicitly popped registers must be grouped "
620	"at top of stack");
621	malformed_asm = true;
622	}
623
624	/* Search for first not-explicitly used reg. */
625	for (i = FIRST_STACK_REG; i < LAST_STACK_REG + 1; i++)
626	if (! implicitly_dies[i] && ! explicitly_used[i])
627	break;
628
629	/* If there are any other explicitly used regs, that's an error. */
630	for (; i < LAST_STACK_REG + 1; i++)
631	if (explicitly_used[i])
632	break;
633
634	if (i != LAST_STACK_REG + 1)
635	{
636	error_for_asm (insn,
637	"explicitly used registers must be grouped "
638	"at top of stack");
639	malformed_asm = true;
640	}
641
642	/* Enforce rule #3: If any input operand uses the "f" constraint, all
643	output constraints must use the "&" earlyclobber.
644
645	??? Detect this more deterministically by having constrain_asm_operands
646	record any earlyclobber. */
647
648	for (i = n_outputs; i < n_outputs + n_inputs; i++)
649	if (STACK_REG_P (recog_data.operand[i]) && op_alt[i].matches == -1)
650	{
651	int j;
652
653	for (j = 0; j < n_outputs; j++)
654	if (operands_match_p (recog_data.operand[j], recog_data.operand[i]))
655	{
656	error_for_asm (insn,
657	"output operand %d must use %<&%> constraint", j);
658	malformed_asm = true;
659	}
660	}
661
662	if (malformed_asm)
663	{
664	/* Avoid further trouble with this insn. */
665	PATTERN (insn) = gen_rtx_USE (VOIDmode, const0_rtx);
666	any_malformed_asm = true;
667	return false;
668	}
669
670	return true;
671	}
672
673	/* Calculate the number of inputs and outputs in BODY, an
674	asm_operands. N_OPERANDS is the total number of operands, and
675	N_INPUTS and N_OUTPUTS are pointers to ints into which the results are
676	placed. */
677
678	static void
679	get_asm_operands_in_out (rtx body, int *pout, int *pin)
680	{
681	rtx asmop = extract_asm_operands (body);
682
683	*pin = ASM_OPERANDS_INPUT_LENGTH (asmop);
684	*pout = (recog_data.n_operands
685	- ASM_OPERANDS_INPUT_LENGTH (asmop)
686	- ASM_OPERANDS_LABEL_LENGTH (asmop));
687	}
688
689	/* If current function returns its result in an fp stack register,
690	return the REG. Otherwise, return 0. */
691
692	static rtx
693	stack_result (tree decl)
694	{
695	rtx result;
696
697	/* If the value is supposed to be returned in memory, then clearly
698	it is not returned in a stack register. */
699	if (aggregate_value_p (DECL_RESULT (decl), decl))
700	return 0;
701
702	result = DECL_RTL_IF_SET (DECL_RESULT (decl));
703	if (result != 0)
704	result = targetm.calls.function_value (TREE_TYPE (DECL_RESULT (decl)),
705	decl, true);
706
707	return result != 0 && STACK_REG_P (result) ? result : 0;
708	}
709
710
711	/*
712	* This section deals with stack register substitution, and forms the second
713	* pass over the RTL.
714	*/
715
716	/* Replace REG, which is a pointer to a stack reg RTX, with an RTX for
717	the desired hard REGNO. */
718
719	static void
720	replace_reg (rtx *reg, int regno)
721	{
722	gcc_assert (IN_RANGE (regno, FIRST_STACK_REG, LAST_STACK_REG));
723	gcc_assert (STACK_REG_P (*reg));
724
725	gcc_assert (GET_MODE_CLASS (GET_MODE (*reg)) == MODE_FLOAT
726	|| GET_MODE_CLASS (GET_MODE (*reg)) == MODE_COMPLEX_FLOAT);
727
728	*reg = FP_MODE_REG (regno, GET_MODE (*reg));
729	}
730
731	/* Remove a note of type NOTE, which must be found, for register
732	number REGNO from INSN. Remove only one such note. */
733
734	static void
735	remove_regno_note (rtx_insn *insn, enum reg_note note, unsigned int regno)
736	{
737	rtx *note_link, this_rtx;
738
739	note_link = &REG_NOTES (insn);
740	for (this_rtx = *note_link; this_rtx; this_rtx = XEXP (this_rtx, 1))
741	if (REG_NOTE_KIND (this_rtx) == note
742	&& REG_P (XEXP (this_rtx, 0)) && REGNO (XEXP (this_rtx, 0)) == regno)
743	{
744	*note_link = XEXP (this_rtx, 1);
745	return;
746	}
747	else
748	note_link = &XEXP (this_rtx, 1);
749
750	gcc_unreachable ();
751	}
752
753	/* Find the hard register number of virtual register REG in REGSTACK.
754	The hard register number is relative to the top of the stack. -1 is
755	returned if the register is not found. */
756
757	static int
758	get_hard_regnum (stack_ptr regstack, rtx reg)
759	{
760	int i;
761
762	gcc_assert (STACK_REG_P (reg));
763
764	for (i = regstack->top; i >= 0; i--)
765	if (regstack->reg[i] == REGNO (reg))
766	break;
767
768	return i >= 0 ? (FIRST_STACK_REG + regstack->top - i) : -1;
769	}
770
771	/* Emit an insn to pop virtual register REG before or after INSN.
772	REGSTACK is the stack state after INSN and is updated to reflect this
773	pop. WHEN is either emit_insn_before or emit_insn_after. A pop insn
774	is represented as a SET whose destination is the register to be popped
775	and source is the top of stack. A death note for the top of stack
776	cases the movdf pattern to pop. */
777
778	static rtx_insn *
779	emit_pop_insn (rtx_insn *insn, stack_ptr regstack, rtx reg,
780	enum emit_where where)
781	{
782	machine_mode raw_mode = reg_raw_mode[FIRST_STACK_REG];
783	rtx_insn *pop_insn;
784	rtx pop_rtx;
785	int hard_regno;
786
787	/* For complex types take care to pop both halves. These may survive in
788	CLOBBER and USE expressions. */
789	if (COMPLEX_MODE_P (GET_MODE (reg)))
790	{
791	rtx reg1 = FP_MODE_REG (REGNO (reg), raw_mode);
792	rtx reg2 = FP_MODE_REG (REGNO (reg) + 1, raw_mode);
793
794	pop_insn = NULL;
795	if (get_hard_regnum (regstack, reg: reg1) >= 0)
796	pop_insn = emit_pop_insn (insn, regstack, reg: reg1, where);
797	if (get_hard_regnum (regstack, reg: reg2) >= 0)
798	pop_insn = emit_pop_insn (insn, regstack, reg: reg2, where);
799	gcc_assert (pop_insn);
800	return pop_insn;
801	}
802
803	hard_regno = get_hard_regnum (regstack, reg);
804
805	gcc_assert (hard_regno >= FIRST_STACK_REG);
806
807	pop_rtx = gen_rtx_SET (FP_MODE_REG (hard_regno, raw_mode),
808	FP_MODE_REG (FIRST_STACK_REG, raw_mode));
809
810	if (where == EMIT_AFTER)
811	pop_insn = emit_insn_after (pop_rtx, insn);
812	else
813	pop_insn = emit_insn_before (pop_rtx, insn);
814
815	add_reg_note (pop_insn, REG_DEAD, FP_MODE_REG (FIRST_STACK_REG, raw_mode));
816
817	regstack->reg[regstack->top - (hard_regno - FIRST_STACK_REG)]
818	= regstack->reg[regstack->top];
819	regstack->top -= 1;
820	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, REGNO (reg));
821
822	return pop_insn;
823	}
824
825	/* Emit an insn before or after INSN to swap virtual register REG with
826	the top of stack. REGSTACK is the stack state before the swap, and
827	is updated to reflect the swap. A swap insn is represented as a
828	PARALLEL of two patterns: each pattern moves one reg to the other.
829
830	If REG is already at the top of the stack, no insn is emitted. */
831
832	static void
833	emit_swap_insn (rtx_insn *insn, stack_ptr regstack, rtx reg)
834	{
835	int hard_regno;
836	int other_reg; /* swap regno temps */
837	rtx_insn *i1; /* the stack-reg insn prior to INSN */
838	rtx i1set = NULL_RTX; /* the SET rtx within I1 */
839
840	hard_regno = get_hard_regnum (regstack, reg);
841
842	if (hard_regno == FIRST_STACK_REG)
843	return;
844	if (hard_regno == -1)
845	{
846	/* Something failed if the register wasn't on the stack. If we had
847	malformed asms, we zapped the instruction itself, but that didn't
848	produce the same pattern of register sets as before. To prevent
849	further failure, adjust REGSTACK to include REG at TOP. */
850	gcc_assert (any_malformed_asm);
851	regstack->reg[++regstack->top] = REGNO (reg);
852	return;
853	}
854	gcc_assert (hard_regno >= FIRST_STACK_REG);
855
856	other_reg = regstack->top - (hard_regno - FIRST_STACK_REG);
857	std::swap (a&: regstack->reg[regstack->top], b&: regstack->reg[other_reg]);
858
859	/* Find the previous insn involving stack regs, but don't pass a
860	block boundary. */
861	i1 = NULL;
862	if (current_block && insn != BB_HEAD (current_block))
863	{
864	rtx_insn *tmp = PREV_INSN (insn);
865	rtx_insn *limit = PREV_INSN (BB_HEAD (current_block));
866	while (tmp != limit)
867	{
868	if (LABEL_P (tmp)
869	|| CALL_P (tmp)
870	|| NOTE_INSN_BASIC_BLOCK_P (tmp)
871	|| (NONJUMP_INSN_P (tmp)
872	&& stack_regs_mentioned (insn: tmp)))
873	{
874	i1 = tmp;
875	break;
876	}
877	tmp = PREV_INSN (insn: tmp);
878	}
879	}
880
881	if (i1 != NULL_RTX
882	&& (i1set = single_set (insn: i1)) != NULL_RTX)
883	{
884	rtx i1src = *get_true_reg (pat: &SET_SRC (i1set));
885	rtx i1dest = *get_true_reg (pat: &SET_DEST (i1set));
886
887	/* If the previous register stack push was from the reg we are to
888	swap with, omit the swap. */
889
890	if (REG_P (i1dest) && REGNO (i1dest) == FIRST_STACK_REG
891	&& REG_P (i1src)
892	&& REGNO (i1src) == (unsigned) hard_regno - 1
893	&& find_regno_note (i1, REG_DEAD, FIRST_STACK_REG) == NULL_RTX)
894	return;
895
896	/* If the previous insn wrote to the reg we are to swap with,
897	omit the swap. */
898
899	if (REG_P (i1dest) && REGNO (i1dest) == (unsigned) hard_regno
900	&& REG_P (i1src) && REGNO (i1src) == FIRST_STACK_REG
901	&& find_regno_note (i1, REG_DEAD, FIRST_STACK_REG) == NULL_RTX)
902	return;
903
904	/* Instead of
905	fld a
906	fld b
907	fxch %st(1)
908	just use
909	fld b
910	fld a
911	if possible. Similarly for fld1, fldz, fldpi etc. instead of any
912	of the loads or for float extension from memory. */
913
914	i1src = SET_SRC (i1set);
915	if (GET_CODE (i1src) == FLOAT_EXTEND)
916	i1src = XEXP (i1src, 0);
917	if (REG_P (i1dest)
918	&& REGNO (i1dest) == FIRST_STACK_REG
919	&& (MEM_P (i1src) || GET_CODE (i1src) == CONST_DOUBLE)
920	&& !side_effects_p (i1src)
921	&& hard_regno == FIRST_STACK_REG + 1
922	&& i1 != BB_HEAD (current_block))
923	{
924	/* i1 is the last insn that involves stack regs before insn, and
925	is known to be a load without other side-effects, i.e. fld b
926	in the above comment. */
927	rtx_insn *i2 = NULL;
928	rtx i2set;
929	rtx_insn *tmp = PREV_INSN (insn: i1);
930	rtx_insn *limit = PREV_INSN (BB_HEAD (current_block));
931	/* Find the previous insn involving stack regs, but don't pass a
932	block boundary. */
933	while (tmp != limit)
934	{
935	if (LABEL_P (tmp)
936	|| CALL_P (tmp)
937	|| NOTE_INSN_BASIC_BLOCK_P (tmp)
938	|| (NONJUMP_INSN_P (tmp)
939	&& stack_regs_mentioned (insn: tmp)))
940	{
941	i2 = tmp;
942	break;
943	}
944	tmp = PREV_INSN (insn: tmp);
945	}
946	if (i2 != NULL_RTX
947	&& (i2set = single_set (insn: i2)) != NULL_RTX)
948	{
949	rtx i2dest = *get_true_reg (pat: &SET_DEST (i2set));
950	rtx i2src = SET_SRC (i2set);
951	if (GET_CODE (i2src) == FLOAT_EXTEND)
952	i2src = XEXP (i2src, 0);
953	/* If the last two insns before insn that involve
954	stack regs are loads, where the latter (i1)
955	pushes onto the register stack and thus
956	moves the value from the first load (i2) from
957	%st to %st(1), consider swapping them. */
958	if (REG_P (i2dest)
959	&& REGNO (i2dest) == FIRST_STACK_REG
960	&& (MEM_P (i2src) || GET_CODE (i2src) == CONST_DOUBLE)
961	/* Ensure i2 doesn't have other side-effects. */
962	&& !side_effects_p (i2src)
963	/* And that the two instructions can actually be
964	swapped, i.e. there shouldn't be any stores
965	in between i2 and i1 that might alias with
966	the i1 memory, and the memory address can't
967	use registers set in between i2 and i1. */
968	&& !modified_between_p (SET_SRC (i1set), i2, i1))
969	{
970	/* Move i1 (fld b above) right before i2 (fld a
971	above. */
972	remove_insn (i1);
973	SET_PREV_INSN (i1) = NULL_RTX;
974	SET_NEXT_INSN (i1) = NULL_RTX;
975	set_block_for_insn (insn: i1, NULL);
976	emit_insn_before (i1, i2);
977	return;
978	}
979	}
980	}
981	}
982
983	/* Avoid emitting the swap if this is the first register stack insn
984	of the current_block. Instead update the current_block's stack_in
985	and let compensate edges take care of this for us. */
986	if (current_block && starting_stack_p)
987	{
988	BLOCK_INFO (current_block)->stack_in = *regstack;
989	starting_stack_p = false;
990	return;
991	}
992
993	machine_mode raw_mode = reg_raw_mode[FIRST_STACK_REG];
994	rtx op1 = FP_MODE_REG (hard_regno, raw_mode);
995	rtx op2 = FP_MODE_REG (FIRST_STACK_REG, raw_mode);
996	rtx swap_rtx
997	= gen_rtx_PARALLEL (VOIDmode,
998	gen_rtvec (2, gen_rtx_SET (op1, op2),
999	gen_rtx_SET (op2, op1)));
1000	if (i1)
1001	emit_insn_after (swap_rtx, i1);
1002	else if (current_block)
1003	emit_insn_before (swap_rtx, BB_HEAD (current_block));
1004	else
1005	emit_insn_before (swap_rtx, insn);
1006	}
1007
1008	/* Emit an insns before INSN to swap virtual register SRC1 with
1009	the top of stack and virtual register SRC2 with second stack
1010	slot. REGSTACK is the stack state before the swaps, and
1011	is updated to reflect the swaps. A swap insn is represented as a
1012	PARALLEL of two patterns: each pattern moves one reg to the other.
1013
1014	If SRC1 and/or SRC2 are already at the right place, no swap insn
1015	is emitted. */
1016
1017	static void
1018	swap_to_top (rtx_insn *insn, stack_ptr regstack, rtx src1, rtx src2)
1019	{
1020	struct stack_def temp_stack;
1021	int regno, j, k;
1022
1023	temp_stack = *regstack;
1024
1025	/* Place operand 1 at the top of stack. */
1026	regno = get_hard_regnum (regstack: &temp_stack, reg: src1);
1027	gcc_assert (regno >= 0);
1028	if (regno != FIRST_STACK_REG)
1029	{
1030	k = temp_stack.top - (regno - FIRST_STACK_REG);
1031	j = temp_stack.top;
1032
1033	std::swap (a&: temp_stack.reg[j], b&: temp_stack.reg[k]);
1034	}
1035
1036	/* Place operand 2 next on the stack. */
1037	regno = get_hard_regnum (regstack: &temp_stack, reg: src2);
1038	gcc_assert (regno >= 0);
1039	if (regno != FIRST_STACK_REG + 1)
1040	{
1041	k = temp_stack.top - (regno - FIRST_STACK_REG);
1042	j = temp_stack.top - 1;
1043
1044	std::swap (a&: temp_stack.reg[j], b&: temp_stack.reg[k]);
1045	}
1046
1047	change_stack (insn, regstack, &temp_stack, EMIT_BEFORE);
1048	}
1049
1050	/* Handle a move to or from a stack register in PAT, which is in INSN.
1051	REGSTACK is the current stack. Return whether a control flow insn
1052	was deleted in the process. */
1053
1054	static bool
1055	move_for_stack_reg (rtx_insn *insn, stack_ptr regstack, rtx pat)
1056	{
1057	rtx *psrc = get_true_reg (pat: &SET_SRC (pat));
1058	rtx *pdest = get_true_reg (pat: &SET_DEST (pat));
1059	rtx src, dest;
1060	rtx note;
1061	bool control_flow_insn_deleted = false;
1062
1063	src = *psrc; dest = *pdest;
1064
1065	if (STACK_REG_P (src) && STACK_REG_P (dest))
1066	{
1067	/* Write from one stack reg to another. If SRC dies here, then
1068	just change the register mapping and delete the insn. */
1069
1070	note = find_regno_note (insn, REG_DEAD, REGNO (src));
1071	if (note)
1072	{
1073	int i;
1074
1075	/* If this is a no-op move, there must not be a REG_DEAD note. */
1076	gcc_assert (REGNO (src) != REGNO (dest));
1077
1078	for (i = regstack->top; i >= 0; i--)
1079	if (regstack->reg[i] == REGNO (src))
1080	break;
1081
1082	/* The destination must be dead, or life analysis is borked. */
1083	gcc_assert (get_hard_regnum (regstack, dest) < FIRST_STACK_REG
1084	|| any_malformed_asm);
1085
1086	/* If the source is not live, this is yet another case of
1087	uninitialized variables. Load up a NaN instead. */
1088	if (i < 0)
1089	return move_nan_for_stack_reg (insn, regstack, dest);
1090
1091	/* It is possible that the dest is unused after this insn.
1092	If so, just pop the src. */
1093
1094	if (find_regno_note (insn, REG_UNUSED, REGNO (dest)))
1095	emit_pop_insn (insn, regstack, reg: src, where: EMIT_AFTER);
1096	else
1097	{
1098	regstack->reg[i] = REGNO (dest);
1099	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (dest));
1100	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, REGNO (src));
1101	}
1102
1103	if (control_flow_insn_p (insn))
1104	control_flow_insn_deleted = true;
1105	delete_insn (insn);
1106	return control_flow_insn_deleted;
1107	}
1108
1109	/* The source reg does not die. */
1110
1111	/* If this appears to be a no-op move, delete it, or else it
1112	will confuse the machine description output patterns. But if
1113	it is REG_UNUSED, we must pop the reg now, as per-insn processing
1114	for REG_UNUSED will not work for deleted insns. */
1115
1116	if (REGNO (src) == REGNO (dest))
1117	{
1118	if (find_regno_note (insn, REG_UNUSED, REGNO (dest)))
1119	emit_pop_insn (insn, regstack, reg: dest, where: EMIT_AFTER);
1120
1121	if (control_flow_insn_p (insn))
1122	control_flow_insn_deleted = true;
1123	delete_insn (insn);
1124	return control_flow_insn_deleted;
1125	}
1126
1127	/* The destination ought to be dead. */
1128	if (get_hard_regnum (regstack, reg: dest) >= FIRST_STACK_REG)
1129	gcc_assert (any_malformed_asm);
1130	else
1131	{
1132	replace_reg (reg: psrc, regno: get_hard_regnum (regstack, reg: src));
1133
1134	regstack->reg[++regstack->top] = REGNO (dest);
1135	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (dest));
1136	replace_reg (reg: pdest, FIRST_STACK_REG);
1137	}
1138	}
1139	else if (STACK_REG_P (src))
1140	{
1141	/* Save from a stack reg to MEM, or possibly integer reg. Since
1142	only top of stack may be saved, emit an exchange first if
1143	needs be. */
1144
1145	emit_swap_insn (insn, regstack, reg: src);
1146
1147	note = find_regno_note (insn, REG_DEAD, REGNO (src));
1148	if (note)
1149	{
1150	replace_reg (reg: &XEXP (note, 0), FIRST_STACK_REG);
1151	regstack->top--;
1152	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, REGNO (src));
1153	}
1154	else if ((GET_MODE (src) == XFmode)
1155	&& regstack->top < REG_STACK_SIZE - 1)
1156	{
1157	/* A 387 cannot write an XFmode value to a MEM without
1158	clobbering the source reg. The output code can handle
1159	this by reading back the value from the MEM.
1160	But it is more efficient to use a temp register if one is
1161	available. Push the source value here if the register
1162	stack is not full, and then write the value to memory via
1163	a pop. */
1164	rtx push_rtx;
1165	rtx top_stack_reg = FP_MODE_REG (FIRST_STACK_REG, GET_MODE (src));
1166
1167	push_rtx = gen_movxf (top_stack_reg, top_stack_reg);
1168	emit_insn_before (push_rtx, insn);
1169	add_reg_note (insn, REG_DEAD, top_stack_reg);
1170	}
1171
1172	replace_reg (reg: psrc, FIRST_STACK_REG);
1173	}
1174	else
1175	{
1176	rtx pat = PATTERN (insn);
1177
1178	gcc_assert (STACK_REG_P (dest));
1179
1180	/* Load from MEM, or possibly integer REG or constant, into the
1181	stack regs. The actual target is always the top of the
1182	stack. The stack mapping is changed to reflect that DEST is
1183	now at top of stack. */
1184
1185	/* The destination ought to be dead. However, there is a
1186	special case with i387 UNSPEC_TAN, where destination is live
1187	(an argument to fptan) but inherent load of 1.0 is modelled
1188	as a load from a constant. */
1189	if (GET_CODE (pat) == PARALLEL
1190	&& XVECLEN (pat, 0) == 2
1191	&& GET_CODE (XVECEXP (pat, 0, 1)) == SET
1192	&& GET_CODE (SET_SRC (XVECEXP (pat, 0, 1))) == UNSPEC
1193	&& XINT (SET_SRC (XVECEXP (pat, 0, 1)), 1) == UNSPEC_TAN)
1194	emit_swap_insn (insn, regstack, reg: dest);
1195	else
1196	gcc_assert (get_hard_regnum (regstack, dest) < FIRST_STACK_REG
1197	|| any_malformed_asm);
1198
1199	gcc_assert (regstack->top < REG_STACK_SIZE);
1200
1201	regstack->reg[++regstack->top] = REGNO (dest);
1202	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (dest));
1203	replace_reg (reg: pdest, FIRST_STACK_REG);
1204	}
1205
1206	return control_flow_insn_deleted;
1207	}
1208
1209	/* A helper function which replaces INSN with a pattern that loads up
1210	a NaN into DEST, then invokes move_for_stack_reg. */
1211
1212	static bool
1213	move_nan_for_stack_reg (rtx_insn *insn, stack_ptr regstack, rtx dest)
1214	{
1215	rtx pat;
1216
1217	dest = FP_MODE_REG (REGNO (dest), SFmode);
1218	pat = gen_rtx_SET (dest, not_a_num);
1219	PATTERN (insn) = pat;
1220	INSN_CODE (insn) = -1;
1221
1222	return move_for_stack_reg (insn, regstack, pat);
1223	}
1224
1225	/* Swap the condition on a branch, if there is one. Return true if we
1226	found a condition to swap. False if the condition was not used as
1227	such. */
1228
1229	static bool
1230	swap_rtx_condition_1 (rtx pat)
1231	{
1232	const char *fmt;
1233	bool r = false;
1234	int i;
1235
1236	if (COMPARISON_P (pat))
1237	{
1238	PUT_CODE (pat, swap_condition (GET_CODE (pat)));
1239	r = true;
1240	}
1241	else
1242	{
1243	fmt = GET_RTX_FORMAT (GET_CODE (pat));
1244	for (i = GET_RTX_LENGTH (GET_CODE (pat)) - 1; i >= 0; i--)
1245	{
1246	if (fmt[i] == 'E')
1247	{
1248	int j;
1249
1250	for (j = XVECLEN (pat, i) - 1; j >= 0; j--)
1251	if (swap_rtx_condition_1 (XVECEXP (pat, i, j)))
1252	r = true;
1253	}
1254	else if (fmt[i] == 'e' && swap_rtx_condition_1 (XEXP (pat, i)))
1255	r = true;
1256	}
1257	}
1258
1259	return r;
1260	}
1261
1262	/* This function swaps condition in cc users and returns true
1263	if successful. It is invoked in 2 different modes, one with
1264	DEBUG_SEEN set initially to 0. In this mode, next_flags_user
1265	will skip DEBUG_INSNs that it would otherwise return and just
1266	sets DEBUG_SEEN to 1 in that case. If DEBUG_SEEN is 0 at
1267	the end of toplevel swap_rtx_condition which returns true,
1268	it means no problematic DEBUG_INSNs were seen and all changes
1269	have been applied. If it returns true but DEBUG_SEEN is 1,
1270	it means some problematic DEBUG_INSNs were seen and no changes
1271	have been applied so far. In that case one needs to call
1272	swap_rtx_condition again with DEBUG_SEEN set to -1, in which
1273	case it doesn't skip DEBUG_INSNs, but instead adjusts the
1274	flags related condition in them or resets them as needed. */
1275
1276	static bool
1277	swap_rtx_condition (rtx_insn *insn, int &debug_seen)
1278	{
1279	rtx pat = PATTERN (insn);
1280
1281	/* We're looking for a single set to an HImode temporary. */
1282
1283	if (GET_CODE (pat) == SET
1284	&& REG_P (SET_DEST (pat))
1285	&& REGNO (SET_DEST (pat)) == FLAGS_REG)
1286	{
1287	insn = next_flags_user (insn, debug_seen);
1288	if (insn == NULL_RTX)
1289	return false;
1290	pat = PATTERN (insn);
1291	}
1292
1293	/* See if this is, or ends in, a fnstsw. If so, we're not doing anything
1294	with the cc value right now. We may be able to search for one
1295	though. */
1296
1297	if (GET_CODE (pat) == SET
1298	&& GET_CODE (SET_SRC (pat)) == UNSPEC
1299	&& XINT (SET_SRC (pat), 1) == UNSPEC_FNSTSW)
1300	{
1301	rtx dest = SET_DEST (pat);
1302
1303	/* Search forward looking for the first use of this value.
1304	Stop at block boundaries. */
1305	while (insn != BB_END (current_block))
1306	{
1307	insn = NEXT_INSN (insn);
1308	if (INSN_P (insn) && reg_mentioned_p (dest, insn))
1309	{
1310	if (DEBUG_INSN_P (insn))
1311	{
1312	if (debug_seen >= 0)
1313	debug_seen = 1;
1314	else
1315	/* Reset the DEBUG insn otherwise. */
1316	INSN_VAR_LOCATION_LOC (insn) = gen_rtx_UNKNOWN_VAR_LOC ();
1317	continue;
1318	}
1319	break;
1320	}
1321	if (CALL_P (insn))
1322	return false;
1323	}
1324
1325	/* We haven't found it. */
1326	if (insn == BB_END (current_block))
1327	return false;
1328
1329	/* So we've found the insn using this value. If it is anything
1330	other than sahf or the value does not die (meaning we'd have
1331	to search further), then we must give up. */
1332	pat = PATTERN (insn);
1333	if (GET_CODE (pat) != SET
1334	|| GET_CODE (SET_SRC (pat)) != UNSPEC
1335	|| XINT (SET_SRC (pat), 1) != UNSPEC_SAHF
1336	|| ! dead_or_set_p (insn, dest))
1337	return false;
1338
1339	/* Now we are prepared to handle this. */
1340	insn = next_flags_user (insn, debug_seen);
1341	if (insn == NULL_RTX)
1342	return false;
1343	pat = PATTERN (insn);
1344	}
1345
1346	if (swap_rtx_condition_1 (pat))
1347	{
1348	bool fail = false;
1349	if (DEBUG_INSN_P (insn))
1350	gcc_assert (debug_seen < 0);
1351	else
1352	{
1353	INSN_CODE (insn) = -1;
1354	if (recog_memoized (insn) == -1)
1355	fail = true;
1356	}
1357	/* In case the flags don't die here, recurse to try fix
1358	following user too. */
1359	if (!fail && !dead_or_set_p (insn, ix86_flags_rtx))
1360	{
1361	insn = next_flags_user (insn, debug_seen);
1362	if (!insn || !swap_rtx_condition (insn, debug_seen))
1363	fail = true;
1364	}
1365	if (fail || debug_seen == 1)
1366	swap_rtx_condition_1 (pat);
1367	return !fail;
1368	}
1369	return false;
1370	}
1371
1372	/* Handle a comparison. Special care needs to be taken to avoid
1373	causing comparisons that a 387 cannot do correctly, such as EQ.
1374
1375	Also, a pop insn may need to be emitted. The 387 does have an
1376	`fcompp' insn that can pop two regs, but it is sometimes too expensive
1377	to do this - a `fcomp' followed by a `fstpl %st(0)' may be easier to
1378	set up. */
1379
1380	static void
1381	compare_for_stack_reg (rtx_insn *insn, stack_ptr regstack,
1382	rtx pat_src, bool can_pop_second_op)
1383	{
1384	rtx *src1, *src2;
1385	rtx src1_note, src2_note;
1386	int debug_seen = 0;
1387
1388	src1 = get_true_reg (pat: &XEXP (pat_src, 0));
1389	src2 = get_true_reg (pat: &XEXP (pat_src, 1));
1390
1391	/* ??? If fxch turns out to be cheaper than fstp, give priority to
1392	registers that die in this insn - move those to stack top first. */
1393	if ((! STACK_REG_P (*src1)
1394	|| (STACK_REG_P (*src2)
1395	&& get_hard_regnum (regstack, reg: *src2) == FIRST_STACK_REG))
1396	&& swap_rtx_condition (insn, debug_seen))
1397	{
1398	/* If swap_rtx_condition succeeded but some debug insns
1399	were seen along the way, it has actually reverted all the
1400	changes. Rerun swap_rtx_condition in a mode where DEBUG_ISNSs
1401	will be adjusted as well. */
1402	if (debug_seen)
1403	{
1404	debug_seen = -1;
1405	swap_rtx_condition (insn, debug_seen);
1406	}
1407	std::swap (XEXP (pat_src, 0), XEXP (pat_src, 1));
1408
1409	src1 = get_true_reg (pat: &XEXP (pat_src, 0));
1410	src2 = get_true_reg (pat: &XEXP (pat_src, 1));
1411
1412	INSN_CODE (insn) = -1;
1413	}
1414
1415	/* We will fix any death note later. */
1416
1417	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1418
1419	if (STACK_REG_P (*src2))
1420	src2_note = find_regno_note (insn, REG_DEAD, REGNO (*src2));
1421	else
1422	src2_note = NULL_RTX;
1423
1424	emit_swap_insn (insn, regstack, reg: *src1);
1425
1426	replace_reg (reg: src1, FIRST_STACK_REG);
1427
1428	if (STACK_REG_P (*src2))
1429	replace_reg (reg: src2, regno: get_hard_regnum (regstack, reg: *src2));
1430
1431	if (src1_note)
1432	{
1433	if (*src2 == CONST0_RTX (GET_MODE (*src2)))
1434	{
1435	/* This is `ftst' insn that can't pop register. */
1436	remove_regno_note (insn, note: REG_DEAD, REGNO (XEXP (src1_note, 0)));
1437	emit_pop_insn (insn, regstack, XEXP (src1_note, 0),
1438	where: EMIT_AFTER);
1439	}
1440	else
1441	{
1442	pop_stack (regstack, REGNO (XEXP (src1_note, 0)));
1443	replace_reg (reg: &XEXP (src1_note, 0), FIRST_STACK_REG);
1444	}
1445	}
1446
1447	/* If the second operand dies, handle that. But if the operands are
1448	the same stack register, don't bother, because only one death is
1449	needed, and it was just handled. */
1450
1451	if (src2_note
1452	&& ! (STACK_REG_P (*src1) && STACK_REG_P (*src2)
1453	&& REGNO (*src1) == REGNO (*src2)))
1454	{
1455	/* As a special case, two regs may die in this insn if src2 is
1456	next to top of stack and the top of stack also dies. Since
1457	we have already popped src1, "next to top of stack" is really
1458	at top (FIRST_STACK_REG) now. */
1459
1460	if (get_hard_regnum (regstack, XEXP (src2_note, 0)) == FIRST_STACK_REG
1461	&& src1_note && can_pop_second_op)
1462	{
1463	pop_stack (regstack, REGNO (XEXP (src2_note, 0)));
1464	replace_reg (reg: &XEXP (src2_note, 0), FIRST_STACK_REG + 1);
1465	}
1466	else
1467	{
1468	/* The 386 can only represent death of the first operand in
1469	the case handled above. In all other cases, emit a separate
1470	pop and remove the death note from here. */
1471	remove_regno_note (insn, note: REG_DEAD, REGNO (XEXP (src2_note, 0)));
1472	emit_pop_insn (insn, regstack, XEXP (src2_note, 0),
1473	where: EMIT_AFTER);
1474	}
1475	}
1476	}
1477
1478	/* Substitute hardware stack regs in debug insn INSN, using stack
1479	layout REGSTACK. If we can't find a hardware stack reg for any of
1480	the REGs in it, reset the debug insn. */
1481
1482	static void
1483	subst_all_stack_regs_in_debug_insn (rtx_insn *insn, struct stack_def *regstack)
1484	{
1485	subrtx_ptr_iterator::array_type array;
1486	FOR_EACH_SUBRTX_PTR (iter, array, &INSN_VAR_LOCATION_LOC (insn), NONCONST)
1487	{
1488	rtx *loc = *iter;
1489	rtx x = *loc;
1490	if (STACK_REG_P (x))
1491	{
1492	int hard_regno = get_hard_regnum (regstack, reg: x);
1493
1494	/* If we can't find an active register, reset this debug insn. */
1495	if (hard_regno == -1)
1496	{
1497	INSN_VAR_LOCATION_LOC (insn) = gen_rtx_UNKNOWN_VAR_LOC ();
1498	return;
1499	}
1500
1501	gcc_assert (hard_regno >= FIRST_STACK_REG);
1502	replace_reg (reg: loc, regno: hard_regno);
1503	iter.skip_subrtxes ();
1504	}
1505	}
1506	}
1507
1508	/* Substitute new registers in PAT, which is part of INSN. REGSTACK
1509	is the current register layout. Return whether a control flow insn
1510	was deleted in the process. */
1511
1512	static bool
1513	subst_stack_regs_pat (rtx_insn *insn, stack_ptr regstack, rtx pat)
1514	{
1515	rtx *dest, *src;
1516	bool control_flow_insn_deleted = false;
1517
1518	switch (GET_CODE (pat))
1519	{
1520	case USE:
1521	/* Deaths in USE insns can happen in non optimizing compilation.
1522	Handle them by popping the dying register. */
1523	src = get_true_reg (pat: &XEXP (pat, 0));
1524	if (STACK_REG_P (*src)
1525	&& find_regno_note (insn, REG_DEAD, REGNO (*src)))
1526	{
1527	/* USEs are ignored for liveness information so USEs of dead
1528	register might happen. */
1529	if (TEST_HARD_REG_BIT (set: regstack->reg_set, REGNO (*src)))
1530	emit_pop_insn (insn, regstack, reg: *src, where: EMIT_AFTER);
1531	return control_flow_insn_deleted;
1532	}
1533	/* Uninitialized USE might happen for functions returning uninitialized
1534	value. We will properly initialize the USE on the edge to EXIT_BLOCK,
1535	so it is safe to ignore the use here. This is consistent with behavior
1536	of dataflow analyzer that ignores USE too. (This also imply that
1537	forcibly initializing the register to NaN here would lead to ICE later,
1538	since the REG_DEAD notes are not issued.) */
1539	break;
1540
1541	case VAR_LOCATION:
1542	gcc_unreachable ();
1543
1544	case CLOBBER:
1545	{
1546	rtx note;
1547
1548	dest = get_true_reg (pat: &XEXP (pat, 0));
1549	if (STACK_REG_P (*dest))
1550	{
1551	note = find_reg_note (insn, REG_DEAD, *dest);
1552
1553	if (pat != PATTERN (insn))
1554	{
1555	/* The fix_truncdi_1 pattern wants to be able to
1556	allocate its own scratch register. It does this by
1557	clobbering an fp reg so that it is assured of an
1558	empty reg-stack register. If the register is live,
1559	kill it now. Remove the DEAD/UNUSED note so we
1560	don't try to kill it later too.
1561
1562	In reality the UNUSED note can be absent in some
1563	complicated cases when the register is reused for
1564	partially set variable. */
1565
1566	if (note)
1567	emit_pop_insn (insn, regstack, reg: *dest, where: EMIT_BEFORE);
1568	else
1569	note = find_reg_note (insn, REG_UNUSED, *dest);
1570	if (note)
1571	remove_note (insn, note);
1572	replace_reg (reg: dest, FIRST_STACK_REG + 1);
1573	}
1574	else
1575	{
1576	/* A top-level clobber with no REG_DEAD, and no hard-regnum
1577	indicates an uninitialized value. Because reload removed
1578	all other clobbers, this must be due to a function
1579	returning without a value. Load up a NaN. */
1580
1581	if (!note)
1582	{
1583	rtx t = *dest;
1584	if (COMPLEX_MODE_P (GET_MODE (t)))
1585	{
1586	rtx u = FP_MODE_REG (REGNO (t) + 1, SFmode);
1587	if (get_hard_regnum (regstack, reg: u) == -1)
1588	{
1589	rtx pat2 = gen_rtx_CLOBBER (VOIDmode, u);
1590	rtx_insn *insn2 = emit_insn_before (pat2, insn);
1591	if (move_nan_for_stack_reg (insn: insn2, regstack, dest: u))
1592	control_flow_insn_deleted = true;
1593	}
1594	}
1595	if (get_hard_regnum (regstack, reg: t) == -1
1596	&& move_nan_for_stack_reg (insn, regstack, dest: t))
1597	control_flow_insn_deleted = true;
1598	}
1599	}
1600	}
1601	break;
1602	}
1603
1604	case SET:
1605	{
1606	rtx *src1 = (rtx *) 0, *src2;
1607	rtx src1_note, src2_note;
1608	rtx pat_src;
1609
1610	dest = get_true_reg (pat: &SET_DEST (pat));
1611	src = get_true_reg (pat: &SET_SRC (pat));
1612	pat_src = SET_SRC (pat);
1613
1614	/* See if this is a `movM' pattern, and handle elsewhere if so. */
1615	if (STACK_REG_P (*src)
1616	|| (STACK_REG_P (*dest)
1617	&& (REG_P (*src) || MEM_P (*src)
1618	|| CONST_DOUBLE_P (*src))))
1619	{
1620	if (move_for_stack_reg (insn, regstack, pat))
1621	control_flow_insn_deleted = true;
1622	break;
1623	}
1624
1625	switch (GET_CODE (pat_src))
1626	{
1627	case CALL:
1628	{
1629	int count;
1630	for (count = REG_NREGS (*dest); --count >= 0;)
1631	{
1632	regstack->reg[++regstack->top] = REGNO (*dest) + count;
1633	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest) + count);
1634	}
1635	}
1636	replace_reg (reg: dest, FIRST_STACK_REG);
1637	break;
1638
1639	case REG:
1640	gcc_unreachable ();
1641
1642	/* Fall through. */
1643
1644	case FLOAT_TRUNCATE:
1645	case SQRT:
1646	case ABS:
1647	case NEG:
1648	/* These insns only operate on the top of the stack. It's
1649	possible that the tstM case results in a REG_DEAD note on the
1650	source. */
1651
1652	if (src1 == 0)
1653	src1 = get_true_reg (pat: &XEXP (pat_src, 0));
1654
1655	emit_swap_insn (insn, regstack, reg: *src1);
1656
1657	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1658
1659	if (STACK_REG_P (*dest))
1660	replace_reg (reg: dest, FIRST_STACK_REG);
1661
1662	if (src1_note)
1663	{
1664	replace_reg (reg: &XEXP (src1_note, 0), FIRST_STACK_REG);
1665	regstack->top--;
1666	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*src1));
1667	}
1668
1669	replace_reg (reg: src1, FIRST_STACK_REG);
1670	break;
1671
1672	case MINUS:
1673	case DIV:
1674	/* On i386, reversed forms of subM3 and divM3 exist for
1675	MODE_FLOAT, so the same code that works for addM3 and mulM3
1676	can be used. */
1677	case MULT:
1678	case PLUS:
1679	/* These insns can accept the top of stack as a destination
1680	from a stack reg or mem, or can use the top of stack as a
1681	source and some other stack register (possibly top of stack)
1682	as a destination. */
1683
1684	src1 = get_true_reg (pat: &XEXP (pat_src, 0));
1685	src2 = get_true_reg (pat: &XEXP (pat_src, 1));
1686
1687	/* We will fix any death note later. */
1688
1689	if (STACK_REG_P (*src1))
1690	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1691	else
1692	src1_note = NULL_RTX;
1693	if (STACK_REG_P (*src2))
1694	src2_note = find_regno_note (insn, REG_DEAD, REGNO (*src2));
1695	else
1696	src2_note = NULL_RTX;
1697
1698	/* If either operand is not a stack register, then the dest
1699	must be top of stack. */
1700
1701	if (! STACK_REG_P (*src1) || ! STACK_REG_P (*src2))
1702	emit_swap_insn (insn, regstack, reg: *dest);
1703	else
1704	{
1705	/* Both operands are REG. If neither operand is already
1706	at the top of stack, choose to make the one that is the
1707	dest the new top of stack. */
1708
1709	int src1_hard_regnum, src2_hard_regnum;
1710
1711	src1_hard_regnum = get_hard_regnum (regstack, reg: *src1);
1712	src2_hard_regnum = get_hard_regnum (regstack, reg: *src2);
1713
1714	/* If the source is not live, this is yet another case of
1715	uninitialized variables. Load up a NaN instead. */
1716	if (src1_hard_regnum == -1)
1717	{
1718	rtx pat2 = gen_rtx_CLOBBER (VOIDmode, *src1);
1719	rtx_insn *insn2 = emit_insn_before (pat2, insn);
1720	if (move_nan_for_stack_reg (insn: insn2, regstack, dest: *src1))
1721	control_flow_insn_deleted = true;
1722	}
1723	if (src2_hard_regnum == -1)
1724	{
1725	rtx pat2 = gen_rtx_CLOBBER (VOIDmode, *src2);
1726	rtx_insn *insn2 = emit_insn_before (pat2, insn);
1727	if (move_nan_for_stack_reg (insn: insn2, regstack, dest: *src2))
1728	control_flow_insn_deleted = true;
1729	}
1730
1731	if (src1_hard_regnum != FIRST_STACK_REG
1732	&& src2_hard_regnum != FIRST_STACK_REG)
1733	emit_swap_insn (insn, regstack, reg: *dest);
1734	}
1735
1736	if (STACK_REG_P (*src1))
1737	replace_reg (reg: src1, regno: get_hard_regnum (regstack, reg: *src1));
1738	if (STACK_REG_P (*src2))
1739	replace_reg (reg: src2, regno: get_hard_regnum (regstack, reg: *src2));
1740
1741	if (src1_note)
1742	{
1743	rtx src1_reg = XEXP (src1_note, 0);
1744
1745	/* If the register that dies is at the top of stack, then
1746	the destination is somewhere else - merely substitute it.
1747	But if the reg that dies is not at top of stack, then
1748	move the top of stack to the dead reg, as though we had
1749	done the insn and then a store-with-pop. */
1750
1751	if (REGNO (src1_reg) == regstack->reg[regstack->top])
1752	{
1753	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1754	replace_reg (reg: dest, regno: get_hard_regnum (regstack, reg: *dest));
1755	}
1756	else
1757	{
1758	int regno = get_hard_regnum (regstack, reg: src1_reg);
1759
1760	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1761	replace_reg (reg: dest, regno);
1762
1763	regstack->reg[regstack->top - (regno - FIRST_STACK_REG)]
1764	= regstack->reg[regstack->top];
1765	}
1766
1767	CLEAR_HARD_REG_BIT (set&: regstack->reg_set,
1768	REGNO (XEXP (src1_note, 0)));
1769	replace_reg (reg: &XEXP (src1_note, 0), FIRST_STACK_REG);
1770	regstack->top--;
1771	}
1772	else if (src2_note)
1773	{
1774	rtx src2_reg = XEXP (src2_note, 0);
1775	if (REGNO (src2_reg) == regstack->reg[regstack->top])
1776	{
1777	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1778	replace_reg (reg: dest, regno: get_hard_regnum (regstack, reg: *dest));
1779	}
1780	else
1781	{
1782	int regno = get_hard_regnum (regstack, reg: src2_reg);
1783
1784	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1785	replace_reg (reg: dest, regno);
1786
1787	regstack->reg[regstack->top - (regno - FIRST_STACK_REG)]
1788	= regstack->reg[regstack->top];
1789	}
1790
1791	CLEAR_HARD_REG_BIT (set&: regstack->reg_set,
1792	REGNO (XEXP (src2_note, 0)));
1793	replace_reg (reg: &XEXP (src2_note, 0), FIRST_STACK_REG);
1794	regstack->top--;
1795	}
1796	else
1797	{
1798	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1799	replace_reg (reg: dest, regno: get_hard_regnum (regstack, reg: *dest));
1800	}
1801
1802	/* Keep operand 1 matching with destination. */
1803	if (COMMUTATIVE_ARITH_P (pat_src)
1804	&& REG_P (*src1) && REG_P (*src2)
1805	&& REGNO (*src1) != REGNO (*dest))
1806	{
1807	int tmp = REGNO (*src1);
1808	replace_reg (reg: src1, REGNO (*src2));
1809	replace_reg (reg: src2, regno: tmp);
1810	}
1811	break;
1812
1813	case UNSPEC:
1814	switch (XINT (pat_src, 1))
1815	{
1816	case UNSPEC_FIST:
1817	case UNSPEC_FIST_ATOMIC:
1818
1819	case UNSPEC_FIST_FLOOR:
1820	case UNSPEC_FIST_CEIL:
1821
1822	/* These insns only operate on the top of the stack. */
1823
1824	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1825	emit_swap_insn (insn, regstack, reg: *src1);
1826
1827	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1828
1829	if (STACK_REG_P (*dest))
1830	replace_reg (reg: dest, FIRST_STACK_REG);
1831
1832	if (src1_note)
1833	{
1834	replace_reg (reg: &XEXP (src1_note, 0), FIRST_STACK_REG);
1835	regstack->top--;
1836	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*src1));
1837	}
1838
1839	replace_reg (reg: src1, FIRST_STACK_REG);
1840	break;
1841
1842	case UNSPEC_FXAM:
1843
1844	/* This insn only operate on the top of the stack. */
1845
1846	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1847	emit_swap_insn (insn, regstack, reg: *src1);
1848
1849	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1850
1851	replace_reg (reg: src1, FIRST_STACK_REG);
1852
1853	if (src1_note)
1854	{
1855	remove_regno_note (insn, note: REG_DEAD,
1856	REGNO (XEXP (src1_note, 0)));
1857	emit_pop_insn (insn, regstack, XEXP (src1_note, 0),
1858	where: EMIT_AFTER);
1859	}
1860
1861	break;
1862
1863	case UNSPEC_SIN:
1864	case UNSPEC_COS:
1865	case UNSPEC_FRNDINT:
1866	case UNSPEC_F2XM1:
1867
1868	case UNSPEC_FRNDINT_ROUNDEVEN:
1869	case UNSPEC_FRNDINT_FLOOR:
1870	case UNSPEC_FRNDINT_CEIL:
1871	case UNSPEC_FRNDINT_TRUNC:
1872
1873	/* Above insns operate on the top of the stack. */
1874
1875	case UNSPEC_SINCOS_COS:
1876	case UNSPEC_XTRACT_FRACT:
1877
1878	/* Above insns operate on the top two stack slots,
1879	first part of one input, double output insn. */
1880
1881	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1882
1883	emit_swap_insn (insn, regstack, reg: *src1);
1884
1885	/* Input should never die, it is replaced with output. */
1886	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1887	gcc_assert (!src1_note);
1888
1889	if (STACK_REG_P (*dest))
1890	replace_reg (reg: dest, FIRST_STACK_REG);
1891
1892	replace_reg (reg: src1, FIRST_STACK_REG);
1893	break;
1894
1895	case UNSPEC_SINCOS_SIN:
1896	case UNSPEC_XTRACT_EXP:
1897
1898	/* These insns operate on the top two stack slots,
1899	second part of one input, double output insn. */
1900
1901	regstack->top++;
1902	/* FALLTHRU */
1903
1904	case UNSPEC_TAN:
1905
1906	/* For UNSPEC_TAN, regstack->top is already increased
1907	by inherent load of constant 1.0. */
1908
1909	/* Output value is generated in the second stack slot.
1910	Move current value from second slot to the top. */
1911	regstack->reg[regstack->top]
1912	= regstack->reg[regstack->top - 1];
1913
1914	gcc_assert (STACK_REG_P (*dest));
1915
1916	regstack->reg[regstack->top - 1] = REGNO (*dest);
1917	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1918	replace_reg (reg: dest, FIRST_STACK_REG + 1);
1919
1920	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1921
1922	replace_reg (reg: src1, FIRST_STACK_REG);
1923	break;
1924
1925	case UNSPEC_FPATAN:
1926	case UNSPEC_FYL2X:
1927	case UNSPEC_FYL2XP1:
1928	/* These insns operate on the top two stack slots. */
1929
1930	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1931	src2 = get_true_reg (pat: &XVECEXP (pat_src, 0, 1));
1932
1933	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1934	src2_note = find_regno_note (insn, REG_DEAD, REGNO (*src2));
1935
1936	swap_to_top (insn, regstack, src1: *src1, src2: *src2);
1937
1938	replace_reg (reg: src1, FIRST_STACK_REG);
1939	replace_reg (reg: src2, FIRST_STACK_REG + 1);
1940
1941	if (src1_note)
1942	replace_reg (reg: &XEXP (src1_note, 0), FIRST_STACK_REG);
1943	if (src2_note)
1944	replace_reg (reg: &XEXP (src2_note, 0), FIRST_STACK_REG + 1);
1945
1946	/* Pop both input operands from the stack. */
1947	CLEAR_HARD_REG_BIT (set&: regstack->reg_set,
1948	bit: regstack->reg[regstack->top]);
1949	CLEAR_HARD_REG_BIT (set&: regstack->reg_set,
1950	bit: regstack->reg[regstack->top - 1]);
1951	regstack->top -= 2;
1952
1953	/* Push the result back onto the stack. */
1954	regstack->reg[++regstack->top] = REGNO (*dest);
1955	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1956	replace_reg (reg: dest, FIRST_STACK_REG);
1957	break;
1958
1959	case UNSPEC_FSCALE_FRACT:
1960	case UNSPEC_FPREM_F:
1961	case UNSPEC_FPREM1_F:
1962	/* These insns operate on the top two stack slots,
1963	first part of double input, double output insn. */
1964
1965	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1966	src2 = get_true_reg (pat: &XVECEXP (pat_src, 0, 1));
1967
1968	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
1969	src2_note = find_regno_note (insn, REG_DEAD, REGNO (*src2));
1970
1971	/* Inputs should never die, they are
1972	replaced with outputs. */
1973	gcc_assert (!src1_note);
1974	gcc_assert (!src2_note);
1975
1976	swap_to_top (insn, regstack, src1: *src1, src2: *src2);
1977
1978	/* Push the result back onto stack. Empty stack slot
1979	will be filled in second part of insn. */
1980	if (STACK_REG_P (*dest))
1981	{
1982	regstack->reg[regstack->top] = REGNO (*dest);
1983	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
1984	replace_reg (reg: dest, FIRST_STACK_REG);
1985	}
1986
1987	replace_reg (reg: src1, FIRST_STACK_REG);
1988	replace_reg (reg: src2, FIRST_STACK_REG + 1);
1989	break;
1990
1991	case UNSPEC_FSCALE_EXP:
1992	case UNSPEC_FPREM_U:
1993	case UNSPEC_FPREM1_U:
1994	/* These insns operate on the top two stack slots,
1995	second part of double input, double output insn. */
1996
1997	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
1998	src2 = get_true_reg (pat: &XVECEXP (pat_src, 0, 1));
1999
2000	/* Push the result back onto stack. Fill empty slot from
2001	first part of insn and fix top of stack pointer. */
2002	if (STACK_REG_P (*dest))
2003	{
2004	regstack->reg[regstack->top - 1] = REGNO (*dest);
2005	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
2006	replace_reg (reg: dest, FIRST_STACK_REG + 1);
2007	}
2008
2009	replace_reg (reg: src1, FIRST_STACK_REG);
2010	replace_reg (reg: src2, FIRST_STACK_REG + 1);
2011	break;
2012
2013	case UNSPEC_C2_FLAG:
2014	/* This insn operates on the top two stack slots,
2015	third part of C2 setting double input insn. */
2016
2017	src1 = get_true_reg (pat: &XVECEXP (pat_src, 0, 0));
2018	src2 = get_true_reg (pat: &XVECEXP (pat_src, 0, 1));
2019
2020	replace_reg (reg: src1, FIRST_STACK_REG);
2021	replace_reg (reg: src2, FIRST_STACK_REG + 1);
2022	break;
2023
2024	case UNSPEC_FNSTSW:
2025	/* Combined fcomp+fnstsw generated for doing well with
2026	CSE. When optimizing this would have been broken
2027	up before now. */
2028
2029	pat_src = XVECEXP (pat_src, 0, 0);
2030	if (GET_CODE (pat_src) == COMPARE)
2031	goto do_compare;
2032
2033	/* Fall through. */
2034
2035	case UNSPEC_NOTRAP:
2036
2037	pat_src = XVECEXP (pat_src, 0, 0);
2038	gcc_assert (GET_CODE (pat_src) == COMPARE);
2039	goto do_compare;
2040
2041	default:
2042	gcc_unreachable ();
2043	}
2044	break;
2045
2046	case COMPARE:
2047	do_compare:
2048	/* `fcomi' insn can't pop two regs. */
2049	compare_for_stack_reg (insn, regstack, pat_src,
2050	REGNO (*dest) != FLAGS_REG);
2051	break;
2052
2053	case IF_THEN_ELSE:
2054	/* This insn requires the top of stack to be the destination. */
2055
2056	src1 = get_true_reg (pat: &XEXP (pat_src, 1));
2057	src2 = get_true_reg (pat: &XEXP (pat_src, 2));
2058
2059	src1_note = find_regno_note (insn, REG_DEAD, REGNO (*src1));
2060	src2_note = find_regno_note (insn, REG_DEAD, REGNO (*src2));
2061
2062	/* If the comparison operator is an FP comparison operator,
2063	it is handled correctly by compare_for_stack_reg () who
2064	will move the destination to the top of stack. But if the
2065	comparison operator is not an FP comparison operator, we
2066	have to handle it here. */
2067	if (get_hard_regnum (regstack, reg: *dest) >= FIRST_STACK_REG
2068	&& REGNO (*dest) != regstack->reg[regstack->top])
2069	{
2070	/* In case one of operands is the top of stack and the operands
2071	dies, it is safe to make it the destination operand by
2072	reversing the direction of cmove and avoid fxch. */
2073	if ((REGNO (*src1) == regstack->reg[regstack->top]
2074	&& src1_note)
2075	|| (REGNO (*src2) == regstack->reg[regstack->top]
2076	&& src2_note))
2077	{
2078	int idx1 = (get_hard_regnum (regstack, reg: *src1)
2079	- FIRST_STACK_REG);
2080	int idx2 = (get_hard_regnum (regstack, reg: *src2)
2081	- FIRST_STACK_REG);
2082
2083	/* Make reg-stack believe that the operands are already
2084	swapped on the stack */
2085	regstack->reg[regstack->top - idx1] = REGNO (*src2);
2086	regstack->reg[regstack->top - idx2] = REGNO (*src1);
2087
2088	/* Reverse condition to compensate the operand swap.
2089	i386 do have comparison always reversible. */
2090	PUT_CODE (XEXP (pat_src, 0),
2091	reversed_comparison_code (XEXP (pat_src, 0), insn));
2092	}
2093	else
2094	emit_swap_insn (insn, regstack, reg: *dest);
2095	}
2096
2097	{
2098	rtx src_note [3];
2099	int i;
2100
2101	src_note[0] = 0;
2102	src_note[1] = src1_note;
2103	src_note[2] = src2_note;
2104
2105	if (STACK_REG_P (*src1))
2106	replace_reg (reg: src1, regno: get_hard_regnum (regstack, reg: *src1));
2107	if (STACK_REG_P (*src2))
2108	replace_reg (reg: src2, regno: get_hard_regnum (regstack, reg: *src2));
2109
2110	for (i = 1; i <= 2; i++)
2111	if (src_note [i])
2112	{
2113	int regno = REGNO (XEXP (src_note[i], 0));
2114
2115	/* If the register that dies is not at the top of
2116	stack, then move the top of stack to the dead reg.
2117	Top of stack should never die, as it is the
2118	destination. */
2119	gcc_assert (regno != regstack->reg[regstack->top]);
2120	remove_regno_note (insn, note: REG_DEAD, regno);
2121	emit_pop_insn (insn, regstack, XEXP (src_note[i], 0),
2122	where: EMIT_AFTER);
2123	}
2124	}
2125
2126	/* Make dest the top of stack. Add dest to regstack if
2127	not present. */
2128	if (get_hard_regnum (regstack, reg: *dest) < FIRST_STACK_REG)
2129	regstack->reg[++regstack->top] = REGNO (*dest);
2130	SET_HARD_REG_BIT (set&: regstack->reg_set, REGNO (*dest));
2131	replace_reg (reg: dest, FIRST_STACK_REG);
2132	break;
2133
2134	default:
2135	gcc_unreachable ();
2136	}
2137	break;
2138	}
2139
2140	default:
2141	break;
2142	}
2143
2144	return control_flow_insn_deleted;
2145	}
2146
2147	/* Substitute hard regnums for any stack regs in INSN, which has
2148	N_INPUTS inputs and N_OUTPUTS outputs. REGSTACK is the stack info
2149	before the insn, and is updated with changes made here.
2150
2151	There are several requirements and assumptions about the use of
2152	stack-like regs in asm statements. These rules are enforced by
2153	record_asm_stack_regs; see comments there for details. Any
2154	asm_operands left in the RTL at this point may be assume to meet the
2155	requirements, since record_asm_stack_regs removes any problem asm. */
2156
2157	static void
2158	subst_asm_stack_regs (rtx_insn *insn, stack_ptr regstack)
2159	{
2160	rtx body = PATTERN (insn);
2161
2162	rtx *note_reg; /* Array of note contents */
2163	rtx **note_loc; /* Address of REG field of each note */
2164	enum reg_note *note_kind; /* The type of each note */
2165
2166	rtx *clobber_reg = 0;
2167	rtx **clobber_loc = 0;
2168
2169	struct stack_def temp_stack;
2170	int n_notes;
2171	int n_clobbers;
2172	rtx note;
2173	int i;
2174	int n_inputs, n_outputs;
2175
2176	if (! check_asm_stack_operands (insn))
2177	return;
2178
2179	/* Find out what the constraints required. If no constraint
2180	alternative matches, that is a compiler bug: we should have caught
2181	such an insn in check_asm_stack_operands. */
2182	extract_constrain_insn (insn);
2183
2184	preprocess_constraints (insn);
2185	const operand_alternative *op_alt = which_op_alt ();
2186
2187	get_asm_operands_in_out (body, pout: &n_outputs, pin: &n_inputs);
2188
2189	/* Strip SUBREGs here to make the following code simpler. */
2190	for (i = 0; i < recog_data.n_operands; i++)
2191	if (GET_CODE (recog_data.operand[i]) == SUBREG
2192	&& REG_P (SUBREG_REG (recog_data.operand[i])))
2193	{
2194	recog_data.operand_loc[i] = & SUBREG_REG (recog_data.operand[i]);
2195	recog_data.operand[i] = SUBREG_REG (recog_data.operand[i]);
2196	}
2197
2198	/* Set up NOTE_REG, NOTE_LOC and NOTE_KIND. */
2199
2200	for (i = 0, note = REG_NOTES (insn); note; note = XEXP (note, 1))
2201	i++;
2202
2203	note_reg = XALLOCAVEC (rtx, i);
2204	note_loc = XALLOCAVEC (rtx *, i);
2205	note_kind = XALLOCAVEC (enum reg_note, i);
2206
2207	n_notes = 0;
2208	for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
2209	{
2210	if (GET_CODE (note) != EXPR_LIST)
2211	continue;
2212	rtx reg = XEXP (note, 0);
2213	rtx *loc = & XEXP (note, 0);
2214
2215	if (GET_CODE (reg) == SUBREG && REG_P (SUBREG_REG (reg)))
2216	{
2217	loc = & SUBREG_REG (reg);
2218	reg = SUBREG_REG (reg);
2219	}
2220
2221	if (STACK_REG_P (reg)
2222	&& (REG_NOTE_KIND (note) == REG_DEAD
2223	|| REG_NOTE_KIND (note) == REG_UNUSED))
2224	{
2225	note_reg[n_notes] = reg;
2226	note_loc[n_notes] = loc;
2227	note_kind[n_notes] = REG_NOTE_KIND (note);
2228	n_notes++;
2229	}
2230	}
2231
2232	/* Set up CLOBBER_REG and CLOBBER_LOC. */
2233
2234	n_clobbers = 0;
2235
2236	if (GET_CODE (body) == PARALLEL)
2237	{
2238	clobber_reg = XALLOCAVEC (rtx, XVECLEN (body, 0));
2239	clobber_loc = XALLOCAVEC (rtx *, XVECLEN (body, 0));
2240
2241	for (i = 0; i < XVECLEN (body, 0); i++)
2242	if (GET_CODE (XVECEXP (body, 0, i)) == CLOBBER)
2243	{
2244	rtx clobber = XVECEXP (body, 0, i);
2245	rtx reg = XEXP (clobber, 0);
2246	rtx *loc = & XEXP (clobber, 0);
2247
2248	if (GET_CODE (reg) == SUBREG && REG_P (SUBREG_REG (reg)))
2249	{
2250	loc = & SUBREG_REG (reg);
2251	reg = SUBREG_REG (reg);
2252	}
2253
2254	if (STACK_REG_P (reg))
2255	{
2256	clobber_reg[n_clobbers] = reg;
2257	clobber_loc[n_clobbers] = loc;
2258	n_clobbers++;
2259	}
2260	}
2261	}
2262
2263	temp_stack = *regstack;
2264
2265	/* Put the input regs into the desired place in TEMP_STACK. */
2266
2267	for (i = n_outputs; i < n_outputs + n_inputs; i++)
2268	if (STACK_REG_P (recog_data.operand[i])
2269	&& reg_class_subset_p (op_alt[i].cl, FLOAT_REGS)
2270	&& op_alt[i].cl != FLOAT_REGS)
2271	{
2272	/* If an operand needs to be in a particular reg in
2273	FLOAT_REGS, the constraint was either 't' or 'u'. Since
2274	these constraints are for single register classes, and
2275	reload guaranteed that operand[i] is already in that class,
2276	we can just use REGNO (recog_data.operand[i]) to know which
2277	actual reg this operand needs to be in. */
2278
2279	int regno = get_hard_regnum (regstack: &temp_stack, reg: recog_data.operand[i]);
2280
2281	gcc_assert (regno >= 0);
2282
2283	if ((unsigned int) regno != REGNO (recog_data.operand[i]))
2284	{
2285	/* recog_data.operand[i] is not in the right place. Find
2286	it and swap it with whatever is already in I's place.
2287	K is where recog_data.operand[i] is now. J is where it
2288	should be. */
2289	int j, k;
2290
2291	k = temp_stack.top - (regno - FIRST_STACK_REG);
2292	j = (temp_stack.top
2293	- (REGNO (recog_data.operand[i]) - FIRST_STACK_REG));
2294
2295	std::swap (a&: temp_stack.reg[j], b&: temp_stack.reg[k]);
2296	}
2297	}
2298
2299	/* Emit insns before INSN to make sure the reg-stack is in the right
2300	order. */
2301
2302	change_stack (insn, regstack, &temp_stack, EMIT_BEFORE);
2303
2304	/* Make the needed input register substitutions. Do death notes and
2305	clobbers too, because these are for inputs, not outputs. */
2306
2307	for (i = n_outputs; i < n_outputs + n_inputs; i++)
2308	if (STACK_REG_P (recog_data.operand[i]))
2309	{
2310	int regnum = get_hard_regnum (regstack, reg: recog_data.operand[i]);
2311
2312	gcc_assert (regnum >= 0);
2313
2314	replace_reg (reg: recog_data.operand_loc[i], regno: regnum);
2315	}
2316
2317	for (i = 0; i < n_notes; i++)
2318	if (note_kind[i] == REG_DEAD)
2319	{
2320	int regnum = get_hard_regnum (regstack, reg: note_reg[i]);
2321
2322	gcc_assert (regnum >= 0);
2323
2324	replace_reg (reg: note_loc[i], regno: regnum);
2325	}
2326
2327	for (i = 0; i < n_clobbers; i++)
2328	{
2329	/* It's OK for a CLOBBER to reference a reg that is not live.
2330	Don't try to replace it in that case. */
2331	int regnum = get_hard_regnum (regstack, reg: clobber_reg[i]);
2332
2333	if (regnum >= 0)
2334	replace_reg (reg: clobber_loc[i], regno: regnum);
2335	}
2336
2337	/* Now remove from REGSTACK any inputs that the asm implicitly popped. */
2338
2339	for (i = n_outputs; i < n_outputs + n_inputs; i++)
2340	if (STACK_REG_P (recog_data.operand[i]))
2341	{
2342	/* An input reg is implicitly popped if it is tied to an
2343	output, or if there is a CLOBBER for it. */
2344	int j;
2345
2346	for (j = 0; j < n_clobbers; j++)
2347	if (operands_match_p (clobber_reg[j], recog_data.operand[i]))
2348	break;
2349
2350	if (j < n_clobbers || op_alt[i].matches >= 0)
2351	{
2352	/* recog_data.operand[i] might not be at the top of stack.
2353	But that's OK, because all we need to do is pop the
2354	right number of regs off of the top of the reg-stack.
2355	record_asm_stack_regs guaranteed that all implicitly
2356	popped regs were grouped at the top of the reg-stack. */
2357
2358	CLEAR_HARD_REG_BIT (set&: regstack->reg_set,
2359	bit: regstack->reg[regstack->top]);
2360	regstack->top--;
2361	}
2362	}
2363
2364	/* Now add to REGSTACK any outputs that the asm implicitly pushed.
2365	Note that there isn't any need to substitute register numbers.
2366	??? Explain why this is true. */
2367
2368	for (i = LAST_STACK_REG; i >= FIRST_STACK_REG; i--)
2369	{
2370	/* See if there is an output for this hard reg. */
2371	int j;
2372
2373	for (j = 0; j < n_outputs; j++)
2374	if (STACK_REG_P (recog_data.operand[j])
2375	&& REGNO (recog_data.operand[j]) == (unsigned) i)
2376	{
2377	regstack->reg[++regstack->top] = i;
2378	SET_HARD_REG_BIT (set&: regstack->reg_set, bit: i);
2379	break;
2380	}
2381	}
2382
2383	/* Now emit a pop insn for any REG_UNUSED output, or any REG_DEAD
2384	input that the asm didn't implicitly pop. If the asm didn't
2385	implicitly pop an input reg, that reg will still be live.
2386
2387	Note that we can't use find_regno_note here: the register numbers
2388	in the death notes have already been substituted. */
2389
2390	for (i = 0; i < n_outputs; i++)
2391	if (STACK_REG_P (recog_data.operand[i]))
2392	{
2393	int j;
2394
2395	for (j = 0; j < n_notes; j++)
2396	if (REGNO (recog_data.operand[i]) == REGNO (note_reg[j])
2397	&& note_kind[j] == REG_UNUSED)
2398	{
2399	insn = emit_pop_insn (insn, regstack, reg: recog_data.operand[i],
2400	where: EMIT_AFTER);
2401	break;
2402	}
2403	}
2404
2405	for (i = n_outputs; i < n_outputs + n_inputs; i++)
2406	if (STACK_REG_P (recog_data.operand[i]))
2407	{
2408	int j;
2409
2410	for (j = 0; j < n_notes; j++)
2411	if (REGNO (recog_data.operand[i]) == REGNO (note_reg[j])
2412	&& note_kind[j] == REG_DEAD
2413	&& TEST_HARD_REG_BIT (set: regstack->reg_set,
2414	REGNO (recog_data.operand[i])))
2415	{
2416	insn = emit_pop_insn (insn, regstack, reg: recog_data.operand[i],
2417	where: EMIT_AFTER);
2418	break;
2419	}
2420	}
2421	}
2422
2423	/* Return true if a function call is allowed to alter some or all bits
2424	of any stack reg. */
2425	static bool
2426	callee_clobbers_any_stack_reg (const function_abi & callee_abi)
2427	{
2428	for (unsigned regno = FIRST_STACK_REG; regno <= LAST_STACK_REG; regno++)
2429	if (callee_abi.clobbers_at_least_part_of_reg_p (regno))
2430	return true;
2431	return false;
2432	}
2433
2434
2435	/* Substitute stack hard reg numbers for stack virtual registers in
2436	INSN. Non-stack register numbers are not changed. REGSTACK is the
2437	current stack content. Insns may be emitted as needed to arrange the
2438	stack for the 387 based on the contents of the insn. Return whether
2439	a control flow insn was deleted in the process. */
2440
2441	static bool
2442	subst_stack_regs (rtx_insn *insn, stack_ptr regstack)
2443	{
2444	rtx *note_link, note;
2445	bool control_flow_insn_deleted = false;
2446	int i;
2447
2448	/* If the target of the call doesn't clobber any stack registers,
2449	Don't clear the arguments. */
2450	if (CALL_P (insn)
2451	&& callee_clobbers_any_stack_reg (callee_abi: insn_callee_abi (insn)))
2452	{
2453	int top = regstack->top;
2454
2455	/* If there are any floating point parameters to be passed in
2456	registers for this call, make sure they are in the right
2457	order. */
2458
2459	if (top >= 0)
2460	{
2461	straighten_stack (insn, regstack);
2462
2463	/* Now mark the arguments as dead after the call. */
2464
2465	while (regstack->top >= 0)
2466	{
2467	CLEAR_HARD_REG_BIT (set&: regstack->reg_set, FIRST_STACK_REG + regstack->top);
2468	regstack->top--;
2469	}
2470	}
2471	}
2472
2473	/* Do the actual substitution if any stack regs are mentioned.
2474	Since we only record whether entire insn mentions stack regs, and
2475	subst_stack_regs_pat only works for patterns that contain stack regs,
2476	we must check each pattern in a parallel here. A call_value_pop could
2477	fail otherwise. */
2478
2479	if (stack_regs_mentioned (insn))
2480	{
2481	int n_operands = asm_noperands (PATTERN (insn));
2482	if (n_operands >= 0)
2483	{
2484	/* This insn is an `asm' with operands. Decode the operands,
2485	decide how many are inputs, and do register substitution.
2486	Any REG_UNUSED notes will be handled by subst_asm_stack_regs. */
2487
2488	subst_asm_stack_regs (insn, regstack);
2489	return control_flow_insn_deleted;
2490	}
2491
2492	if (GET_CODE (PATTERN (insn)) == PARALLEL)
2493	for (i = 0; i < XVECLEN (PATTERN (insn), 0); i++)
2494	{
2495	if (stack_regs_mentioned_p (XVECEXP (PATTERN (insn), 0, i)))
2496	{
2497	if (GET_CODE (XVECEXP (PATTERN (insn), 0, i)) == CLOBBER)
2498	XVECEXP (PATTERN (insn), 0, i)
2499	= shallow_copy_rtx (XVECEXP (PATTERN (insn), 0, i));
2500	if (subst_stack_regs_pat (insn, regstack,
2501	XVECEXP (PATTERN (insn), 0, i)))
2502	control_flow_insn_deleted = true;
2503	}
2504	}
2505	else if (subst_stack_regs_pat (insn, regstack, pat: PATTERN (insn)))
2506	control_flow_insn_deleted = true;
2507	}
2508
2509	/* subst_stack_regs_pat may have deleted a no-op insn. If so, any
2510	REG_UNUSED will already have been dealt with, so just return. */
2511
2512	if (NOTE_P (insn) || insn->deleted ())
2513	return control_flow_insn_deleted;
2514
2515	/* If this a noreturn call, we can't insert pop insns after it.
2516	Instead, reset the stack state to empty. */
2517	if (CALL_P (insn)
2518	&& find_reg_note (insn, REG_NORETURN, NULL))
2519	{
2520	regstack->top = -1;
2521	CLEAR_HARD_REG_SET (set&: regstack->reg_set);
2522	return control_flow_insn_deleted;
2523	}
2524
2525	/* If there is a REG_UNUSED note on a stack register on this insn,
2526	the indicated reg must be popped. The REG_UNUSED note is removed,
2527	since the form of the newly emitted pop insn references the reg,
2528	making it no longer `unset'. */
2529
2530	note_link = &REG_NOTES (insn);
2531	for (note = *note_link; note; note = XEXP (note, 1))
2532	if (REG_NOTE_KIND (note) == REG_UNUSED && STACK_REG_P (XEXP (note, 0)))
2533	{
2534	*note_link = XEXP (note, 1);
2535	insn = emit_pop_insn (insn, regstack, XEXP (note, 0), where: EMIT_AFTER);
2536	}
2537	else
2538	note_link = &XEXP (note, 1);
2539
2540	return control_flow_insn_deleted;
2541	}
2542
2543	/* Change the organization of the stack so that it fits a new basic
2544	block. Some registers might have to be popped, but there can never be
2545	a register live in the new block that is not now live.
2546
2547	Insert any needed insns before or after INSN, as indicated by
2548	WHERE. OLD is the original stack layout, and NEW is the desired
2549	form. OLD is updated to reflect the code emitted, i.e., it will be
2550	the same as NEW upon return.
2551
2552	This function will not preserve block_end[]. But that information
2553	is no longer needed once this has executed. */
2554
2555	static void
2556	change_stack (rtx_insn *insn, stack_ptr old, stack_ptr new_stack,
2557	enum emit_where where)
2558	{
2559	int reg;
2560	machine_mode raw_mode = reg_raw_mode[FIRST_STACK_REG];
2561	rtx_insn *update_end = NULL;
2562	int i;
2563
2564	/* Stack adjustments for the first insn in a block update the
2565	current_block's stack_in instead of inserting insns directly.
2566	compensate_edges will add the necessary code later. */
2567	if (current_block
2568	&& starting_stack_p
2569	&& where == EMIT_BEFORE)
2570	{
2571	BLOCK_INFO (current_block)->stack_in = *new_stack;
2572	starting_stack_p = false;
2573	*old = *new_stack;
2574	return;
2575	}
2576
2577	/* We will be inserting new insns "backwards". If we are to insert
2578	after INSN, find the next insn, and insert before it. */
2579
2580	if (where == EMIT_AFTER)
2581	{
2582	if (current_block && BB_END (current_block) == insn)
2583	update_end = insn;
2584	insn = NEXT_INSN (insn);
2585	}
2586
2587	/* Initialize partially dead variables. */
2588	for (i = FIRST_STACK_REG; i < LAST_STACK_REG + 1; i++)
2589	if (TEST_HARD_REG_BIT (set: new_stack->reg_set, bit: i)
2590	&& !TEST_HARD_REG_BIT (set: old->reg_set, bit: i))
2591	{
2592	old->reg[++old->top] = i;
2593	SET_HARD_REG_BIT (set&: old->reg_set, bit: i);
2594	emit_insn_before (gen_rtx_SET (FP_MODE_REG (i, SFmode), not_a_num),
2595	insn);
2596	}
2597
2598	/* Pop any registers that are not needed in the new block. */
2599
2600	/* If the destination block's stack already has a specified layout
2601	and contains two or more registers, use a more intelligent algorithm
2602	to pop registers that minimizes the number of fxchs below. */
2603	if (new_stack->top > 0)
2604	{
2605	bool slots[REG_STACK_SIZE];
2606	int pops[REG_STACK_SIZE];
2607	int next, dest, topsrc;
2608
2609	/* First pass to determine the free slots. */
2610	for (reg = 0; reg <= new_stack->top; reg++)
2611	slots[reg] = TEST_HARD_REG_BIT (set: new_stack->reg_set, bit: old->reg[reg]);
2612
2613	/* Second pass to allocate preferred slots. */
2614	topsrc = -1;
2615	for (reg = old->top; reg > new_stack->top; reg--)
2616	if (TEST_HARD_REG_BIT (set: new_stack->reg_set, bit: old->reg[reg]))
2617	{
2618	dest = -1;
2619	for (next = 0; next <= new_stack->top; next++)
2620	if (!slots[next] && new_stack->reg[next] == old->reg[reg])
2621	{
2622	/* If this is a preference for the new top of stack, record
2623	the fact by remembering it's old->reg in topsrc. */
2624	if (next == new_stack->top)
2625	topsrc = reg;
2626	slots[next] = true;
2627	dest = next;
2628	break;
2629	}
2630	pops[reg] = dest;
2631	}
2632	else
2633	pops[reg] = reg;
2634
2635	/* Intentionally, avoid placing the top of stack in it's correct
2636	location, if we still need to permute the stack below and we
2637	can usefully place it somewhere else. This is the case if any
2638	slot is still unallocated, in which case we should place the
2639	top of stack there. */
2640	if (topsrc != -1)
2641	for (reg = 0; reg < new_stack->top; reg++)
2642	if (!slots[reg])
2643	{
2644	pops[topsrc] = reg;
2645	slots[new_stack->top] = false;
2646	slots[reg] = true;
2647	break;
2648	}
2649
2650	/* Third pass allocates remaining slots and emits pop insns. */
2651	next = new_stack->top;
2652	for (reg = old->top; reg > new_stack->top; reg--)
2653	{
2654	dest = pops[reg];
2655	if (dest == -1)
2656	{
2657	/* Find next free slot. */
2658	while (slots[next])
2659	next--;
2660	dest = next--;
2661	}
2662	emit_pop_insn (insn, regstack: old, FP_MODE_REG (old->reg[dest], raw_mode),
2663	where: EMIT_BEFORE);
2664	}
2665	}
2666	else
2667	{
2668	/* The following loop attempts to maximize the number of times we
2669	pop the top of the stack, as this permits the use of the faster
2670	ffreep instruction on platforms that support it. */
2671	int live, next;
2672
2673	live = 0;
2674	for (reg = 0; reg <= old->top; reg++)
2675	if (TEST_HARD_REG_BIT (set: new_stack->reg_set, bit: old->reg[reg]))
2676	live++;
2677
2678	next = live;
2679	while (old->top >= live)
2680	if (TEST_HARD_REG_BIT (set: new_stack->reg_set, bit: old->reg[old->top]))
2681	{
2682	while (TEST_HARD_REG_BIT (set: new_stack->reg_set, bit: old->reg[next]))
2683	next--;
2684	emit_pop_insn (insn, regstack: old, FP_MODE_REG (old->reg[next], raw_mode),
2685	where: EMIT_BEFORE);
2686	}
2687	else
2688	emit_pop_insn (insn, regstack: old, FP_MODE_REG (old->reg[old->top], raw_mode),
2689	where: EMIT_BEFORE);
2690	}
2691
2692	if (new_stack->top == -2)
2693	{
2694	/* If the new block has never been processed, then it can inherit
2695	the old stack order. */
2696
2697	new_stack->top = old->top;
2698	memcpy (dest: new_stack->reg, src: old->reg, n: sizeof (new_stack->reg));
2699	}
2700	else
2701	{
2702	/* This block has been entered before, and we must match the
2703	previously selected stack order. */
2704
2705	/* By now, the only difference should be the order of the stack,
2706	not their depth or liveliness. */
2707
2708	gcc_assert (old->reg_set == new_stack->reg_set);
2709	gcc_assert (old->top == new_stack->top);
2710
2711	/* If the stack is not empty (new_stack->top != -1), loop here emitting
2712	swaps until the stack is correct.
2713
2714	The worst case number of swaps emitted is N + 2, where N is the
2715	depth of the stack. In some cases, the reg at the top of
2716	stack may be correct, but swapped anyway in order to fix
2717	other regs. But since we never swap any other reg away from
2718	its correct slot, this algorithm will converge. */
2719
2720	if (new_stack->top != -1)
2721	do
2722	{
2723	/* Swap the reg at top of stack into the position it is
2724	supposed to be in, until the correct top of stack appears. */
2725
2726	while (old->reg[old->top] != new_stack->reg[new_stack->top])
2727	{
2728	for (reg = new_stack->top; reg >= 0; reg--)
2729	if (new_stack->reg[reg] == old->reg[old->top])
2730	break;
2731
2732	gcc_assert (reg != -1);
2733
2734	emit_swap_insn (insn, regstack: old,
2735	FP_MODE_REG (old->reg[reg], raw_mode));
2736	}
2737
2738	/* See if any regs remain incorrect. If so, bring an
2739	incorrect reg to the top of stack, and let the while loop
2740	above fix it. */
2741
2742	for (reg = new_stack->top; reg >= 0; reg--)
2743	if (new_stack->reg[reg] != old->reg[reg])
2744	{
2745	emit_swap_insn (insn, regstack: old,
2746	FP_MODE_REG (old->reg[reg], raw_mode));
2747	break;
2748	}
2749	} while (reg >= 0);
2750
2751	/* At this point there must be no differences. */
2752
2753	for (reg = old->top; reg >= 0; reg--)
2754	gcc_assert (old->reg[reg] == new_stack->reg[reg]);
2755	}
2756
2757	if (update_end)
2758	{
2759	for (update_end = NEXT_INSN (insn: update_end); update_end != insn;
2760	update_end = NEXT_INSN (insn: update_end))
2761	{
2762	set_block_for_insn (insn: update_end, bb: current_block);
2763	if (INSN_P (update_end))
2764	df_insn_rescan (update_end);
2765	}
2766	BB_END (current_block) = PREV_INSN (insn);
2767	}
2768	}
2769
2770	/* Print stack configuration. */
2771
2772	static void
2773	print_stack (FILE *file, stack_ptr s)
2774	{
2775	if (! file)
2776	return;
2777
2778	if (s->top == -2)
2779	fprintf (stream: file, format: "uninitialized\n");
2780	else if (s->top == -1)
2781	fprintf (stream: file, format: "empty\n");
2782	else
2783	{
2784	int i;
2785	fputs (s: "[ ", stream: file);
2786	for (i = 0; i <= s->top; ++i)
2787	fprintf (stream: file, format: "%d ", s->reg[i]);
2788	fputs (s: "]\n", stream: file);
2789	}
2790	}
2791
2792	/* This function was doing life analysis. We now let the regular live
2793	code do it's job, so we only need to check some extra invariants
2794	that reg-stack expects. Primary among these being that all registers
2795	are initialized before use.
2796
2797	The function returns true when code was emitted to CFG edges and
2798	commit_edge_insertions needs to be called. */
2799
2800	static bool
2801	convert_regs_entry (void)
2802	{
2803	bool inserted = false;
2804	edge e;
2805	edge_iterator ei;
2806
2807	/* Load something into each stack register live at function entry.
2808	Such live registers can be caused by uninitialized variables or
2809	functions not returning values on all paths. In order to keep
2810	the push/pop code happy, and to not scrog the register stack, we
2811	must put something in these registers. Use a QNaN.
2812
2813	Note that we are inserting converted code here. This code is
2814	never seen by the convert_regs pass. */
2815
2816	FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs)
2817	{
2818	basic_block block = e->dest;
2819	block_info bi = BLOCK_INFO (block);
2820	int reg, top = -1;
2821
2822	for (reg = LAST_STACK_REG; reg >= FIRST_STACK_REG; --reg)
2823	if (TEST_HARD_REG_BIT (set: bi->stack_in.reg_set, bit: reg))
2824	{
2825	rtx init;
2826
2827	bi->stack_in.reg[++top] = reg;
2828
2829	init = gen_rtx_SET (FP_MODE_REG (FIRST_STACK_REG, SFmode),
2830	not_a_num);
2831	insert_insn_on_edge (init, e);
2832	inserted = true;
2833	}
2834
2835	bi->stack_in.top = top;
2836	}
2837
2838	return inserted;
2839	}
2840
2841	/* Construct the desired stack for function exit. This will either
2842	be `empty', or the function return value at top-of-stack. */
2843
2844	static void
2845	convert_regs_exit (void)
2846	{
2847	int value_reg_low, value_reg_high;
2848	stack_ptr output_stack;
2849	rtx retvalue;
2850
2851	retvalue = stack_result (decl: current_function_decl);
2852	value_reg_low = value_reg_high = -1;
2853	if (retvalue)
2854	{
2855	value_reg_low = REGNO (retvalue);
2856	value_reg_high = END_REGNO (x: retvalue) - 1;
2857	}
2858
2859	output_stack = &BLOCK_INFO (EXIT_BLOCK_PTR_FOR_FN (cfun))->stack_in;
2860	if (value_reg_low == -1)
2861	output_stack->top = -1;
2862	else
2863	{
2864	int reg;
2865
2866	output_stack->top = value_reg_high - value_reg_low;
2867	for (reg = value_reg_low; reg <= value_reg_high; ++reg)
2868	{
2869	output_stack->reg[value_reg_high - reg] = reg;
2870	SET_HARD_REG_BIT (set&: output_stack->reg_set, bit: reg);
2871	}
2872	}
2873	}
2874
2875	/* Copy the stack info from the end of edge E's source block to the
2876	start of E's destination block. */
2877
2878	static void
2879	propagate_stack (edge e)
2880	{
2881	stack_ptr src_stack = &BLOCK_INFO (e->src)->stack_out;
2882	stack_ptr dest_stack = &BLOCK_INFO (e->dest)->stack_in;
2883	int reg;
2884
2885	/* Preserve the order of the original stack, but check whether
2886	any pops are needed. */
2887	dest_stack->top = -1;
2888	for (reg = 0; reg <= src_stack->top; ++reg)
2889	if (TEST_HARD_REG_BIT (set: dest_stack->reg_set, bit: src_stack->reg[reg]))
2890	dest_stack->reg[++dest_stack->top] = src_stack->reg[reg];
2891
2892	/* Push in any partially dead values. */
2893	for (reg = FIRST_STACK_REG; reg < LAST_STACK_REG + 1; reg++)
2894	if (TEST_HARD_REG_BIT (set: dest_stack->reg_set, bit: reg)
2895	&& !TEST_HARD_REG_BIT (set: src_stack->reg_set, bit: reg))
2896	dest_stack->reg[++dest_stack->top] = reg;
2897	}
2898
2899
2900	/* Adjust the stack of edge E's source block on exit to match the stack
2901	of it's target block upon input. The stack layouts of both blocks
2902	should have been defined by now. */
2903
2904	static bool
2905	compensate_edge (edge e)
2906	{
2907	basic_block source = e->src, target = e->dest;
2908	stack_ptr target_stack = &BLOCK_INFO (target)->stack_in;
2909	stack_ptr source_stack = &BLOCK_INFO (source)->stack_out;
2910	struct stack_def regstack;
2911	int reg;
2912
2913	if (dump_file)
2914	fprintf (stream: dump_file, format: "Edge %d->%d: ", source->index, target->index);
2915
2916	gcc_assert (target_stack->top != -2);
2917
2918	/* Check whether stacks are identical. */
2919	if (target_stack->top == source_stack->top)
2920	{
2921	for (reg = target_stack->top; reg >= 0; --reg)
2922	if (target_stack->reg[reg] != source_stack->reg[reg])
2923	break;
2924
2925	if (reg == -1)
2926	{
2927	if (dump_file)
2928	fprintf (stream: dump_file, format: "no changes needed\n");
2929	return false;
2930	}
2931	}
2932
2933	if (dump_file)
2934	{
2935	fprintf (stream: dump_file, format: "correcting stack to ");
2936	print_stack (file: dump_file, s: target_stack);
2937	}
2938
2939	/* Abnormal calls may appear to have values live in st(0), but the
2940	abnormal return path will not have actually loaded the values. */
2941	if (e->flags & EDGE_ABNORMAL_CALL)
2942	{
2943	/* Assert that the lifetimes are as we expect -- one value
2944	live at st(0) on the end of the source block, and no
2945	values live at the beginning of the destination block.
2946	For complex return values, we may have st(1) live as well. */
2947	gcc_assert (source_stack->top == 0 || source_stack->top == 1);
2948	gcc_assert (target_stack->top == -1);
2949	return false;
2950	}
2951
2952	/* Handle non-call EH edges specially. The normal return path have
2953	values in registers. These will be popped en masse by the unwind
2954	library. */
2955	if (e->flags & EDGE_EH)
2956	{
2957	gcc_assert (target_stack->top == -1);
2958	return false;
2959	}
2960
2961	/* We don't support abnormal edges. Global takes care to
2962	avoid any live register across them, so we should never
2963	have to insert instructions on such edges. */
2964	gcc_assert (! (e->flags & EDGE_ABNORMAL));
2965
2966	/* Make a copy of source_stack as change_stack is destructive. */
2967	regstack = *source_stack;
2968
2969	/* It is better to output directly to the end of the block
2970	instead of to the edge, because emit_swap can do minimal
2971	insn scheduling. We can do this when there is only one
2972	edge out, and it is not abnormal. */
2973	if (EDGE_COUNT (source->succs) == 1)
2974	{
2975	current_block = source;
2976	change_stack (BB_END (source), old: &regstack, new_stack: target_stack,
2977	where: (JUMP_P (BB_END (source)) ? EMIT_BEFORE : EMIT_AFTER));
2978	}
2979	else
2980	{
2981	rtx_insn *seq;
2982	rtx_note *after;
2983
2984	current_block = NULL;
2985	start_sequence ();
2986
2987	/* ??? change_stack needs some point to emit insns after. */
2988	after = emit_note (NOTE_INSN_DELETED);
2989
2990	change_stack (insn: after, old: &regstack, new_stack: target_stack, where: EMIT_BEFORE);
2991
2992	seq = get_insns ();
2993	end_sequence ();
2994
2995	set_insn_locations (seq, e->goto_locus);
2996	insert_insn_on_edge (seq, e);
2997	return true;
2998	}
2999	return false;
3000	}
3001
3002	/* Traverse all non-entry edges in the CFG, and emit the necessary
3003	edge compensation code to change the stack from stack_out of the
3004	source block to the stack_in of the destination block. */
3005
3006	static bool
3007	compensate_edges (void)
3008	{
3009	bool inserted = false;
3010	basic_block bb;
3011
3012	starting_stack_p = false;
3013
3014	FOR_EACH_BB_FN (bb, cfun)
3015	if (bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
3016	{
3017	edge e;
3018	edge_iterator ei;
3019
3020	FOR_EACH_EDGE (e, ei, bb->succs)
3021	if (compensate_edge (e))
3022	inserted = true;
3023	}
3024	return inserted;
3025	}
3026
3027	/* Select the better of two edges E1 and E2 to use to determine the
3028	stack layout for their shared destination basic block. This is
3029	typically the more frequently executed. The edge E1 may be NULL
3030	(in which case E2 is returned), but E2 is always non-NULL. */
3031
3032	static edge
3033	better_edge (edge e1, edge e2)
3034	{
3035	if (!e1)
3036	return e2;
3037
3038	if (e1->count () > e2->count ())
3039	return e1;
3040	if (e1->count () < e2->count ())
3041	return e2;
3042
3043	/* Prefer critical edges to minimize inserting compensation code on
3044	critical edges. */
3045
3046	if (EDGE_CRITICAL_P (e1) != EDGE_CRITICAL_P (e2))
3047	return EDGE_CRITICAL_P (e1) ? e1 : e2;
3048
3049	/* Avoid non-deterministic behavior. */
3050	return (e1->src->index < e2->src->index) ? e1 : e2;
3051	}
3052
3053	/* Convert stack register references in one block. Return true if the CFG
3054	has been modified in the process. */
3055
3056	static bool
3057	convert_regs_1 (basic_block block)
3058	{
3059	struct stack_def regstack;
3060	block_info bi = BLOCK_INFO (block);
3061	int reg;
3062	rtx_insn *insn, *next;
3063	bool control_flow_insn_deleted = false;
3064	bool cfg_altered = false;
3065	int debug_insns_with_starting_stack = 0;
3066
3067	/* Choose an initial stack layout, if one hasn't already been chosen. */
3068	if (bi->stack_in.top == -2)
3069	{
3070	edge e, beste = NULL;
3071	edge_iterator ei;
3072
3073	/* Select the best incoming edge (typically the most frequent) to
3074	use as a template for this basic block. */
3075	FOR_EACH_EDGE (e, ei, block->preds)
3076	if (BLOCK_INFO (e->src)->done)
3077	beste = better_edge (e1: beste, e2: e);
3078
3079	if (beste)
3080	propagate_stack (e: beste);
3081	else
3082	{
3083	/* No predecessors. Create an arbitrary input stack. */
3084	bi->stack_in.top = -1;
3085	for (reg = LAST_STACK_REG; reg >= FIRST_STACK_REG; --reg)
3086	if (TEST_HARD_REG_BIT (set: bi->stack_in.reg_set, bit: reg))
3087	bi->stack_in.reg[++bi->stack_in.top] = reg;
3088	}
3089	}
3090
3091	if (dump_file)
3092	{
3093	fprintf (stream: dump_file, format: "\nBasic block %d\nInput stack: ", block->index);
3094	print_stack (file: dump_file, s: &bi->stack_in);
3095	}
3096
3097	/* Process all insns in this block. Keep track of NEXT so that we
3098	don't process insns emitted while substituting in INSN. */
3099	current_block = block;
3100	next = BB_HEAD (block);
3101	regstack = bi->stack_in;
3102	starting_stack_p = true;
3103
3104	do
3105	{
3106	insn = next;
3107	next = NEXT_INSN (insn);
3108
3109	/* Ensure we have not missed a block boundary. */
3110	gcc_assert (next);
3111	if (insn == BB_END (block))
3112	next = NULL;
3113
3114	/* Don't bother processing unless there is a stack reg
3115	mentioned or if it's a CALL_INSN. */
3116	if (DEBUG_BIND_INSN_P (insn))
3117	{
3118	if (starting_stack_p)
3119	debug_insns_with_starting_stack++;
3120	else
3121	{
3122	subst_all_stack_regs_in_debug_insn (insn, regstack: &regstack);
3123
3124	/* Nothing must ever die at a debug insn. If something
3125	is referenced in it that becomes dead, it should have
3126	died before and the reference in the debug insn
3127	should have been removed so as to avoid changing code
3128	generation. */
3129	gcc_assert (!find_reg_note (insn, REG_DEAD, NULL));
3130	}
3131	}
3132	else if (stack_regs_mentioned (insn)
3133	|| CALL_P (insn))
3134	{
3135	if (dump_file)
3136	{
3137	fprintf (stream: dump_file, format: " insn %d input stack: ",
3138	INSN_UID (insn));
3139	print_stack (file: dump_file, s: &regstack);
3140	}
3141	if (subst_stack_regs (insn, regstack: &regstack))
3142	control_flow_insn_deleted = true;
3143	starting_stack_p = false;
3144	}
3145	}
3146	while (next);
3147
3148	if (debug_insns_with_starting_stack)
3149	{
3150	/* Since it's the first non-debug instruction that determines
3151	the stack requirements of the current basic block, we refrain
3152	from updating debug insns before it in the loop above, and
3153	fix them up here. */
3154	for (insn = BB_HEAD (block); debug_insns_with_starting_stack;
3155	insn = NEXT_INSN (insn))
3156	{
3157	if (!DEBUG_BIND_INSN_P (insn))
3158	continue;
3159
3160	debug_insns_with_starting_stack--;
3161	subst_all_stack_regs_in_debug_insn (insn, regstack: &bi->stack_in);
3162	}
3163	}
3164
3165	if (dump_file)
3166	{
3167	fprintf (stream: dump_file, format: "Expected live registers [");
3168	for (reg = FIRST_STACK_REG; reg <= LAST_STACK_REG; ++reg)
3169	if (TEST_HARD_REG_BIT (set: bi->out_reg_set, bit: reg))
3170	fprintf (stream: dump_file, format: " %d", reg);
3171	fprintf (stream: dump_file, format: " ]\nOutput stack: ");
3172	print_stack (file: dump_file, s: &regstack);
3173	}
3174
3175	insn = BB_END (block);
3176	if (JUMP_P (insn))
3177	insn = PREV_INSN (insn);
3178
3179	/* If the function is declared to return a value, but it returns one
3180	in only some cases, some registers might come live here. Emit
3181	necessary moves for them. */
3182
3183	for (reg = FIRST_STACK_REG; reg <= LAST_STACK_REG; ++reg)
3184	{
3185	if (TEST_HARD_REG_BIT (set: bi->out_reg_set, bit: reg)
3186	&& ! TEST_HARD_REG_BIT (set: regstack.reg_set, bit: reg))
3187	{
3188	rtx set;
3189
3190	if (dump_file)
3191	fprintf (stream: dump_file, format: "Emitting insn initializing reg %d\n", reg);
3192
3193	set = gen_rtx_SET (FP_MODE_REG (reg, SFmode), not_a_num);
3194	insn = emit_insn_after (set, insn);
3195	if (subst_stack_regs (insn, regstack: &regstack))
3196	control_flow_insn_deleted = true;
3197	}
3198	}
3199
3200	/* Amongst the insns possibly deleted during the substitution process above,
3201	might have been the only trapping insn in the block. We purge the now
3202	possibly dead EH edges here to avoid an ICE from fixup_abnormal_edges,
3203	called at the end of convert_regs. The order in which we process the
3204	blocks ensures that we never delete an already processed edge.
3205
3206	Note that, at this point, the CFG may have been damaged by the emission
3207	of instructions after an abnormal call, which moves the basic block end
3208	(and is the reason why we call fixup_abnormal_edges later). So we must
3209	be sure that the trapping insn has been deleted before trying to purge
3210	dead edges, otherwise we risk purging valid edges.
3211
3212	??? We are normally supposed not to delete trapping insns, so we pretend
3213	that the insns deleted above don't actually trap. It would have been
3214	better to detect this earlier and avoid creating the EH edge in the first
3215	place, still, but we don't have enough information at that time. */
3216
3217	if (control_flow_insn_deleted && purge_dead_edges (block))
3218	cfg_altered = true;
3219
3220	/* Something failed if the stack lives don't match. If we had malformed
3221	asms, we zapped the instruction itself, but that didn't produce the
3222	same pattern of register kills as before. */
3223
3224	gcc_assert (regstack.reg_set == bi->out_reg_set || any_malformed_asm);
3225	bi->stack_out = regstack;
3226	bi->done = true;
3227
3228	return cfg_altered;
3229	}
3230
3231	/* Convert registers in all blocks reachable from BLOCK. Return true if the
3232	CFG has been modified in the process. */
3233
3234	static bool
3235	convert_regs_2 (basic_block block)
3236	{
3237	basic_block *stack, *sp;
3238	bool cfg_altered = false;
3239
3240	/* We process the blocks in a top-down manner, in a way such that one block
3241	is only processed after all its predecessors. The number of predecessors
3242	of every block has already been computed. */
3243
3244	stack = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
3245	sp = stack;
3246
3247	*sp++ = block;
3248
3249	do
3250	{
3251	edge e;
3252	edge_iterator ei;
3253
3254	block = *--sp;
3255
3256	/* Processing BLOCK is achieved by convert_regs_1, which may purge
3257	some dead EH outgoing edge after the deletion of the trapping
3258	insn inside the block. Since the number of predecessors of
3259	BLOCK's successors was computed based on the initial edge set,
3260	we check the necessity to process some of these successors
3261	before such an edge deletion may happen. However, there is
3262	a pitfall: if BLOCK is the only predecessor of a successor and
3263	the edge between them happens to be deleted, the successor
3264	becomes unreachable and should not be processed. The problem
3265	is that there is no way to preventively detect this case so we
3266	stack the successor in all cases and hand over the task of
3267	fixing up the discrepancy to convert_regs_1. */
3268
3269	FOR_EACH_EDGE (e, ei, block->succs)
3270	if (! (e->flags & EDGE_DFS_BACK))
3271	{
3272	BLOCK_INFO (e->dest)->predecessors--;
3273	if (!BLOCK_INFO (e->dest)->predecessors)
3274	*sp++ = e->dest;
3275	}
3276
3277	if (convert_regs_1 (block))
3278	cfg_altered = true;
3279	}
3280	while (sp != stack);
3281
3282	free (ptr: stack);
3283
3284	return cfg_altered;
3285	}
3286
3287	/* Traverse all basic blocks in a function, converting the register
3288	references in each insn from the "flat" register file that gcc uses,
3289	to the stack-like registers the 387 uses. */
3290
3291	static void
3292	convert_regs (void)
3293	{
3294	bool cfg_altered = false;
3295	bool inserted;
3296	basic_block b;
3297	edge e;
3298	edge_iterator ei;
3299
3300	/* Initialize uninitialized registers on function entry. */
3301	inserted = convert_regs_entry ();
3302
3303	/* Construct the desired stack for function exit. */
3304	convert_regs_exit ();
3305	BLOCK_INFO (EXIT_BLOCK_PTR_FOR_FN (cfun))->done = true;
3306
3307	/* ??? Future: process inner loops first, and give them arbitrary
3308	initial stacks which emit_swap_insn can modify. This ought to
3309	prevent double fxch that often appears at the head of a loop. */
3310
3311	/* Process all blocks reachable from all entry points. */
3312	FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs)
3313	if (convert_regs_2 (block: e->dest))
3314	cfg_altered = true;
3315
3316	/* ??? Process all unreachable blocks. Though there's no excuse
3317	for keeping these even when not optimizing. */
3318	FOR_EACH_BB_FN (b, cfun)
3319	{
3320	block_info bi = BLOCK_INFO (b);
3321
3322	if (!bi->done && convert_regs_2 (block: b))
3323	cfg_altered = true;
3324	}
3325
3326	/* We must fix up abnormal edges before inserting compensation code
3327	because both mechanisms insert insns on edges. */
3328	if (fixup_abnormal_edges ())
3329	inserted = true;
3330
3331	if (compensate_edges ())
3332	inserted = true;
3333
3334	clear_aux_for_blocks ();
3335
3336	if (inserted)
3337	commit_edge_insertions ();
3338
3339	if (cfg_altered)
3340	cleanup_cfg (0);
3341
3342	if (dump_file)
3343	fputc (c: '\n', stream: dump_file);
3344	}
3345
3346	/* Convert register usage from "flat" register file usage to a "stack
3347	register file. FILE is the dump file, if used.
3348
3349	Construct a CFG and run life analysis. Then convert each insn one
3350	by one. Run a last cleanup_cfg pass, if optimizing, to eliminate
3351	code duplication created when the converter inserts pop insns on
3352	the edges. */
3353
3354	static bool
3355	reg_to_stack (void)
3356	{
3357	basic_block bb;
3358	int i;
3359	int max_uid;
3360
3361	/* Clean up previous run. */
3362	stack_regs_mentioned_data.release ();
3363
3364	/* See if there is something to do. Flow analysis is quite
3365	expensive so we might save some compilation time. */
3366	for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
3367	if (df_regs_ever_live_p (i))
3368	break;
3369	if (i > LAST_STACK_REG)
3370	return false;
3371
3372	df_note_add_problem ();
3373	df_analyze ();
3374
3375	mark_dfs_back_edges ();
3376
3377	/* Set up block info for each basic block. */
3378	alloc_aux_for_blocks (sizeof (struct block_info_def));
3379	FOR_EACH_BB_FN (bb, cfun)
3380	{
3381	block_info bi = BLOCK_INFO (bb);
3382	edge_iterator ei;
3383	edge e;
3384	int reg;
3385
3386	FOR_EACH_EDGE (e, ei, bb->preds)
3387	if (!(e->flags & EDGE_DFS_BACK)
3388	&& e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun))
3389	bi->predecessors++;
3390
3391	/* Set current register status at last instruction `uninitialized'. */
3392	bi->stack_in.top = -2;
3393
3394	/* Copy live_at_end and live_at_start into temporaries. */
3395	for (reg = FIRST_STACK_REG; reg <= LAST_STACK_REG; reg++)
3396	{
3397	if (REGNO_REG_SET_P (DF_LR_OUT (bb), reg))
3398	SET_HARD_REG_BIT (set&: bi->out_reg_set, bit: reg);
3399	if (REGNO_REG_SET_P (DF_LR_IN (bb), reg))
3400	SET_HARD_REG_BIT (set&: bi->stack_in.reg_set, bit: reg);
3401	}
3402	}
3403
3404	/* Create the replacement registers up front. */
3405	for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
3406	{
3407	machine_mode mode;
3408	FOR_EACH_MODE_IN_CLASS (mode, MODE_FLOAT)
3409	FP_MODE_REG (i, mode) = gen_rtx_REG (mode, i);
3410	FOR_EACH_MODE_IN_CLASS (mode, MODE_COMPLEX_FLOAT)
3411	FP_MODE_REG (i, mode) = gen_rtx_REG (mode, i);
3412	}
3413
3414	ix86_flags_rtx = gen_rtx_REG (CCmode, FLAGS_REG);
3415
3416	/* A QNaN for initializing uninitialized variables.
3417
3418	??? We can't load from constant memory in PIC mode, because
3419	we're inserting these instructions before the prologue and
3420	the PIC register hasn't been set up. In that case, fall back
3421	on zero, which we can get from `fldz'. */
3422
3423	if ((flag_pic && !TARGET_64BIT)
3424	|| ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
3425	not_a_num = CONST0_RTX (SFmode);
3426	else
3427	{
3428	REAL_VALUE_TYPE r;
3429
3430	real_nan (&r, "", 1, SFmode);
3431	not_a_num = const_double_from_real_value (r, SFmode);
3432	not_a_num = force_const_mem (SFmode, not_a_num);
3433	}
3434
3435	/* Allocate a cache for stack_regs_mentioned. */
3436	max_uid = get_max_uid ();
3437	stack_regs_mentioned_data.create (nelems: max_uid + 1);
3438	memset (s: stack_regs_mentioned_data.address (),
3439	c: 0, n: sizeof (char) * (max_uid + 1));
3440
3441	convert_regs ();
3442	any_malformed_asm = false;
3443
3444	free_aux_for_blocks ();
3445	return true;
3446	}
3447	#endif /* STACK_REGS */
3448
3449	namespace {
3450
3451	const pass_data pass_data_stack_regs =
3452	{
3453	.type: RTL_PASS, /* type */
3454	.name: "*stack_regs", /* name */
3455	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
3456	.tv_id: TV_REG_STACK, /* tv_id */
3457	.properties_required: 0, /* properties_required */
3458	.properties_provided: 0, /* properties_provided */
3459	.properties_destroyed: 0, /* properties_destroyed */
3460	.todo_flags_start: 0, /* todo_flags_start */
3461	.todo_flags_finish: 0, /* todo_flags_finish */
3462	};
3463
3464	class pass_stack_regs : public rtl_opt_pass
3465	{
3466	public:
3467	pass_stack_regs (gcc::context *ctxt)
3468	: rtl_opt_pass (pass_data_stack_regs, ctxt)
3469	{}
3470
3471	/* opt_pass methods: */
3472	bool gate (function *) final override
3473	{
3474	#ifdef STACK_REGS
3475	return true;
3476	#else
3477	return false;
3478	#endif
3479	}
3480
3481	}; // class pass_stack_regs
3482
3483	} // anon namespace
3484
3485	rtl_opt_pass *
3486	make_pass_stack_regs (gcc::context *ctxt)
3487	{
3488	return new pass_stack_regs (ctxt);
3489	}
3490
3491	/* Convert register usage from flat register file usage to a stack
3492	register file. */
3493	static unsigned int
3494	rest_of_handle_stack_regs (void)
3495	{
3496	#ifdef STACK_REGS
3497	if (reg_to_stack ())
3498	df_insn_rescan_all ();
3499	regstack_completed = 1;
3500	#endif
3501	return 0;
3502	}
3503
3504	namespace {
3505
3506	const pass_data pass_data_stack_regs_run =
3507	{
3508	.type: RTL_PASS, /* type */
3509	.name: "stack", /* name */
3510	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
3511	.tv_id: TV_REG_STACK, /* tv_id */
3512	.properties_required: 0, /* properties_required */
3513	.properties_provided: 0, /* properties_provided */
3514	.properties_destroyed: 0, /* properties_destroyed */
3515	.todo_flags_start: 0, /* todo_flags_start */
3516	TODO_df_finish, /* todo_flags_finish */
3517	};
3518
3519	class pass_stack_regs_run : public rtl_opt_pass
3520	{
3521	public:
3522	pass_stack_regs_run (gcc::context *ctxt)
3523	: rtl_opt_pass (pass_data_stack_regs_run, ctxt)
3524	{}
3525
3526	/* opt_pass methods: */
3527	unsigned int execute (function *) final override
3528	{
3529	return rest_of_handle_stack_regs ();
3530	}
3531
3532	}; // class pass_stack_regs_run
3533
3534	} // anon namespace
3535
3536	rtl_opt_pass *
3537	make_pass_stack_regs_run (gcc::context *ctxt)
3538	{
3539	return new pass_stack_regs_run (ctxt);
3540	}
3541