1	/* Statement simplification on GIMPLE.
2	Copyright (C) 2010-2026 Free Software Foundation, Inc.
3	Split out from tree-ssa-ccp.cc.
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify it
8	under the terms of the GNU General Public License as published by the
9	Free Software Foundation; either version 3, or (at your option) any
10	later version.
11
12	GCC is distributed in the hope that it will be useful, but WITHOUT
13	ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15	for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21	#include "config.h"
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "target.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "gimple.h"
29	#include "predict.h"
30	#include "ssa.h"
31	#include "cgraph.h"
32	#include "gimple-pretty-print.h"
33	#include "gimple-ssa-warn-access.h"
34	#include "gimple-ssa-warn-restrict.h"
35	#include "fold-const.h"
36	#include "stmt.h"
37	#include "expr.h"
38	#include "stor-layout.h"
39	#include "dumpfile.h"
40	#include "gimple-iterator.h"
41	#include "tree-pass.h"
42	#include "gimple-fold.h"
43	#include "gimplify.h"
44	#include "tree-into-ssa.h"
45	#include "tree-dfa.h"
46	#include "tree-object-size.h"
47	#include "tree-ssa.h"
48	#include "tree-ssa-propagate.h"
49	#include "ipa-utils.h"
50	#include "tree-ssa-address.h"
51	#include "langhooks.h"
52	#include "gimplify-me.h"
53	#include "dbgcnt.h"
54	#include "builtins.h"
55	#include "tree-eh.h"
56	#include "gimple-match.h"
57	#include "gomp-constants.h"
58	#include "optabs-query.h"
59	#include "omp-general.h"
60	#include "tree-cfg.h"
61	#include "fold-const-call.h"
62	#include "stringpool.h"
63	#include "attribs.h"
64	#include "asan.h"
65	#include "diagnostic-core.h"
66	#include "intl.h"
67	#include "calls.h"
68	#include "tree-vector-builder.h"
69	#include "tree-ssa-strlen.h"
70	#include "varasm.h"
71	#include "internal-fn.h"
72	#include "gimple-range.h"
73
74	enum strlen_range_kind {
75	/* Compute the exact constant string length. */
76	SRK_STRLEN,
77	/* Compute the maximum constant string length. */
78	SRK_STRLENMAX,
79	/* Compute a range of string lengths bounded by object sizes. When
80	the length of a string cannot be determined, consider as the upper
81	bound the size of the enclosing object the string may be a member
82	or element of. Also determine the size of the largest character
83	array the string may refer to. */
84	SRK_LENRANGE,
85	/* Determine the integer value of the argument (not string length). */
86	SRK_INT_VALUE
87	};
88
89	static bool
90	get_range_strlen (tree, bitmap, strlen_range_kind, c_strlen_data *, unsigned);
91
92	/* Return true when DECL can be referenced from current unit.
93	FROM_DECL (if non-null) specify constructor of variable DECL was taken from.
94	We can get declarations that are not possible to reference for various
95	reasons:
96
97	1) When analyzing C++ virtual tables.
98	C++ virtual tables do have known constructors even
99	when they are keyed to other compilation unit.
100	Those tables can contain pointers to methods and vars
101	in other units. Those methods have both STATIC and EXTERNAL
102	set.
103	2) In WHOPR mode devirtualization might lead to reference
104	to method that was partitioned elsehwere.
105	In this case we have static VAR_DECL or FUNCTION_DECL
106	that has no corresponding callgraph/varpool node
107	declaring the body.
108	3) COMDAT functions referred by external vtables that
109	we devirtualize only during final compilation stage.
110	At this time we already decided that we will not output
111	the function body and thus we can't reference the symbol
112	directly. */
113
114	static bool
115	can_refer_decl_in_current_unit_p (tree decl, tree from_decl)
116	{
117	varpool_node *vnode;
118	struct cgraph_node *node;
119	symtab_node *snode;
120
121	if (DECL_ABSTRACT_P (decl))
122	return false;
123
124	/* We are concerned only about static/external vars and functions. */
125	if ((!TREE_STATIC (decl) && !DECL_EXTERNAL (decl))
126	|| !VAR_OR_FUNCTION_DECL_P (decl))
127	return true;
128
129	/* Static objects can be referred only if they are defined and not optimized
130	out yet. */
131	if (!TREE_PUBLIC (decl))
132	{
133	if (DECL_EXTERNAL (decl))
134	return false;
135	/* Before we start optimizing unreachable code we can be sure all
136	static objects are defined. */
137	if (symtab->function_flags_ready)
138	return true;
139	snode = symtab_node::get (decl);
140	if (!snode || !snode->definition)
141	return false;
142	node = dyn_cast <cgraph_node *> (p: snode);
143	return !node || !node->inlined_to;
144	}
145
146	/* We will later output the initializer, so we can refer to it.
147	So we are concerned only when DECL comes from initializer of
148	external var or var that has been optimized out. */
149	if (!from_decl
150	|| !VAR_P (from_decl)
151	|| (!DECL_EXTERNAL (from_decl)
152	&& (vnode = varpool_node::get (decl: from_decl)) != NULL
153	&& vnode->definition)
154	|| (flag_ltrans
155	&& (vnode = varpool_node::get (decl: from_decl)) != NULL
156	&& vnode->in_other_partition))
157	return true;
158	/* We are folding reference from external vtable. The vtable may reffer
159	to a symbol keyed to other compilation unit. The other compilation
160	unit may be in separate DSO and the symbol may be hidden. */
161	if (DECL_VISIBILITY_SPECIFIED (decl)
162	&& DECL_EXTERNAL (decl)
163	&& DECL_VISIBILITY (decl) != VISIBILITY_DEFAULT
164	&& (!(snode = symtab_node::get (decl)) || !snode->in_other_partition))
165	return false;
166	/* When function is public, we always can introduce new reference.
167	Exception are the COMDAT functions where introducing a direct
168	reference imply need to include function body in the curren tunit. */
169	if (TREE_PUBLIC (decl) && !DECL_COMDAT (decl))
170	return true;
171	/* We have COMDAT. We are going to check if we still have definition
172	or if the definition is going to be output in other partition.
173	Bypass this when gimplifying; all needed functions will be produced.
174
175	As observed in PR20991 for already optimized out comdat virtual functions
176	it may be tempting to not necessarily give up because the copy will be
177	output elsewhere when corresponding vtable is output.
178	This is however not possible - ABI specify that COMDATs are output in
179	units where they are used and when the other unit was compiled with LTO
180	it is possible that vtable was kept public while the function itself
181	was privatized. */
182	if (!symtab->function_flags_ready)
183	return true;
184
185	snode = symtab_node::get (decl);
186	if (!snode
187	|| ((!snode->definition || DECL_EXTERNAL (decl))
188	&& (!snode->in_other_partition
189	|| (!snode->forced_by_abi && !snode->force_output))))
190	return false;
191	node = dyn_cast <cgraph_node *> (p: snode);
192	return !node || !node->inlined_to;
193	}
194
195	/* CVAL is value taken from DECL_INITIAL of variable. Try to transform it into
196	acceptable form for is_gimple_min_invariant.
197	FROM_DECL (if non-NULL) specify variable whose constructor contains CVAL. */
198
199	tree
200	canonicalize_constructor_val (tree cval, tree from_decl)
201	{
202	if (CONSTANT_CLASS_P (cval))
203	return cval;
204
205	tree orig_cval = cval;
206	STRIP_NOPS (cval);
207	if (TREE_CODE (cval) == POINTER_PLUS_EXPR
208	&& TREE_CODE (TREE_OPERAND (cval, 1)) == INTEGER_CST)
209	{
210	tree ptr = TREE_OPERAND (cval, 0);
211	if (is_gimple_min_invariant (ptr))
212	cval = build1_loc (EXPR_LOCATION (cval),
213	code: ADDR_EXPR, TREE_TYPE (ptr),
214	fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (ptr)),
215	ptr,
216	fold_convert (ptr_type_node,
217	TREE_OPERAND (cval, 1))));
218	}
219	if (TREE_CODE (cval) == ADDR_EXPR)
220	{
221	tree base = NULL_TREE;
222	if (TREE_CODE (TREE_OPERAND (cval, 0)) == COMPOUND_LITERAL_EXPR)
223	{
224	base = COMPOUND_LITERAL_EXPR_DECL (TREE_OPERAND (cval, 0));
225	if (base)
226	TREE_OPERAND (cval, 0) = base;
227	}
228	else
229	base = get_base_address (TREE_OPERAND (cval, 0));
230	if (!base)
231	return NULL_TREE;
232
233	if (VAR_OR_FUNCTION_DECL_P (base)
234	&& !can_refer_decl_in_current_unit_p (decl: base, from_decl))
235	return NULL_TREE;
236	if (TREE_TYPE (base) == error_mark_node)
237	return NULL_TREE;
238	if (VAR_P (base))
239	/* ??? We should be able to assert that TREE_ADDRESSABLE is set,
240	but since the use can be in a debug stmt we can't. */
241	;
242	else if (TREE_CODE (base) == FUNCTION_DECL)
243	{
244	/* Make sure we create a cgraph node for functions we'll reference.
245	They can be non-existent if the reference comes from an entry
246	of an external vtable for example. */
247	cgraph_node::get_create (base);
248	}
249	/* Fixup types in global initializers. */
250	if (TREE_TYPE (TREE_TYPE (cval)) != TREE_TYPE (TREE_OPERAND (cval, 0)))
251	cval = build_fold_addr_expr (TREE_OPERAND (cval, 0));
252
253	if (!useless_type_conversion_p (TREE_TYPE (orig_cval), TREE_TYPE (cval)))
254	cval = fold_convert (TREE_TYPE (orig_cval), cval);
255	return cval;
256	}
257	/* In CONSTRUCTORs we may see unfolded constants like (int (*) ()) 0. */
258	if (TREE_CODE (cval) == INTEGER_CST)
259	{
260	if (TREE_OVERFLOW_P (cval))
261	cval = drop_tree_overflow (cval);
262	if (!useless_type_conversion_p (TREE_TYPE (orig_cval), TREE_TYPE (cval)))
263	cval = fold_convert (TREE_TYPE (orig_cval), cval);
264	return cval;
265	}
266	return orig_cval;
267	}
268
269	/* If SYM is a constant variable with known value, return the value.
270	NULL_TREE is returned otherwise. */
271
272	tree
273	get_symbol_constant_value (tree sym)
274	{
275	tree val = ctor_for_folding (sym);
276	if (val != error_mark_node)
277	{
278	if (val)
279	{
280	val = canonicalize_constructor_val (cval: unshare_expr (val), from_decl: sym);
281	if (val
282	&& is_gimple_min_invariant (val)
283	&& useless_type_conversion_p (TREE_TYPE (sym), TREE_TYPE (val)))
284	return val;
285	else
286	return NULL_TREE;
287	}
288	/* Variables declared 'const' without an initializer
289	have zero as the initializer if they may not be
290	overridden at link or run time. */
291	if (!val
292	&& is_gimple_reg_type (TREE_TYPE (sym)))
293	return build_zero_cst (TREE_TYPE (sym));
294	}
295
296	return NULL_TREE;
297	}
298
299
300
301	/* Subroutine of fold_stmt. We perform constant folding of the
302	memory reference tree EXPR. */
303
304	static tree
305	maybe_fold_reference (tree expr)
306	{
307	tree result = NULL_TREE;
308
309	if ((TREE_CODE (expr) == VIEW_CONVERT_EXPR
310	|| TREE_CODE (expr) == REALPART_EXPR
311	|| TREE_CODE (expr) == IMAGPART_EXPR)
312	&& CONSTANT_CLASS_P (TREE_OPERAND (expr, 0)))
313	result = fold_unary_loc (EXPR_LOCATION (expr),
314	TREE_CODE (expr),
315	TREE_TYPE (expr),
316	TREE_OPERAND (expr, 0));
317	else if (TREE_CODE (expr) == BIT_FIELD_REF
318	&& CONSTANT_CLASS_P (TREE_OPERAND (expr, 0)))
319	result = fold_ternary_loc (EXPR_LOCATION (expr),
320	TREE_CODE (expr),
321	TREE_TYPE (expr),
322	TREE_OPERAND (expr, 0),
323	TREE_OPERAND (expr, 1),
324	TREE_OPERAND (expr, 2));
325	else
326	result = fold_const_aggregate_ref (expr);
327
328	if (result && is_gimple_min_invariant (result))
329	return result;
330
331	return NULL_TREE;
332	}
333
334	/* Return true if EXPR is an acceptable right-hand-side for a
335	GIMPLE assignment. We validate the entire tree, not just
336	the root node, thus catching expressions that embed complex
337	operands that are not permitted in GIMPLE. This function
338	is needed because the folding routines in fold-const.cc
339	may return such expressions in some cases, e.g., an array
340	access with an embedded index addition. It may make more
341	sense to have folding routines that are sensitive to the
342	constraints on GIMPLE operands, rather than abandoning any
343	any attempt to fold if the usual folding turns out to be too
344	aggressive. */
345
346	bool
347	valid_gimple_rhs_p (tree expr)
348	{
349	enum tree_code code = TREE_CODE (expr);
350
351	switch (TREE_CODE_CLASS (code))
352	{
353	case tcc_declaration:
354	if (!is_gimple_variable (t: expr))
355	return false;
356	break;
357
358	case tcc_constant:
359	/* All constants are ok. */
360	break;
361
362	case tcc_comparison:
363	/* GENERIC allows comparisons with non-boolean types, reject
364	those for GIMPLE. Let vector-typed comparisons pass - rules
365	for GENERIC and GIMPLE are the same here. */
366	if (!(INTEGRAL_TYPE_P (TREE_TYPE (expr))
367	&& (TREE_CODE (TREE_TYPE (expr)) == BOOLEAN_TYPE
368	|| TYPE_PRECISION (TREE_TYPE (expr)) == 1))
369	&& ! VECTOR_TYPE_P (TREE_TYPE (expr)))
370	return false;
371
372	/* Fallthru. */
373	case tcc_binary:
374	if (!is_gimple_val (TREE_OPERAND (expr, 0))
375	|| !is_gimple_val (TREE_OPERAND (expr, 1)))
376	return false;
377	break;
378
379	case tcc_unary:
380	if (!is_gimple_val (TREE_OPERAND (expr, 0)))
381	return false;
382	break;
383
384	case tcc_expression:
385	switch (code)
386	{
387	case ADDR_EXPR:
388	{
389	tree t;
390	if (is_gimple_min_invariant (expr))
391	return true;
392	t = TREE_OPERAND (expr, 0);
393	while (handled_component_p (t))
394	{
395	/* ??? More checks needed, see the GIMPLE verifier. */
396	if ((TREE_CODE (t) == ARRAY_REF
397	|| TREE_CODE (t) == ARRAY_RANGE_REF)
398	&& !is_gimple_val (TREE_OPERAND (t, 1)))
399	return false;
400	t = TREE_OPERAND (t, 0);
401	}
402	if (!is_gimple_id (t))
403	return false;
404	}
405	break;
406
407	default:
408	if (get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS)
409	{
410	if (!is_gimple_val (TREE_OPERAND (expr, 0))
411	|| !is_gimple_val (TREE_OPERAND (expr, 1))
412	|| !is_gimple_val (TREE_OPERAND (expr, 2)))
413	return false;
414	break;
415	}
416	return false;
417	}
418	break;
419
420	case tcc_vl_exp:
421	return false;
422
423	case tcc_exceptional:
424	if (code == CONSTRUCTOR)
425	{
426	unsigned i;
427	tree elt;
428	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (expr), i, elt)
429	if (!is_gimple_val (elt))
430	return false;
431	return true;
432	}
433	if (code != SSA_NAME)
434	return false;
435	break;
436
437	case tcc_reference:
438	if (code == BIT_FIELD_REF)
439	return is_gimple_val (TREE_OPERAND (expr, 0));
440	return false;
441
442	default:
443	return false;
444	}
445
446	return true;
447	}
448
449
450	/* Attempt to fold an assignment statement pointed-to by SI. Returns a
451	replacement rhs for the statement or NULL_TREE if no simplification
452	could be made. It is assumed that the operands have been previously
453	folded. */
454
455	static tree
456	fold_gimple_assign (gimple_stmt_iterator *si)
457	{
458	gimple *stmt = gsi_stmt (i: *si);
459	enum tree_code subcode = gimple_assign_rhs_code (gs: stmt);
460	location_t loc = gimple_location (g: stmt);
461
462	tree result = NULL_TREE;
463
464	switch (get_gimple_rhs_class (code: subcode))
465	{
466	case GIMPLE_SINGLE_RHS:
467	{
468	tree rhs = gimple_assign_rhs1 (gs: stmt);
469
470	if (TREE_CLOBBER_P (rhs))
471	return NULL_TREE;
472
473	if (REFERENCE_CLASS_P (rhs))
474	return maybe_fold_reference (expr: rhs);
475
476	else if (TREE_CODE (rhs) == OBJ_TYPE_REF)
477	{
478	tree val = OBJ_TYPE_REF_EXPR (rhs);
479	if (is_gimple_min_invariant (val))
480	return val;
481	else if (flag_devirtualize && virtual_method_call_p (rhs))
482	{
483	bool final;
484	vec <cgraph_node *>targets
485	= possible_polymorphic_call_targets (ref: rhs, call: stmt, completep: &final);
486	if (final && targets.length () <= 1 && dbg_cnt (index: devirt))
487	{
488	if (dump_enabled_p ())
489	{
490	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
491	"resolving virtual function address "
492	"reference to function %s\n",
493	targets.length () == 1
494	? targets[0]->name ()
495	: "NULL");
496	}
497	if (targets.length () == 1)
498	{
499	val = fold_convert (TREE_TYPE (val),
500	build_fold_addr_expr_loc
501	(loc, targets[0]->decl));
502	STRIP_USELESS_TYPE_CONVERSION (val);
503	}
504	else
505	/* We cannot use __builtin_unreachable here because it
506	cannot have address taken. */
507	val = build_int_cst (TREE_TYPE (val), 0);
508	return val;
509	}
510	}
511	}
512
513	else if (TREE_CODE (rhs) == ADDR_EXPR)
514	{
515	tree ref = TREE_OPERAND (rhs, 0);
516	if (TREE_CODE (ref) == MEM_REF
517	&& integer_zerop (TREE_OPERAND (ref, 1)))
518	{
519	result = TREE_OPERAND (ref, 0);
520	if (!useless_type_conversion_p (TREE_TYPE (rhs),
521	TREE_TYPE (result)))
522	result = build1 (NOP_EXPR, TREE_TYPE (rhs), result);
523	return result;
524	}
525	}
526
527	else if (TREE_CODE (rhs) == CONSTRUCTOR
528	&& TREE_CODE (TREE_TYPE (rhs)) == VECTOR_TYPE)
529	{
530	/* Fold a constant vector CONSTRUCTOR to VECTOR_CST. */
531	unsigned i;
532	tree val;
533
534	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), i, val)
535	if (! CONSTANT_CLASS_P (val))
536	return NULL_TREE;
537
538	return build_vector_from_ctor (TREE_TYPE (rhs),
539	CONSTRUCTOR_ELTS (rhs));
540	}
541
542	else if (DECL_P (rhs)
543	&& is_gimple_reg_type (TREE_TYPE (rhs)))
544	return get_symbol_constant_value (sym: rhs);
545	}
546	break;
547
548	case GIMPLE_UNARY_RHS:
549	break;
550
551	case GIMPLE_BINARY_RHS:
552	break;
553
554	case GIMPLE_TERNARY_RHS:
555	result = fold_ternary_loc (loc, subcode,
556	TREE_TYPE (gimple_assign_lhs (stmt)),
557	gimple_assign_rhs1 (gs: stmt),
558	gimple_assign_rhs2 (gs: stmt),
559	gimple_assign_rhs3 (gs: stmt));
560
561	if (result)
562	{
563	STRIP_USELESS_TYPE_CONVERSION (result);
564	if (valid_gimple_rhs_p (expr: result))
565	return result;
566	}
567	break;
568
569	case GIMPLE_INVALID_RHS:
570	gcc_unreachable ();
571	}
572
573	return NULL_TREE;
574	}
575
576
577	/* Replace a statement at *SI_P with a sequence of statements in STMTS,
578	adjusting the replacement stmts location and virtual operands.
579	If the statement has a lhs the last stmt in the sequence is expected
580	to assign to that lhs. */
581
582	void
583	gsi_replace_with_seq_vops (gimple_stmt_iterator *si_p, gimple_seq stmts)
584	{
585	gimple *stmt = gsi_stmt (i: *si_p);
586
587	if (gimple_has_location (g: stmt))
588	annotate_all_with_location (stmts, gimple_location (g: stmt));
589
590	/* First iterate over the replacement statements backward, assigning
591	virtual operands to their defining statements. */
592	gimple *laststore = NULL;
593	for (gimple_stmt_iterator i = gsi_last (seq&: stmts);
594	!gsi_end_p (i); gsi_prev (i: &i))
595	{
596	gimple *new_stmt = gsi_stmt (i);
597	if ((gimple_assign_single_p (gs: new_stmt)
598	&& !is_gimple_reg (gimple_assign_lhs (gs: new_stmt)))
599	|| (is_gimple_call (gs: new_stmt)
600	&& (gimple_call_flags (new_stmt)
601	& (ECF_NOVOPS | ECF_PURE | ECF_CONST | ECF_NORETURN)) == 0))
602	{
603	tree vdef;
604	if (!laststore)
605	vdef = gimple_vdef (g: stmt);
606	else
607	vdef = make_ssa_name (var: gimple_vop (cfun), stmt: new_stmt);
608	gimple_set_vdef (g: new_stmt, vdef);
609	if (vdef && TREE_CODE (vdef) == SSA_NAME)
610	SSA_NAME_DEF_STMT (vdef) = new_stmt;
611	laststore = new_stmt;
612	}
613	}
614
615	/* Second iterate over the statements forward, assigning virtual
616	operands to their uses. */
617	tree reaching_vuse = gimple_vuse (g: stmt);
618	for (gimple_stmt_iterator i = gsi_start (seq&: stmts);
619	!gsi_end_p (i); gsi_next (i: &i))
620	{
621	gimple *new_stmt = gsi_stmt (i);
622	/* If the new statement possibly has a VUSE, update it with exact SSA
623	name we know will reach this one. */
624	if (gimple_has_mem_ops (g: new_stmt))
625	gimple_set_vuse (g: new_stmt, vuse: reaching_vuse);
626	gimple_set_modified (s: new_stmt, modifiedp: true);
627	if (gimple_vdef (g: new_stmt))
628	reaching_vuse = gimple_vdef (g: new_stmt);
629	}
630
631	/* If the new sequence does not do a store release the virtual
632	definition of the original statement. */
633	if (reaching_vuse
634	&& reaching_vuse == gimple_vuse (g: stmt))
635	{
636	tree vdef = gimple_vdef (g: stmt);
637	if (vdef
638	&& TREE_CODE (vdef) == SSA_NAME)
639	{
640	unlink_stmt_vdef (stmt);
641	release_ssa_name (name: vdef);
642	}
643	}
644
645	/* Finally replace the original statement with the sequence. */
646	gsi_replace_with_seq (si_p, stmts, false);
647	}
648
649	/* Helper function for update_gimple_call and
650	gimplify_and_update_call_from_tree. A GIMPLE_CALL STMT is being replaced
651	with GIMPLE_CALL NEW_STMT. */
652
653	static void
654	finish_update_gimple_call (gimple_stmt_iterator *si_p, gimple *new_stmt,
655	gimple *stmt)
656	{
657	tree lhs = gimple_call_lhs (gs: stmt);
658	gimple_call_set_lhs (gs: new_stmt, lhs);
659	if (lhs && TREE_CODE (lhs) == SSA_NAME)
660	SSA_NAME_DEF_STMT (lhs) = new_stmt;
661	gimple_move_vops (new_stmt, stmt);
662	gimple_set_location (g: new_stmt, location: gimple_location (g: stmt));
663	if (gimple_block (g: new_stmt) == NULL_TREE)
664	gimple_set_block (g: new_stmt, block: gimple_block (g: stmt));
665	gsi_replace (si_p, new_stmt, false);
666	}
667
668	/* Update a GIMPLE_CALL statement at iterator *SI_P to call to FN
669	with number of arguments NARGS, where the arguments in GIMPLE form
670	follow NARGS argument. */
671
672	bool
673	update_gimple_call (gimple_stmt_iterator *si_p, tree fn, int nargs, ...)
674	{
675	va_list ap;
676	gcall *new_stmt, *stmt = as_a <gcall *> (p: gsi_stmt (i: *si_p));
677
678	gcc_assert (is_gimple_call (stmt));
679	va_start (ap, nargs);
680	new_stmt = gimple_build_call_valist (fn, nargs, ap);
681	finish_update_gimple_call (si_p, new_stmt, stmt);
682	va_end (ap);
683	return true;
684	}
685
686	/* Return true if EXPR is a CALL_EXPR suitable for representation
687	as a single GIMPLE_CALL statement. If the arguments require
688	further gimplification, return false. */
689
690	static bool
691	valid_gimple_call_p (tree expr)
692	{
693	unsigned i, nargs;
694
695	if (TREE_CODE (expr) != CALL_EXPR)
696	return false;
697
698	nargs = call_expr_nargs (expr);
699	for (i = 0; i < nargs; i++)
700	{
701	tree arg = CALL_EXPR_ARG (expr, i);
702	if (is_gimple_reg_type (TREE_TYPE (arg)))
703	{
704	if (!is_gimple_val (arg))
705	return false;
706	}
707	else
708	if (!is_gimple_lvalue (arg))
709	return false;
710	}
711
712	return true;
713	}
714
715	/* Convert EXPR into a GIMPLE value suitable for substitution on the
716	RHS of an assignment. Insert the necessary statements before
717	iterator *SI_P. The statement at *SI_P, which must be a GIMPLE_CALL
718	is replaced. If the call is expected to produces a result, then it
719	is replaced by an assignment of the new RHS to the result variable.
720	If the result is to be ignored, then the call is replaced by a
721	GIMPLE_NOP. A proper VDEF chain is retained by making the first
722	VUSE and the last VDEF of the whole sequence be the same as the replaced
723	statement and using new SSA names for stores in between. */
724
725	void
726	gimplify_and_update_call_from_tree (gimple_stmt_iterator *si_p, tree expr)
727	{
728	tree lhs;
729	gimple *stmt, *new_stmt;
730	gimple_stmt_iterator i;
731	gimple_seq stmts = NULL;
732
733	stmt = gsi_stmt (i: *si_p);
734
735	gcc_assert (is_gimple_call (stmt));
736
737	if (valid_gimple_call_p (expr))
738	{
739	/* The call has simplified to another call. */
740	tree fn = CALL_EXPR_FN (expr);
741	unsigned i;
742	unsigned nargs = call_expr_nargs (expr);
743	vec<tree> args = vNULL;
744	gcall *new_stmt;
745
746	if (nargs > 0)
747	{
748	args.create (nelems: nargs);
749	args.safe_grow_cleared (len: nargs, exact: true);
750
751	for (i = 0; i < nargs; i++)
752	args[i] = CALL_EXPR_ARG (expr, i);
753	}
754
755	new_stmt = gimple_build_call_vec (fn, args);
756	finish_update_gimple_call (si_p, new_stmt, stmt);
757	args.release ();
758	return;
759	}
760
761	lhs = gimple_call_lhs (gs: stmt);
762	if (lhs == NULL_TREE)
763	{
764	push_gimplify_context (in_ssa: gimple_in_ssa_p (cfun));
765	gimplify_and_add (expr, &stmts);
766	pop_gimplify_context (NULL);
767
768	/* We can end up with folding a memcpy of an empty class assignment
769	which gets optimized away by C++ gimplification. */
770	if (gimple_seq_empty_p (s: stmts))
771	{
772	if (gimple_in_ssa_p (cfun))
773	{
774	unlink_stmt_vdef (stmt);
775	release_defs (stmt);
776	}
777	gsi_replace (si_p, gimple_build_nop (), false);
778	return;
779	}
780	}
781	else
782	{
783	tree tmp = force_gimple_operand (expr, &stmts, false, NULL_TREE);
784	new_stmt = gimple_build_assign (lhs, tmp);
785	i = gsi_last (seq&: stmts);
786	gsi_insert_after_without_update (&i, new_stmt,
787	GSI_CONTINUE_LINKING);
788	}
789
790	gsi_replace_with_seq_vops (si_p, stmts);
791	}
792
793	/* Print a message in the dump file recording transformation of FROM to TO. */
794
795	static void
796	dump_transformation (gcall *from, gcall *to)
797	{
798	if (dump_enabled_p ())
799	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, from, "simplified %T to %T\n",
800	gimple_call_fn (gs: from), gimple_call_fn (gs: to));
801	}
802
803	/* Replace the call at *GSI with the gimple value VAL. */
804
805	void
806	replace_call_with_value (gimple_stmt_iterator *gsi, tree val)
807	{
808	gimple *stmt = gsi_stmt (i: *gsi);
809	tree lhs = gimple_call_lhs (gs: stmt);
810	gimple *repl;
811	if (lhs)
812	{
813	if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (val)))
814	val = fold_convert (TREE_TYPE (lhs), val);
815	repl = gimple_build_assign (lhs, val);
816	}
817	else
818	repl = gimple_build_nop ();
819	tree vdef = gimple_vdef (g: stmt);
820	if (vdef && TREE_CODE (vdef) == SSA_NAME)
821	{
822	unlink_stmt_vdef (stmt);
823	release_ssa_name (name: vdef);
824	}
825	gsi_replace (gsi, repl, false);
826	}
827
828	/* Replace the call at *GSI with the new call REPL and fold that
829	again. */
830
831	static void
832	replace_call_with_call_and_fold (gimple_stmt_iterator *gsi, gimple *repl)
833	{
834	gimple *stmt = gsi_stmt (i: *gsi);
835	dump_transformation (from: as_a <gcall *> (p: stmt), to: as_a <gcall *> (p: repl));
836	gimple_call_set_lhs (gs: repl, lhs: gimple_call_lhs (gs: stmt));
837	gimple_set_location (g: repl, location: gimple_location (g: stmt));
838	gimple_move_vops (repl, stmt);
839	gsi_replace (gsi, repl, false);
840	fold_stmt (gsi);
841	}
842
843	/* Return true if VAR is a VAR_DECL or a component thereof. */
844
845	static bool
846	var_decl_component_p (tree var)
847	{
848	tree inner = var;
849	while (handled_component_p (t: inner))
850	inner = TREE_OPERAND (inner, 0);
851	return (DECL_P (inner)
852	|| (TREE_CODE (inner) == MEM_REF
853	&& TREE_CODE (TREE_OPERAND (inner, 0)) == ADDR_EXPR));
854	}
855
856	/* Return TRUE if the SIZE argument, representing the size of an
857	object, is in a range of values of which exactly zero is valid. */
858
859	static bool
860	size_must_be_zero_p (tree size)
861	{
862	if (integer_zerop (size))
863	return true;
864
865	if (TREE_CODE (size) != SSA_NAME || !INTEGRAL_TYPE_P (TREE_TYPE (size)))
866	return false;
867
868	tree type = TREE_TYPE (size);
869	int prec = TYPE_PRECISION (type);
870
871	/* Compute the value of SSIZE_MAX, the largest positive value that
872	can be stored in ssize_t, the signed counterpart of size_t. */
873	wide_int ssize_max = wi::lshift (x: wi::one (precision: prec), y: prec - 1) - 1;
874	wide_int zero = wi::zero (TYPE_PRECISION (type));
875	int_range_max valid_range (type, zero, ssize_max);
876	int_range_max vr;
877	get_range_query (cfun)->range_of_expr (r&: vr, expr: size);
878
879	if (vr.undefined_p ())
880	vr.set_varying (TREE_TYPE (size));
881	vr.intersect (valid_range);
882	return vr.zero_p ();
883	}
884
885	/* Fold function call to builtin mem{{,p}cpy,move}. Try to detect and
886	diagnose (otherwise undefined) overlapping copies without preventing
887	folding. When folded, GCC guarantees that overlapping memcpy has
888	the same semantics as memmove. Call to the library memcpy need not
889	provide the same guarantee. Return false if no simplification can
890	be made. */
891
892	static bool
893	gimple_fold_builtin_memory_op (gimple_stmt_iterator *gsi,
894	tree dest, tree src, enum built_in_function code)
895	{
896	gimple *stmt = gsi_stmt (i: *gsi);
897	tree lhs = gimple_call_lhs (gs: stmt);
898	tree len = gimple_call_arg (gs: stmt, index: 2);
899	location_t loc = gimple_location (g: stmt);
900
901	/* If the LEN parameter is a constant zero or in range where
902	the only valid value is zero, return DEST. */
903	if (size_must_be_zero_p (size: len))
904	{
905	gimple *repl;
906	if (gimple_call_lhs (gs: stmt))
907	repl = gimple_build_assign (gimple_call_lhs (gs: stmt), dest);
908	else
909	repl = gimple_build_nop ();
910	tree vdef = gimple_vdef (g: stmt);
911	if (vdef && TREE_CODE (vdef) == SSA_NAME)
912	{
913	unlink_stmt_vdef (stmt);
914	release_ssa_name (name: vdef);
915	}
916	gsi_replace (gsi, repl, false);
917	return true;
918	}
919
920	/* If SRC and DEST are the same (and not volatile), return
921	DEST{,+LEN,+LEN-1}. */
922	if (operand_equal_p (src, dest, flags: 0))
923	{
924	/* Avoid diagnosing exact overlap in calls to __builtin_memcpy.
925	It's safe and may even be emitted by GCC itself (see bug
926	32667). */
927	unlink_stmt_vdef (stmt);
928	if (gimple_vdef (g: stmt) && TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
929	release_ssa_name (name: gimple_vdef (g: stmt));
930	if (!lhs)
931	{
932	gsi_replace (gsi, gimple_build_nop (), false);
933	return true;
934	}
935	goto done;
936	}
937	else if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
938	return false;
939	else
940	{
941	/* We cannot (easily) change the type of the copy if it is a storage
942	order barrier, i.e. is equivalent to a VIEW_CONVERT_EXPR that can
943	modify the storage order of objects (see storage_order_barrier_p). */
944	tree srctype
945	= POINTER_TYPE_P (TREE_TYPE (src))
946	? TREE_TYPE (TREE_TYPE (src)) : NULL_TREE;
947	tree desttype
948	= POINTER_TYPE_P (TREE_TYPE (dest))
949	? TREE_TYPE (TREE_TYPE (dest)) : NULL_TREE;
950	tree destvar, srcvar, srcoff;
951	unsigned int src_align, dest_align;
952	unsigned HOST_WIDE_INT tmp_len;
953	const char *tmp_str;
954
955	/* Build accesses at offset zero with a ref-all character type. */
956	tree off0
957	= build_int_cst (build_pointer_type_for_mode (char_type_node,
958	ptr_mode, true), 0);
959
960	/* If we can perform the copy efficiently with first doing all loads
961	and then all stores inline it that way. Currently efficiently
962	means that we can load all the memory into a single integer
963	register which is what MOVE_MAX gives us. */
964	src_align = get_pointer_alignment (src);
965	dest_align = get_pointer_alignment (dest);
966	if (tree_fits_uhwi_p (len)
967	&& compare_tree_int (len, MOVE_MAX) <= 0
968	/* FIXME: Don't transform copies from strings with known length.
969	Until GCC 9 this prevented a case in gcc.dg/strlenopt-8.c
970	from being handled, and the case was XFAILed for that reason.
971	Now that it is handled and the XFAIL removed, as soon as other
972	strlenopt tests that rely on it for passing are adjusted, this
973	hack can be removed. */
974	&& !c_strlen (src, 1)
975	&& !((tmp_str = getbyterep (src, &tmp_len)) != NULL
976	&& memchr (s: tmp_str, c: 0, n: tmp_len) == NULL)
977	&& !(srctype
978	&& AGGREGATE_TYPE_P (srctype)
979	&& TYPE_REVERSE_STORAGE_ORDER (srctype))
980	&& !(desttype
981	&& AGGREGATE_TYPE_P (desttype)
982	&& TYPE_REVERSE_STORAGE_ORDER (desttype)))
983	{
984	unsigned ilen = tree_to_uhwi (len);
985	if (pow2p_hwi (x: ilen))
986	{
987	/* Detect out-of-bounds accesses without issuing warnings.
988	Avoid folding out-of-bounds copies but to avoid false
989	positives for unreachable code defer warning until after
990	DCE has worked its magic.
991	-Wrestrict is still diagnosed. */
992	if (int warning = check_bounds_or_overlap (as_a <gcall *>(p: stmt),
993	dest, src, len, len,
994	false, false))
995	if (warning != OPT_Wrestrict)
996	return false;
997
998	scalar_int_mode imode;
999	machine_mode mode;
1000	if (int_mode_for_size (size: ilen * BITS_PER_UNIT, limit: 0).exists (mode: &imode)
1001	&& bitwise_mode_for_size (ilen
1002	* BITS_PER_UNIT).exists (mode: &mode)
1003	&& known_eq (GET_MODE_BITSIZE (mode), ilen * BITS_PER_UNIT)
1004	/* If the destination pointer is not aligned we must be able
1005	to emit an unaligned store. */
1006	&& (dest_align >= GET_MODE_ALIGNMENT (mode)
1007	|| !targetm.slow_unaligned_access (mode, dest_align)
1008	|| (optab_handler (op: movmisalign_optab, mode)
1009	!= CODE_FOR_nothing)))
1010	{
1011	tree type = bitwise_type_for_mode (mode);
1012	tree srctype = type;
1013	tree desttype = type;
1014	if (src_align < GET_MODE_ALIGNMENT (mode))
1015	srctype = build_aligned_type (type, src_align);
1016	tree srcmem = fold_build2 (MEM_REF, srctype, src, off0);
1017	tree tem = fold_const_aggregate_ref (srcmem);
1018	if (tem)
1019	srcmem = tem;
1020	else if (src_align < GET_MODE_ALIGNMENT (mode)
1021	&& targetm.slow_unaligned_access (mode, src_align)
1022	&& (optab_handler (op: movmisalign_optab, mode)
1023	== CODE_FOR_nothing))
1024	srcmem = NULL_TREE;
1025	if (srcmem)
1026	{
1027	gimple *new_stmt;
1028	if (is_gimple_reg_type (TREE_TYPE (srcmem)))
1029	{
1030	new_stmt = gimple_build_assign (NULL_TREE, srcmem);
1031	srcmem
1032	= make_ssa_name (TREE_TYPE (srcmem), stmt: new_stmt);
1033	gimple_assign_set_lhs (gs: new_stmt, lhs: srcmem);
1034	gimple_set_vuse (g: new_stmt, vuse: gimple_vuse (g: stmt));
1035	gimple_set_location (g: new_stmt, location: loc);
1036	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
1037	}
1038	if (dest_align < GET_MODE_ALIGNMENT (mode))
1039	desttype = build_aligned_type (type, dest_align);
1040	new_stmt
1041	= gimple_build_assign (fold_build2 (MEM_REF, desttype,
1042	dest, off0),
1043	srcmem);
1044	gimple_move_vops (new_stmt, stmt);
1045	if (!lhs)
1046	{
1047	gsi_replace (gsi, new_stmt, false);
1048	return true;
1049	}
1050	gimple_set_location (g: new_stmt, location: loc);
1051	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
1052	goto done;
1053	}
1054	}
1055	}
1056	}
1057
1058	if (code == BUILT_IN_MEMMOVE)
1059	{
1060	/* Both DEST and SRC must be pointer types.
1061	??? This is what old code did. Is the testing for pointer types
1062	really mandatory?
1063
1064	If either SRC is readonly or length is 1, we can use memcpy. */
1065	if (!dest_align || !src_align)
1066	return false;
1067	if (readonly_data_expr (exp: src)
1068	|| (tree_fits_uhwi_p (len)
1069	&& (MIN (src_align, dest_align) / BITS_PER_UNIT
1070	>= tree_to_uhwi (len))))
1071	{
1072	tree fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
1073	if (!fn)
1074	return false;
1075	gimple_call_set_fndecl (gs: stmt, decl: fn);
1076	gimple_call_set_arg (gs: stmt, index: 0, arg: dest);
1077	gimple_call_set_arg (gs: stmt, index: 1, arg: src);
1078	fold_stmt (gsi);
1079	return true;
1080	}
1081
1082	/* If *src and *dest can't overlap, optimize into memcpy as well. */
1083	if (TREE_CODE (src) == ADDR_EXPR
1084	&& TREE_CODE (dest) == ADDR_EXPR)
1085	{
1086	tree src_base, dest_base, fn;
1087	poly_int64 src_offset = 0, dest_offset = 0;
1088	poly_uint64 maxsize;
1089
1090	srcvar = TREE_OPERAND (src, 0);
1091	src_base = get_addr_base_and_unit_offset (srcvar, &src_offset);
1092	if (src_base == NULL)
1093	src_base = srcvar;
1094	destvar = TREE_OPERAND (dest, 0);
1095	dest_base = get_addr_base_and_unit_offset (destvar,
1096	&dest_offset);
1097	if (dest_base == NULL)
1098	dest_base = destvar;
1099	if (!poly_int_tree_p (t: len, value: &maxsize))
1100	maxsize = -1;
1101	if (SSA_VAR_P (src_base)
1102	&& SSA_VAR_P (dest_base))
1103	{
1104	if (operand_equal_p (src_base, dest_base, flags: 0)
1105	&& ranges_maybe_overlap_p (pos1: src_offset, size1: maxsize,
1106	pos2: dest_offset, size2: maxsize))
1107	return false;
1108	}
1109	else if (TREE_CODE (src_base) == MEM_REF
1110	&& TREE_CODE (dest_base) == MEM_REF)
1111	{
1112	if (! operand_equal_p (TREE_OPERAND (src_base, 0),
1113	TREE_OPERAND (dest_base, 0), flags: 0))
1114	return false;
1115	poly_offset_int full_src_offset
1116	= mem_ref_offset (src_base) + src_offset;
1117	poly_offset_int full_dest_offset
1118	= mem_ref_offset (dest_base) + dest_offset;
1119	if (ranges_maybe_overlap_p (pos1: full_src_offset, size1: maxsize,
1120	pos2: full_dest_offset, size2: maxsize))
1121	return false;
1122	}
1123	else
1124	return false;
1125
1126	fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
1127	if (!fn)
1128	return false;
1129	gimple_call_set_fndecl (gs: stmt, decl: fn);
1130	gimple_call_set_arg (gs: stmt, index: 0, arg: dest);
1131	gimple_call_set_arg (gs: stmt, index: 1, arg: src);
1132	fold_stmt (gsi);
1133	return true;
1134	}
1135
1136	/* If the destination and source do not alias optimize into
1137	memcpy as well. */
1138	if ((is_gimple_min_invariant (dest)
1139	|| TREE_CODE (dest) == SSA_NAME)
1140	&& (is_gimple_min_invariant (src)
1141	|| TREE_CODE (src) == SSA_NAME))
1142	{
1143	ao_ref destr, srcr;
1144	ao_ref_init_from_ptr_and_size (&destr, dest, len);
1145	ao_ref_init_from_ptr_and_size (&srcr, src, len);
1146	if (!refs_may_alias_p_1 (&destr, &srcr, false))
1147	{
1148	tree fn;
1149	fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
1150	if (!fn)
1151	return false;
1152	gimple_call_set_fndecl (gs: stmt, decl: fn);
1153	gimple_call_set_arg (gs: stmt, index: 0, arg: dest);
1154	gimple_call_set_arg (gs: stmt, index: 1, arg: src);
1155	fold_stmt (gsi);
1156	return true;
1157	}
1158	}
1159
1160	return false;
1161	}
1162
1163	if (!tree_fits_shwi_p (len))
1164	return false;
1165	if (!srctype
1166	|| (AGGREGATE_TYPE_P (srctype)
1167	&& TYPE_REVERSE_STORAGE_ORDER (srctype)))
1168	return false;
1169	if (!desttype
1170	|| (AGGREGATE_TYPE_P (desttype)
1171	&& TYPE_REVERSE_STORAGE_ORDER (desttype)))
1172	return false;
1173	/* In the following try to find a type that is most natural to be
1174	used for the memcpy source and destination and that allows
1175	the most optimization when memcpy is turned into a plain assignment
1176	using that type. In theory we could always use a char[len] type
1177	but that only gains us that the destination and source possibly
1178	no longer will have their address taken. */
1179	if (TREE_CODE (srctype) == ARRAY_TYPE
1180	&& !tree_int_cst_equal (TYPE_SIZE_UNIT (srctype), len))
1181	srctype = TREE_TYPE (srctype);
1182	if (TREE_CODE (desttype) == ARRAY_TYPE
1183	&& !tree_int_cst_equal (TYPE_SIZE_UNIT (desttype), len))
1184	desttype = TREE_TYPE (desttype);
1185	if (TREE_ADDRESSABLE (srctype)
1186	|| TREE_ADDRESSABLE (desttype))
1187	return false;
1188
1189	/* Make sure we are not copying using a floating-point mode or
1190	a type whose size possibly does not match its precision. */
1191	if (FLOAT_MODE_P (TYPE_MODE (desttype))
1192	|| TREE_CODE (desttype) == BOOLEAN_TYPE
1193	|| TREE_CODE (desttype) == ENUMERAL_TYPE)
1194	desttype = bitwise_type_for_mode (TYPE_MODE (desttype));
1195	if (FLOAT_MODE_P (TYPE_MODE (srctype))
1196	|| TREE_CODE (srctype) == BOOLEAN_TYPE
1197	|| TREE_CODE (srctype) == ENUMERAL_TYPE)
1198	srctype = bitwise_type_for_mode (TYPE_MODE (srctype));
1199	if (!srctype)
1200	srctype = desttype;
1201	if (!desttype)
1202	desttype = srctype;
1203	if (!srctype)
1204	return false;
1205
1206	src_align = get_pointer_alignment (src);
1207	dest_align = get_pointer_alignment (dest);
1208
1209	/* Choose between src and destination type for the access based
1210	on alignment, whether the access constitutes a register access
1211	and whether it may actually expose a declaration for SSA rewrite
1212	or SRA decomposition. Also try to expose a string constant, we
1213	might be able to concatenate several of them later into a single
1214	string store. */
1215	destvar = NULL_TREE;
1216	srcvar = NULL_TREE;
1217	if (TREE_CODE (dest) == ADDR_EXPR
1218	&& var_decl_component_p (TREE_OPERAND (dest, 0))
1219	&& tree_int_cst_equal (TYPE_SIZE_UNIT (desttype), len)
1220	&& dest_align >= TYPE_ALIGN (desttype)
1221	&& (is_gimple_reg_type (type: desttype)
1222	|| src_align >= TYPE_ALIGN (desttype)))
1223	destvar = fold_build2 (MEM_REF, desttype, dest, off0);
1224	else if (TREE_CODE (src) == ADDR_EXPR
1225	&& var_decl_component_p (TREE_OPERAND (src, 0))
1226	&& tree_int_cst_equal (TYPE_SIZE_UNIT (srctype), len)
1227	&& src_align >= TYPE_ALIGN (srctype)
1228	&& (is_gimple_reg_type (type: srctype)
1229	|| dest_align >= TYPE_ALIGN (srctype)))
1230	srcvar = fold_build2 (MEM_REF, srctype, src, off0);
1231	/* FIXME: Don't transform copies from strings with known original length.
1232	As soon as strlenopt tests that rely on it for passing are adjusted,
1233	this hack can be removed. */
1234	else if (gimple_call_alloca_for_var_p (s: stmt)
1235	&& (srcvar = string_constant (src, &srcoff, NULL, NULL))
1236	&& integer_zerop (srcoff)
1237	&& tree_int_cst_equal (TYPE_SIZE_UNIT (TREE_TYPE (srcvar)), len)
1238	&& dest_align >= TYPE_ALIGN (TREE_TYPE (srcvar)))
1239	srctype = TREE_TYPE (srcvar);
1240	else
1241	return false;
1242
1243	/* Now that we chose an access type express the other side in
1244	terms of it if the target allows that with respect to alignment
1245	constraints. */
1246	if (srcvar == NULL_TREE)
1247	{
1248	if (src_align >= TYPE_ALIGN (desttype))
1249	srcvar = fold_build2 (MEM_REF, desttype, src, off0);
1250	else
1251	{
1252	enum machine_mode mode = TYPE_MODE (desttype);
1253	if ((mode == BLKmode && STRICT_ALIGNMENT)
1254	|| (targetm.slow_unaligned_access (mode, src_align)
1255	&& (optab_handler (op: movmisalign_optab, mode)
1256	== CODE_FOR_nothing)))
1257	return false;
1258	srctype = build_aligned_type (TYPE_MAIN_VARIANT (desttype),
1259	src_align);
1260	srcvar = fold_build2 (MEM_REF, srctype, src, off0);
1261	}
1262	}
1263	else if (destvar == NULL_TREE)
1264	{
1265	if (dest_align >= TYPE_ALIGN (srctype))
1266	destvar = fold_build2 (MEM_REF, srctype, dest, off0);
1267	else
1268	{
1269	enum machine_mode mode = TYPE_MODE (srctype);
1270	if ((mode == BLKmode && STRICT_ALIGNMENT)
1271	|| (targetm.slow_unaligned_access (mode, dest_align)
1272	&& (optab_handler (op: movmisalign_optab, mode)
1273	== CODE_FOR_nothing)))
1274	return false;
1275	desttype = build_aligned_type (TYPE_MAIN_VARIANT (srctype),
1276	dest_align);
1277	destvar = fold_build2 (MEM_REF, desttype, dest, off0);
1278	}
1279	}
1280
1281	/* Same as above, detect out-of-bounds accesses without issuing
1282	warnings. Avoid folding out-of-bounds copies but to avoid
1283	false positives for unreachable code defer warning until
1284	after DCE has worked its magic.
1285	-Wrestrict is still diagnosed. */
1286	if (int warning = check_bounds_or_overlap (as_a <gcall *>(p: stmt),
1287	dest, src, len, len,
1288	false, false))
1289	if (warning != OPT_Wrestrict)
1290	return false;
1291
1292	gimple *new_stmt;
1293	if (is_gimple_reg_type (TREE_TYPE (srcvar)))
1294	{
1295	tree tem = fold_const_aggregate_ref (srcvar);
1296	if (tem)
1297	srcvar = tem;
1298	if (! is_gimple_min_invariant (srcvar))
1299	{
1300	new_stmt = gimple_build_assign (NULL_TREE, srcvar);
1301	srcvar = make_ssa_name (TREE_TYPE (srcvar), stmt: new_stmt);
1302	gimple_assign_set_lhs (gs: new_stmt, lhs: srcvar);
1303	gimple_set_vuse (g: new_stmt, vuse: gimple_vuse (g: stmt));
1304	gimple_set_location (g: new_stmt, location: loc);
1305	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
1306	}
1307	new_stmt = gimple_build_assign (destvar, srcvar);
1308	goto set_vop_and_replace;
1309	}
1310
1311	/* We get an aggregate copy. If the source is a STRING_CST, then
1312	directly use its type to perform the copy. */
1313	if (TREE_CODE (srcvar) == STRING_CST)
1314	desttype = srctype;
1315
1316	/* Or else, use an unsigned char[] type to perform the copy in order
1317	to preserve padding and to avoid any issues with TREE_ADDRESSABLE
1318	types or float modes behavior on copying. */
1319	else
1320	{
1321	desttype = build_array_type_nelts (unsigned_char_type_node,
1322	tree_to_uhwi (len));
1323	srctype = desttype;
1324	if (src_align > TYPE_ALIGN (srctype))
1325	srctype = build_aligned_type (srctype, src_align);
1326	srcvar = fold_build2 (MEM_REF, srctype, src, off0);
1327	}
1328
1329	if (dest_align > TYPE_ALIGN (desttype))
1330	desttype = build_aligned_type (desttype, dest_align);
1331	destvar = fold_build2 (MEM_REF, desttype, dest, off0);
1332	new_stmt = gimple_build_assign (destvar, srcvar);
1333
1334	set_vop_and_replace:
1335	gimple_move_vops (new_stmt, stmt);
1336	if (!lhs)
1337	{
1338	gsi_replace (gsi, new_stmt, false);
1339	return true;
1340	}
1341	gimple_set_location (g: new_stmt, location: loc);
1342	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
1343	}
1344
1345	done:
1346	gimple_seq stmts = NULL;
1347	if (code == BUILT_IN_MEMCPY || code == BUILT_IN_MEMMOVE)
1348	len = NULL_TREE;
1349	else if (code == BUILT_IN_MEMPCPY)
1350	{
1351	len = gimple_convert_to_ptrofftype (seq: &stmts, loc, op: len);
1352	dest = gimple_build (seq: &stmts, loc, code: POINTER_PLUS_EXPR,
1353	TREE_TYPE (dest), ops: dest, ops: len);
1354	}
1355	else
1356	gcc_unreachable ();
1357
1358	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1359	gimple *repl = gimple_build_assign (lhs, dest);
1360	gsi_replace (gsi, repl, false);
1361	return true;
1362	}
1363
1364	/* Transform a call to built-in bcmp(a, b, len) at *GSI into one
1365	to built-in memcmp (a, b, len). */
1366
1367	static bool
1368	gimple_fold_builtin_bcmp (gimple_stmt_iterator *gsi)
1369	{
1370	tree fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCMP);
1371
1372	if (!fn)
1373	return false;
1374
1375	/* Transform bcmp (a, b, len) into memcmp (a, b, len). */
1376
1377	gimple *stmt = gsi_stmt (i: *gsi);
1378	if (!gimple_vuse (g: stmt) && gimple_in_ssa_p (cfun))
1379	return false;
1380	tree a = gimple_call_arg (gs: stmt, index: 0);
1381	tree b = gimple_call_arg (gs: stmt, index: 1);
1382	tree len = gimple_call_arg (gs: stmt, index: 2);
1383
1384	gimple *repl = gimple_build_call (fn, 3, a, b, len);
1385	replace_call_with_call_and_fold (gsi, repl);
1386
1387	return true;
1388	}
1389
1390	/* Transform a call to built-in bcopy (src, dest, len) at *GSI into one
1391	to built-in memmove (dest, src, len). */
1392
1393	static bool
1394	gimple_fold_builtin_bcopy (gimple_stmt_iterator *gsi)
1395	{
1396	tree fn = builtin_decl_implicit (fncode: BUILT_IN_MEMMOVE);
1397
1398	if (!fn)
1399	return false;
1400
1401	/* bcopy has been removed from POSIX in Issue 7 but Issue 6 specifies
1402	it's quivalent to memmove (not memcpy). Transform bcopy (src, dest,
1403	len) into memmove (dest, src, len). */
1404
1405	gimple *stmt = gsi_stmt (i: *gsi);
1406	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
1407	return false;
1408	tree src = gimple_call_arg (gs: stmt, index: 0);
1409	tree dest = gimple_call_arg (gs: stmt, index: 1);
1410	tree len = gimple_call_arg (gs: stmt, index: 2);
1411
1412	gimple *repl = gimple_build_call (fn, 3, dest, src, len);
1413	gimple_call_set_fntype (call_stmt: as_a <gcall *> (p: stmt), TREE_TYPE (fn));
1414	replace_call_with_call_and_fold (gsi, repl);
1415
1416	return true;
1417	}
1418
1419	/* Transform a call to built-in bzero (dest, len) at *GSI into one
1420	to built-in memset (dest, 0, len). */
1421
1422	static bool
1423	gimple_fold_builtin_bzero (gimple_stmt_iterator *gsi)
1424	{
1425	tree fn = builtin_decl_implicit (fncode: BUILT_IN_MEMSET);
1426
1427	if (!fn)
1428	return false;
1429
1430	/* Transform bzero (dest, len) into memset (dest, 0, len). */
1431
1432	gimple *stmt = gsi_stmt (i: *gsi);
1433	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
1434	return false;
1435	tree dest = gimple_call_arg (gs: stmt, index: 0);
1436	tree len = gimple_call_arg (gs: stmt, index: 1);
1437
1438	gimple_seq seq = NULL;
1439	gimple *repl = gimple_build_call (fn, 3, dest, integer_zero_node, len);
1440	gimple_seq_add_stmt_without_update (&seq, repl);
1441	gsi_replace_with_seq_vops (si_p: gsi, stmts: seq);
1442	fold_stmt (gsi);
1443
1444	return true;
1445	}
1446
1447	/* Fold function call to builtin memset or bzero at *GSI setting the
1448	memory of size LEN to VAL. Return whether a simplification was made. */
1449
1450	static bool
1451	gimple_fold_builtin_memset (gimple_stmt_iterator *gsi, tree c, tree len)
1452	{
1453	gimple *stmt = gsi_stmt (i: *gsi);
1454	tree etype;
1455	unsigned HOST_WIDE_INT length, cval;
1456
1457	/* If the LEN parameter is zero, return DEST. */
1458	if (integer_zerop (len))
1459	{
1460	replace_call_with_value (gsi, val: gimple_call_arg (gs: stmt, index: 0));
1461	return true;
1462	}
1463
1464	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
1465	return false;
1466
1467	if (! tree_fits_uhwi_p (len))
1468	return false;
1469
1470	if (TREE_CODE (c) != INTEGER_CST)
1471	return false;
1472
1473	tree dest = gimple_call_arg (gs: stmt, index: 0);
1474	tree var = dest;
1475	if (TREE_CODE (var) != ADDR_EXPR)
1476	return false;
1477
1478	var = TREE_OPERAND (var, 0);
1479	if (TREE_THIS_VOLATILE (var))
1480	return false;
1481
1482	etype = TREE_TYPE (var);
1483	if (TREE_CODE (etype) == ARRAY_TYPE)
1484	etype = TREE_TYPE (etype);
1485
1486	if ((!INTEGRAL_TYPE_P (etype)
1487	&& !POINTER_TYPE_P (etype))
1488	|| TREE_CODE (etype) == BITINT_TYPE)
1489	return false;
1490
1491	if (! var_decl_component_p (var))
1492	return false;
1493
1494	length = tree_to_uhwi (len);
1495	if (GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (etype)) != length
1496	|| (GET_MODE_PRECISION (SCALAR_INT_TYPE_MODE (etype))
1497	!= GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (etype)))
1498	|| get_pointer_alignment (dest) / BITS_PER_UNIT < length)
1499	return false;
1500
1501	if (length > HOST_BITS_PER_WIDE_INT / BITS_PER_UNIT)
1502	return false;
1503
1504	if (!type_has_mode_precision_p (t: etype))
1505	etype = lang_hooks.types.type_for_mode (SCALAR_INT_TYPE_MODE (etype),
1506	TYPE_UNSIGNED (etype));
1507
1508	if (integer_zerop (c))
1509	cval = 0;
1510	else
1511	{
1512	if (CHAR_BIT != 8 || BITS_PER_UNIT != 8 || HOST_BITS_PER_WIDE_INT > 64)
1513	return NULL_TREE;
1514
1515	cval = TREE_INT_CST_LOW (c);
1516	cval &= 0xff;
1517	cval |= cval << 8;
1518	cval |= cval << 16;
1519	cval |= (cval << 31) << 1;
1520	}
1521
1522	var = fold_build2 (MEM_REF, etype, dest, build_int_cst (ptr_type_node, 0));
1523	gimple *store = gimple_build_assign (var, build_int_cst_type (etype, cval));
1524	gimple_move_vops (store, stmt);
1525	gimple_set_location (g: store, location: gimple_location (g: stmt));
1526	gsi_insert_before (gsi, store, GSI_SAME_STMT);
1527	if (gimple_call_lhs (gs: stmt))
1528	{
1529	gimple *asgn = gimple_build_assign (gimple_call_lhs (gs: stmt), dest);
1530	gsi_replace (gsi, asgn, false);
1531	}
1532	else
1533	{
1534	gimple_stmt_iterator gsi2 = *gsi;
1535	gsi_prev (i: gsi);
1536	gsi_remove (&gsi2, true);
1537	}
1538
1539	return true;
1540	}
1541
1542	/* Helper of get_range_strlen for ARG that is not an SSA_NAME. */
1543
1544	static bool
1545	get_range_strlen_tree (tree arg, bitmap visited, strlen_range_kind rkind,
1546	c_strlen_data *pdata, unsigned eltsize)
1547	{
1548	gcc_assert (TREE_CODE (arg) != SSA_NAME);
1549
1550	/* The length computed by this invocation of the function. */
1551	tree val = NULL_TREE;
1552
1553	/* True if VAL is an optimistic (tight) bound determined from
1554	the size of the character array in which the string may be
1555	stored. In that case, the computed VAL is used to set
1556	PDATA->MAXBOUND. */
1557	bool tight_bound = false;
1558
1559	/* We can end up with &(*iftmp_1)[0] here as well, so handle it. */
1560	if (TREE_CODE (arg) == ADDR_EXPR
1561	&& TREE_CODE (TREE_OPERAND (arg, 0)) == ARRAY_REF)
1562	{
1563	tree op = TREE_OPERAND (arg, 0);
1564	if (integer_zerop (TREE_OPERAND (op, 1)))
1565	{
1566	tree aop0 = TREE_OPERAND (op, 0);
1567	if (TREE_CODE (aop0) == INDIRECT_REF
1568	&& TREE_CODE (TREE_OPERAND (aop0, 0)) == SSA_NAME)
1569	return get_range_strlen (TREE_OPERAND (aop0, 0), visited, rkind,
1570	pdata, eltsize);
1571	}
1572	else if (TREE_CODE (TREE_OPERAND (op, 0)) == COMPONENT_REF
1573	&& rkind == SRK_LENRANGE)
1574	{
1575	/* Fail if an array is the last member of a struct object
1576	since it could be treated as a (fake) flexible array
1577	member. */
1578	tree idx = TREE_OPERAND (op, 1);
1579
1580	arg = TREE_OPERAND (op, 0);
1581	tree optype = TREE_TYPE (arg);
1582	if (tree dom = TYPE_DOMAIN (optype))
1583	if (tree bound = TYPE_MAX_VALUE (dom))
1584	if (TREE_CODE (bound) == INTEGER_CST
1585	&& TREE_CODE (idx) == INTEGER_CST
1586	&& tree_int_cst_lt (t1: bound, t2: idx))
1587	return false;
1588	}
1589	}
1590
1591	if (rkind == SRK_INT_VALUE)
1592	{
1593	/* We are computing the maximum value (not string length). */
1594	val = arg;
1595	if (TREE_CODE (val) != INTEGER_CST
1596	|| tree_int_cst_sgn (val) < 0)
1597	return false;
1598	}
1599	else
1600	{
1601	c_strlen_data lendata = { };
1602	val = c_strlen (arg, 1, &lendata, eltsize);
1603
1604	if (!val && lendata.decl)
1605	{
1606	/* ARG refers to an unterminated const character array.
1607	DATA.DECL with size DATA.LEN. */
1608	val = lendata.minlen;
1609	pdata->decl = lendata.decl;
1610	}
1611	}
1612
1613	/* Set if VAL represents the maximum length based on array size (set
1614	when exact length cannot be determined). */
1615	bool maxbound = false;
1616
1617	if (!val && rkind == SRK_LENRANGE)
1618	{
1619	if (TREE_CODE (arg) == ADDR_EXPR)
1620	return get_range_strlen (TREE_OPERAND (arg, 0), visited, rkind,
1621	pdata, eltsize);
1622
1623	if (TREE_CODE (arg) == ARRAY_REF)
1624	{
1625	tree optype = TREE_TYPE (TREE_OPERAND (arg, 0));
1626
1627	/* Determine the "innermost" array type. */
1628	while (TREE_CODE (optype) == ARRAY_TYPE
1629	&& TREE_CODE (TREE_TYPE (optype)) == ARRAY_TYPE)
1630	optype = TREE_TYPE (optype);
1631
1632	/* Avoid arrays of pointers. */
1633	tree eltype = TREE_TYPE (optype);
1634	if (TREE_CODE (optype) != ARRAY_TYPE
1635	|| !INTEGRAL_TYPE_P (eltype))
1636	return false;
1637
1638	/* Fail when the array bound is unknown or zero. */
1639	val = TYPE_SIZE_UNIT (optype);
1640	if (!val
1641	|| TREE_CODE (val) != INTEGER_CST
1642	|| integer_zerop (val))
1643	return false;
1644
1645	val = fold_build2 (MINUS_EXPR, TREE_TYPE (val), val,
1646	integer_one_node);
1647
1648	/* Set the minimum size to zero since the string in
1649	the array could have zero length. */
1650	pdata->minlen = ssize_int (0);
1651
1652	tight_bound = true;
1653	}
1654	else if (TREE_CODE (arg) == COMPONENT_REF
1655	&& (TREE_CODE (TREE_TYPE (TREE_OPERAND (arg, 1)))
1656	== ARRAY_TYPE))
1657	{
1658	/* Use the type of the member array to determine the upper
1659	bound on the length of the array. This may be overly
1660	optimistic if the array itself isn't NUL-terminated and
1661	the caller relies on the subsequent member to contain
1662	the NUL but that would only be considered valid if
1663	the array were the last member of a struct. */
1664
1665	tree fld = TREE_OPERAND (arg, 1);
1666
1667	tree optype = TREE_TYPE (fld);
1668
1669	/* Determine the "innermost" array type. */
1670	while (TREE_CODE (optype) == ARRAY_TYPE
1671	&& TREE_CODE (TREE_TYPE (optype)) == ARRAY_TYPE)
1672	optype = TREE_TYPE (optype);
1673
1674	/* Fail when the array bound is unknown or zero. */
1675	val = TYPE_SIZE_UNIT (optype);
1676	if (!val
1677	|| TREE_CODE (val) != INTEGER_CST
1678	|| integer_zerop (val))
1679	return false;
1680	val = fold_build2 (MINUS_EXPR, TREE_TYPE (val), val,
1681	integer_one_node);
1682
1683	/* Set the minimum size to zero since the string in
1684	the array could have zero length. */
1685	pdata->minlen = ssize_int (0);
1686
1687	/* The array size determined above is an optimistic bound
1688	on the length. If the array isn't nul-terminated the
1689	length computed by the library function would be greater.
1690	Even though using strlen to cross the subobject boundary
1691	is undefined, avoid drawing conclusions from the member
1692	type about the length here. */
1693	tight_bound = true;
1694	}
1695	else if (TREE_CODE (arg) == MEM_REF
1696	&& TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE
1697	&& TREE_CODE (TREE_TYPE (TREE_TYPE (arg))) == INTEGER_TYPE
1698	&& TREE_CODE (TREE_OPERAND (arg, 0)) == ADDR_EXPR)
1699	{
1700	/* Handle a MEM_REF into a DECL accessing an array of integers,
1701	being conservative about references to extern structures with
1702	flexible array members that can be initialized to arbitrary
1703	numbers of elements as an extension (static structs are okay). */
1704	tree ref = TREE_OPERAND (TREE_OPERAND (arg, 0), 0);
1705	if ((TREE_CODE (ref) == PARM_DECL || VAR_P (ref))
1706	&& (decl_binds_to_current_def_p (ref)
1707	|| !array_ref_flexible_size_p (arg)))
1708	{
1709	/* Fail if the offset is out of bounds. Such accesses
1710	should be diagnosed at some point. */
1711	val = DECL_SIZE_UNIT (ref);
1712	if (!val
1713	|| TREE_CODE (val) != INTEGER_CST
1714	|| integer_zerop (val))
1715	return false;
1716
1717	poly_offset_int psiz = wi::to_offset (t: val);
1718	poly_offset_int poff = mem_ref_offset (arg);
1719	if (known_le (psiz, poff))
1720	return false;
1721
1722	pdata->minlen = ssize_int (0);
1723
1724	/* Subtract the offset and one for the terminating nul. */
1725	psiz -= poff;
1726	psiz -= 1;
1727	val = wide_int_to_tree (TREE_TYPE (val), cst: psiz);
1728	/* Since VAL reflects the size of a declared object
1729	rather the type of the access it is not a tight bound. */
1730	}
1731	}
1732	else if (TREE_CODE (arg) == PARM_DECL || VAR_P (arg))
1733	{
1734	/* Avoid handling pointers to arrays. GCC might misuse
1735	a pointer to an array of one bound to point to an array
1736	object of a greater bound. */
1737	tree argtype = TREE_TYPE (arg);
1738	if (TREE_CODE (argtype) == ARRAY_TYPE)
1739	{
1740	val = TYPE_SIZE_UNIT (argtype);
1741	if (!val
1742	|| TREE_CODE (val) != INTEGER_CST
1743	|| integer_zerop (val))
1744	return false;
1745	val = wide_int_to_tree (TREE_TYPE (val),
1746	cst: wi::sub (x: wi::to_wide (t: val), y: 1));
1747
1748	/* Set the minimum size to zero since the string in
1749	the array could have zero length. */
1750	pdata->minlen = ssize_int (0);
1751	}
1752	}
1753	maxbound = true;
1754	}
1755
1756	if (!val)
1757	return false;
1758
1759	/* Adjust the lower bound on the string length as necessary. */
1760	if (!pdata->minlen
1761	|| (rkind != SRK_STRLEN
1762	&& TREE_CODE (pdata->minlen) == INTEGER_CST
1763	&& TREE_CODE (val) == INTEGER_CST
1764	&& tree_int_cst_lt (t1: val, t2: pdata->minlen)))
1765	pdata->minlen = val;
1766
1767	if (pdata->maxbound && TREE_CODE (pdata->maxbound) == INTEGER_CST)
1768	{
1769	/* Adjust the tighter (more optimistic) string length bound
1770	if necessary and proceed to adjust the more conservative
1771	bound. */
1772	if (TREE_CODE (val) == INTEGER_CST)
1773	{
1774	if (tree_int_cst_lt (t1: pdata->maxbound, t2: val))
1775	pdata->maxbound = val;
1776	}
1777	else
1778	pdata->maxbound = val;
1779	}
1780	else if (pdata->maxbound || maxbound)
1781	/* Set PDATA->MAXBOUND only if it either isn't INTEGER_CST or
1782	if VAL corresponds to the maximum length determined based
1783	on the type of the object. */
1784	pdata->maxbound = val;
1785
1786	if (tight_bound)
1787	{
1788	/* VAL computed above represents an optimistically tight bound
1789	on the length of the string based on the referenced object's
1790	or subobject's type. Determine the conservative upper bound
1791	based on the enclosing object's size if possible. */
1792	if (rkind == SRK_LENRANGE)
1793	{
1794	poly_int64 offset;
1795	tree base = get_addr_base_and_unit_offset (arg, &offset);
1796	if (!base)
1797	{
1798	/* When the call above fails due to a non-constant offset
1799	assume the offset is zero and use the size of the whole
1800	enclosing object instead. */
1801	base = get_base_address (t: arg);
1802	offset = 0;
1803	}
1804	/* If the base object is a pointer no upper bound on the length
1805	can be determined. Otherwise the maximum length is equal to
1806	the size of the enclosing object minus the offset of
1807	the referenced subobject minus 1 (for the terminating nul). */
1808	tree type = TREE_TYPE (base);
1809	if (POINTER_TYPE_P (type)
1810	|| (TREE_CODE (base) != PARM_DECL && !VAR_P (base))
1811	|| !(val = DECL_SIZE_UNIT (base)))
1812	val = build_all_ones_cst (size_type_node);
1813	else
1814	{
1815	val = DECL_SIZE_UNIT (base);
1816	val = fold_build2 (MINUS_EXPR, TREE_TYPE (val), val,
1817	size_int (offset + 1));
1818	}
1819	}
1820	else
1821	return false;
1822	}
1823
1824	if (pdata->maxlen)
1825	{
1826	/* Adjust the more conservative bound if possible/necessary
1827	and fail otherwise. */
1828	if (rkind != SRK_STRLEN)
1829	{
1830	if (TREE_CODE (pdata->maxlen) != INTEGER_CST
1831	|| TREE_CODE (val) != INTEGER_CST)
1832	return false;
1833
1834	if (tree_int_cst_lt (t1: pdata->maxlen, t2: val))
1835	pdata->maxlen = val;
1836	return true;
1837	}
1838	else if (simple_cst_equal (val, pdata->maxlen) != 1)
1839	{
1840	/* Fail if the length of this ARG is different from that
1841	previously determined from another ARG. */
1842	return false;
1843	}
1844	}
1845
1846	pdata->maxlen = val;
1847	return rkind == SRK_LENRANGE || !integer_all_onesp (val);
1848	}
1849
1850	/* For an ARG referencing one or more strings, try to obtain the range
1851	of their lengths, or the size of the largest array ARG referes to if
1852	the range of lengths cannot be determined, and store all in *PDATA.
1853	For an integer ARG (when RKIND == SRK_INT_VALUE), try to determine
1854	the maximum constant value.
1855	If ARG is an SSA_NAME, follow its use-def chains. When RKIND ==
1856	SRK_STRLEN, then if PDATA->MAXLEN is not equal to the determined
1857	length or if we are unable to determine the length, return false.
1858	VISITED is a bitmap of visited variables.
1859	RKIND determines the kind of value or range to obtain (see
1860	strlen_range_kind).
1861	Set PDATA->DECL if ARG refers to an unterminated constant array.
1862	On input, set ELTSIZE to 1 for normal single byte character strings,
1863	and either 2 or 4 for wide characer strings (the size of wchar_t).
1864	Return true if *PDATA was successfully populated and false otherwise. */
1865
1866	static bool
1867	get_range_strlen (tree arg, bitmap visited,
1868	strlen_range_kind rkind,
1869	c_strlen_data *pdata, unsigned eltsize)
1870	{
1871
1872	if (TREE_CODE (arg) != SSA_NAME)
1873	return get_range_strlen_tree (arg, visited, rkind, pdata, eltsize);
1874
1875	/* If ARG is registered for SSA update we cannot look at its defining
1876	statement. */
1877	if (name_registered_for_update_p (arg))
1878	return false;
1879
1880	/* If we were already here, break the infinite cycle. */
1881	if (!bitmap_set_bit (visited, SSA_NAME_VERSION (arg)))
1882	return true;
1883
1884	tree var = arg;
1885	gimple *def_stmt = SSA_NAME_DEF_STMT (var);
1886
1887	switch (gimple_code (g: def_stmt))
1888	{
1889	case GIMPLE_ASSIGN:
1890	/* The RHS of the statement defining VAR must either have a
1891	constant length or come from another SSA_NAME with a constant
1892	length. */
1893	if (gimple_assign_single_p (gs: def_stmt)
1894	|| gimple_assign_unary_nop_p (def_stmt))
1895	{
1896	tree rhs = gimple_assign_rhs1 (gs: def_stmt);
1897	return get_range_strlen (arg: rhs, visited, rkind, pdata, eltsize);
1898	}
1899	else if (gimple_assign_rhs_code (gs: def_stmt) == COND_EXPR)
1900	{
1901	tree ops[2] = { gimple_assign_rhs2 (gs: def_stmt),
1902	gimple_assign_rhs3 (gs: def_stmt) };
1903
1904	for (unsigned int i = 0; i < 2; i++)
1905	if (!get_range_strlen (arg: ops[i], visited, rkind, pdata, eltsize))
1906	{
1907	if (rkind != SRK_LENRANGE)
1908	return false;
1909	/* Set the upper bound to the maximum to prevent
1910	it from being adjusted in the next iteration but
1911	leave MINLEN and the more conservative MAXBOUND
1912	determined so far alone (or leave them null if
1913	they haven't been set yet). That the MINLEN is
1914	in fact zero can be determined from MAXLEN being
1915	unbounded but the discovered minimum is used for
1916	diagnostics. */
1917	pdata->maxlen = build_all_ones_cst (size_type_node);
1918	}
1919	return true;
1920	}
1921	return false;
1922
1923	case GIMPLE_PHI:
1924	/* Unless RKIND == SRK_LENRANGE, all arguments of the PHI node
1925	must have a constant length. */
1926	for (unsigned i = 0; i < gimple_phi_num_args (gs: def_stmt); i++)
1927	{
1928	tree arg = gimple_phi_arg (gs: def_stmt, index: i)->def;
1929
1930	/* If this PHI has itself as an argument, we cannot
1931	determine the string length of this argument. However,
1932	if we can find a constant string length for the other
1933	PHI args then we can still be sure that this is a
1934	constant string length. So be optimistic and just
1935	continue with the next argument. */
1936	if (arg == gimple_phi_result (gs: def_stmt))
1937	continue;
1938
1939	if (!get_range_strlen (arg, visited, rkind, pdata, eltsize))
1940	{
1941	if (rkind != SRK_LENRANGE)
1942	return false;
1943	/* Set the upper bound to the maximum to prevent
1944	it from being adjusted in the next iteration but
1945	leave MINLEN and the more conservative MAXBOUND
1946	determined so far alone (or leave them null if
1947	they haven't been set yet). That the MINLEN is
1948	in fact zero can be determined from MAXLEN being
1949	unbounded but the discovered minimum is used for
1950	diagnostics. */
1951	pdata->maxlen = build_all_ones_cst (size_type_node);
1952	}
1953	}
1954	return true;
1955
1956	default:
1957	return false;
1958	}
1959	}
1960
1961	/* Try to obtain the range of the lengths of the string(s) referenced
1962	by ARG, or the size of the largest array ARG refers to if the range
1963	of lengths cannot be determined, and store all in *PDATA which must
1964	be zero-initialized on input except PDATA->MAXBOUND may be set to
1965	a non-null tree node other than INTEGER_CST to request to have it
1966	set to the length of the longest string in a PHI. ELTSIZE is
1967	the expected size of the string element in bytes: 1 for char and
1968	some power of 2 for wide characters.
1969	Return true if the range [PDATA->MINLEN, PDATA->MAXLEN] is suitable
1970	for optimization. Returning false means that a nonzero PDATA->MINLEN
1971	doesn't reflect the true lower bound of the range when PDATA->MAXLEN
1972	is -1 (in that case, the actual range is indeterminate, i.e.,
1973	[0, PTRDIFF_MAX - 2]. */
1974
1975	bool
1976	get_range_strlen (tree arg, c_strlen_data *pdata, unsigned eltsize)
1977	{
1978	auto_bitmap visited;
1979	tree maxbound = pdata->maxbound;
1980
1981	if (!get_range_strlen (arg, visited, rkind: SRK_LENRANGE, pdata, eltsize))
1982	{
1983	/* On failure extend the length range to an impossible maximum
1984	(a valid MAXLEN must be less than PTRDIFF_MAX - 1). Other
1985	members can stay unchanged regardless. */
1986	pdata->minlen = ssize_int (0);
1987	pdata->maxlen = build_all_ones_cst (size_type_node);
1988	}
1989	else if (!pdata->minlen)
1990	pdata->minlen = ssize_int (0);
1991
1992	/* If it's unchanged from it initial non-null value, set the conservative
1993	MAXBOUND to SIZE_MAX. Otherwise leave it null (if it is null). */
1994	if (maxbound && pdata->maxbound == maxbound)
1995	pdata->maxbound = build_all_ones_cst (size_type_node);
1996
1997	return !integer_all_onesp (pdata->maxlen);
1998	}
1999
2000	/* Return the maximum value for ARG given RKIND (see strlen_range_kind).
2001	For ARG of pointer types, NONSTR indicates if the caller is prepared
2002	to handle unterminated strings. For integer ARG and when RKIND ==
2003	SRK_INT_VALUE, NONSTR must be null.
2004
2005	If an unterminated array is discovered and our caller handles
2006	unterminated arrays, then bubble up the offending DECL and
2007	return the maximum size. Otherwise return NULL. */
2008
2009	static tree
2010	get_maxval_strlen (tree arg, strlen_range_kind rkind, tree *nonstr = NULL)
2011	{
2012	/* A non-null NONSTR is meaningless when determining the maximum
2013	value of an integer ARG. */
2014	gcc_assert (rkind != SRK_INT_VALUE || nonstr == NULL);
2015	/* ARG must have an integral type when RKIND says so. */
2016	gcc_assert (rkind != SRK_INT_VALUE || INTEGRAL_TYPE_P (TREE_TYPE (arg)));
2017
2018	auto_bitmap visited;
2019
2020	/* Reset DATA.MAXLEN if the call fails or when DATA.MAXLEN
2021	is unbounded. */
2022	c_strlen_data lendata = { };
2023	if (!get_range_strlen (arg, visited, rkind, pdata: &lendata, /* eltsize = */1))
2024	lendata.maxlen = NULL_TREE;
2025	else if (lendata.maxlen && integer_all_onesp (lendata.maxlen))
2026	lendata.maxlen = NULL_TREE;
2027
2028	if (nonstr)
2029	{
2030	/* For callers prepared to handle unterminated arrays set
2031	*NONSTR to point to the declaration of the array and return
2032	the maximum length/size. */
2033	*nonstr = lendata.decl;
2034	return lendata.maxlen;
2035	}
2036
2037	/* Fail if the constant array isn't nul-terminated. */
2038	return lendata.decl ? NULL_TREE : lendata.maxlen;
2039	}
2040
2041	/* Return true if LEN is known to be less than or equal to (or if STRICT is
2042	true, strictly less than) the lower bound of SIZE at compile time and false
2043	otherwise. */
2044
2045	static bool
2046	known_lower (gimple *stmt, tree len, tree size, bool strict = false)
2047	{
2048	if (len == NULL_TREE)
2049	return false;
2050
2051	wide_int size_range[2];
2052	wide_int len_range[2];
2053	if (get_range (len, stmt, len_range) && get_range (size, stmt, size_range))
2054	{
2055	if (strict)
2056	return wi::ltu_p (x: len_range[1], y: size_range[0]);
2057	else
2058	return wi::leu_p (x: len_range[1], y: size_range[0]);
2059	}
2060
2061	return false;
2062	}
2063
2064	/* Fold function call to builtin strcpy with arguments DEST and SRC.
2065	If LEN is not NULL, it represents the length of the string to be
2066	copied. Return NULL_TREE if no simplification can be made. */
2067
2068	static bool
2069	gimple_fold_builtin_strcpy (gimple_stmt_iterator *gsi,
2070	tree dest, tree src)
2071	{
2072	gimple *stmt = gsi_stmt (i: *gsi);
2073	location_t loc = gimple_location (g: stmt);
2074	tree fn;
2075
2076	/* If SRC and DEST are the same (and not volatile), return DEST. */
2077	if (operand_equal_p (src, dest, flags: 0))
2078	{
2079	/* Issue -Wrestrict unless the pointers are null (those do
2080	not point to objects and so do not indicate an overlap;
2081	such calls could be the result of sanitization and jump
2082	threading). */
2083	if (!integer_zerop (dest) && !warning_suppressed_p (stmt, OPT_Wrestrict))
2084	{
2085	tree func = gimple_call_fndecl (gs: stmt);
2086
2087	warning_at (loc, OPT_Wrestrict,
2088	"%qD source argument is the same as destination",
2089	func);
2090	}
2091
2092	replace_call_with_value (gsi, val: dest);
2093	return true;
2094	}
2095
2096	if (optimize_function_for_size_p (cfun))
2097	return false;
2098
2099	fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
2100	if (!fn)
2101	return false;
2102
2103	/* Set to non-null if ARG refers to an unterminated array. */
2104	tree nonstr = NULL;
2105	tree len = get_maxval_strlen (arg: src, rkind: SRK_STRLEN, nonstr: &nonstr);
2106
2107	if (nonstr)
2108	{
2109	/* Avoid folding calls with unterminated arrays. */
2110	if (!warning_suppressed_p (stmt, OPT_Wstringop_overread))
2111	warn_string_no_nul (loc, stmt, "strcpy", src, nonstr);
2112	suppress_warning (stmt, OPT_Wstringop_overread);
2113	return false;
2114	}
2115
2116	if (!len || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
2117	return false;
2118
2119	len = fold_convert_loc (loc, size_type_node, len);
2120	len = size_binop_loc (loc, PLUS_EXPR, len, build_int_cst (size_type_node, 1));
2121	len = force_gimple_operand_gsi (gsi, len, true,
2122	NULL_TREE, true, GSI_SAME_STMT);
2123	gimple *repl = gimple_build_call (fn, 3, dest, src, len);
2124	replace_call_with_call_and_fold (gsi, repl);
2125	return true;
2126	}
2127
2128	/* Fold function call to builtin strncpy with arguments DEST, SRC, and LEN.
2129	If SLEN is not NULL, it represents the length of the source string.
2130	Return NULL_TREE if no simplification can be made. */
2131
2132	static bool
2133	gimple_fold_builtin_strncpy (gimple_stmt_iterator *gsi,
2134	tree dest, tree src, tree len)
2135	{
2136	gimple *stmt = gsi_stmt (i: *gsi);
2137	location_t loc = gimple_location (g: stmt);
2138	bool nonstring = get_attr_nonstring_decl (dest) != NULL_TREE;
2139
2140	/* If the LEN parameter is zero, return DEST. */
2141	if (integer_zerop (len))
2142	{
2143	/* Avoid warning if the destination refers to an array/pointer
2144	decorate with attribute nonstring. */
2145	if (!nonstring)
2146	{
2147	tree fndecl = gimple_call_fndecl (gs: stmt);
2148
2149	/* Warn about the lack of nul termination: the result is not
2150	a (nul-terminated) string. */
2151	tree slen = get_maxval_strlen (arg: src, rkind: SRK_STRLEN);
2152	if (slen && !integer_zerop (slen))
2153	warning_at (loc, OPT_Wstringop_truncation,
2154	"%qD destination unchanged after copying no bytes "
2155	"from a string of length %E",
2156	fndecl, slen);
2157	else
2158	warning_at (loc, OPT_Wstringop_truncation,
2159	"%qD destination unchanged after copying no bytes",
2160	fndecl);
2161	}
2162
2163	replace_call_with_value (gsi, val: dest);
2164	return true;
2165	}
2166
2167	/* We can't compare slen with len as constants below if len is not a
2168	constant. */
2169	if (TREE_CODE (len) != INTEGER_CST)
2170	return false;
2171
2172	/* Now, we must be passed a constant src ptr parameter. */
2173	tree slen = get_maxval_strlen (arg: src, rkind: SRK_STRLEN);
2174	if (!slen || TREE_CODE (slen) != INTEGER_CST)
2175	return false;
2176
2177	/* The size of the source string including the terminating nul. */
2178	tree ssize = size_binop_loc (loc, PLUS_EXPR, slen, ssize_int (1));
2179
2180	/* We do not support simplification of this case, though we do
2181	support it when expanding trees into RTL. */
2182	/* FIXME: generate a call to __builtin_memset. */
2183	if (tree_int_cst_lt (t1: ssize, t2: len))
2184	return false;
2185
2186	/* Diagnose truncation that leaves the copy unterminated. */
2187	maybe_diag_stxncpy_trunc (*gsi, src, len);
2188
2189	/* OK transform into builtin memcpy. */
2190	tree fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
2191	if (!fn || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
2192	return false;
2193
2194	len = fold_convert_loc (loc, size_type_node, len);
2195	len = force_gimple_operand_gsi (gsi, len, true,
2196	NULL_TREE, true, GSI_SAME_STMT);
2197	gimple *repl = gimple_build_call (fn, 3, dest, src, len);
2198	replace_call_with_call_and_fold (gsi, repl);
2199
2200	return true;
2201	}
2202
2203	/* Fold function call to builtin strchr or strrchr.
2204	If both arguments are constant, evaluate and fold the result,
2205	otherwise simplify str(r)chr (str, 0) into str + strlen (str).
2206	In general strlen is significantly faster than strchr
2207	due to being a simpler operation. */
2208	static bool
2209	gimple_fold_builtin_strchr (gimple_stmt_iterator *gsi, bool is_strrchr)
2210	{
2211	gimple *stmt = gsi_stmt (i: *gsi);
2212	tree str = gimple_call_arg (gs: stmt, index: 0);
2213	tree c = gimple_call_arg (gs: stmt, index: 1);
2214	location_t loc = gimple_location (g: stmt);
2215	const char *p;
2216	char ch;
2217
2218	if (!gimple_call_lhs (gs: stmt))
2219	return false;
2220
2221	/* Avoid folding if the first argument is not a nul-terminated array.
2222	Defer warning until later. */
2223	if (!check_nul_terminated_array (NULL_TREE, str))
2224	return false;
2225
2226	if ((p = c_getstr (str)) && target_char_cst_p (t: c, p: &ch))
2227	{
2228	const char *p1 = is_strrchr ? strrchr (s: p, c: ch) : strchr (s: p, c: ch);
2229
2230	if (p1 == NULL)
2231	{
2232	replace_call_with_value (gsi, integer_zero_node);
2233	return true;
2234	}
2235
2236	tree len = build_int_cst (size_type_node, p1 - p);
2237	gimple_seq stmts = NULL;
2238	gimple *new_stmt = gimple_build_assign (gimple_call_lhs (gs: stmt),
2239	POINTER_PLUS_EXPR, str, len);
2240	gimple_seq_add_stmt_without_update (&stmts, new_stmt);
2241	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2242	return true;
2243	}
2244
2245	if (!integer_zerop (c) || (!gimple_vuse (g: stmt) && gimple_in_ssa_p (cfun)))
2246	return false;
2247
2248	/* Transform strrchr (s, 0) to strchr (s, 0) when optimizing for size. */
2249	if (is_strrchr && optimize_function_for_size_p (cfun))
2250	{
2251	tree strchr_fn = builtin_decl_implicit (fncode: BUILT_IN_STRCHR);
2252
2253	if (strchr_fn)
2254	{
2255	gimple *repl = gimple_build_call (strchr_fn, 2, str, c);
2256	replace_call_with_call_and_fold (gsi, repl);
2257	return true;
2258	}
2259
2260	return false;
2261	}
2262
2263	tree len;
2264	tree strlen_fn = builtin_decl_implicit (fncode: BUILT_IN_STRLEN);
2265
2266	if (!strlen_fn)
2267	return false;
2268
2269	/* Create newstr = strlen (str). */
2270	gimple_seq stmts = NULL;
2271	gimple *new_stmt = gimple_build_call (strlen_fn, 1, str);
2272	gimple_set_location (g: new_stmt, location: loc);
2273	len = make_ssa_name (size_type_node);
2274	gimple_call_set_lhs (gs: new_stmt, lhs: len);
2275	gimple_seq_add_stmt_without_update (&stmts, new_stmt);
2276
2277	/* Create (str p+ strlen (str)). */
2278	new_stmt = gimple_build_assign (gimple_call_lhs (gs: stmt),
2279	POINTER_PLUS_EXPR, str, len);
2280	gimple_seq_add_stmt_without_update (&stmts, new_stmt);
2281	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2282	/* gsi now points at the assignment to the lhs, get a
2283	stmt iterator to the strlen.
2284	??? We can't use gsi_for_stmt as that doesn't work when the
2285	CFG isn't built yet. */
2286	gimple_stmt_iterator gsi2 = *gsi;
2287	gsi_prev (i: &gsi2);
2288	fold_stmt (&gsi2);
2289	return true;
2290	}
2291
2292	/* Fold function call to builtin strstr.
2293	If both arguments are constant, evaluate and fold the result,
2294	additionally fold strstr (x, "") into x and strstr (x, "c")
2295	into strchr (x, 'c'). */
2296	static bool
2297	gimple_fold_builtin_strstr (gimple_stmt_iterator *gsi)
2298	{
2299	gimple *stmt = gsi_stmt (i: *gsi);
2300	if (!gimple_call_lhs (gs: stmt))
2301	return false;
2302
2303	tree haystack = gimple_call_arg (gs: stmt, index: 0);
2304	tree needle = gimple_call_arg (gs: stmt, index: 1);
2305
2306	/* Avoid folding if either argument is not a nul-terminated array.
2307	Defer warning until later. */
2308	if (!check_nul_terminated_array (NULL_TREE, haystack)
2309	|| !check_nul_terminated_array (NULL_TREE, needle))
2310	return false;
2311
2312	const char *q = c_getstr (needle);
2313	if (q == NULL)
2314	return false;
2315
2316	if (const char *p = c_getstr (haystack))
2317	{
2318	const char *r = strstr (haystack: p, needle: q);
2319
2320	if (r == NULL)
2321	{
2322	replace_call_with_value (gsi, integer_zero_node);
2323	return true;
2324	}
2325
2326	tree len = build_int_cst (size_type_node, r - p);
2327	gimple_seq stmts = NULL;
2328	gimple *new_stmt
2329	= gimple_build_assign (gimple_call_lhs (gs: stmt), POINTER_PLUS_EXPR,
2330	haystack, len);
2331	gimple_seq_add_stmt_without_update (&stmts, new_stmt);
2332	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2333	return true;
2334	}
2335
2336	/* For strstr (x, "") return x. */
2337	if (q[0] == '\0')
2338	{
2339	replace_call_with_value (gsi, val: haystack);
2340	return true;
2341	}
2342
2343	if (!gimple_vuse (g: stmt) && gimple_in_ssa_p (cfun))
2344	return false;
2345
2346	/* Transform strstr (x, "c") into strchr (x, 'c'). */
2347	if (q[1] == '\0')
2348	{
2349	tree strchr_fn = builtin_decl_implicit (fncode: BUILT_IN_STRCHR);
2350	if (strchr_fn)
2351	{
2352	tree c = build_int_cst (integer_type_node, q[0]);
2353	gimple *repl = gimple_build_call (strchr_fn, 2, haystack, c);
2354	replace_call_with_call_and_fold (gsi, repl);
2355	return true;
2356	}
2357	}
2358
2359	return false;
2360	}
2361
2362	/* Simplify a call to the strcat builtin. DST and SRC are the arguments
2363	to the call.
2364
2365	Return NULL_TREE if no simplification was possible, otherwise return the
2366	simplified form of the call as a tree.
2367
2368	The simplified form may be a constant or other expression which
2369	computes the same value, but in a more efficient manner (including
2370	calls to other builtin functions).
2371
2372	The call may contain arguments which need to be evaluated, but
2373	which are not useful to determine the result of the call. In
2374	this case we return a chain of COMPOUND_EXPRs. The LHS of each
2375	COMPOUND_EXPR will be an argument which must be evaluated.
2376	COMPOUND_EXPRs are chained through their RHS. The RHS of the last
2377	COMPOUND_EXPR in the chain will contain the tree for the simplified
2378	form of the builtin function call. */
2379
2380	static bool
2381	gimple_fold_builtin_strcat (gimple_stmt_iterator *gsi, tree dst, tree src)
2382	{
2383	gimple *stmt = gsi_stmt (i: *gsi);
2384	location_t loc = gimple_location (g: stmt);
2385
2386	const char *p = c_getstr (src);
2387
2388	/* If the string length is zero, return the dst parameter. */
2389	if (p && *p == '\0')
2390	{
2391	replace_call_with_value (gsi, val: dst);
2392	return true;
2393	}
2394
2395	if (!optimize_bb_for_speed_p (gimple_bb (g: stmt)))
2396	return false;
2397
2398	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
2399	return false;
2400
2401	/* See if we can store by pieces into (dst + strlen(dst)). */
2402	tree newdst;
2403	tree strlen_fn = builtin_decl_implicit (fncode: BUILT_IN_STRLEN);
2404	tree memcpy_fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
2405
2406	if (!strlen_fn || !memcpy_fn)
2407	return false;
2408
2409	/* If the length of the source string isn't computable don't
2410	split strcat into strlen and memcpy. */
2411	tree len = get_maxval_strlen (arg: src, rkind: SRK_STRLEN);
2412	if (! len)
2413	return false;
2414
2415	/* Create strlen (dst). */
2416	gimple_seq stmts = NULL, stmts2;
2417	gimple *repl = gimple_build_call (strlen_fn, 1, dst);
2418	gimple_set_location (g: repl, location: loc);
2419	newdst = make_ssa_name (size_type_node);
2420	gimple_call_set_lhs (gs: repl, lhs: newdst);
2421	gimple_seq_add_stmt_without_update (&stmts, repl);
2422
2423	/* Create (dst p+ strlen (dst)). */
2424	newdst = fold_build_pointer_plus_loc (loc, ptr: dst, off: newdst);
2425	newdst = force_gimple_operand (newdst, &stmts2, true, NULL_TREE);
2426	gimple_seq_add_seq_without_update (&stmts, stmts2);
2427
2428	len = fold_convert_loc (loc, size_type_node, len);
2429	len = size_binop_loc (loc, PLUS_EXPR, len,
2430	build_int_cst (size_type_node, 1));
2431	len = force_gimple_operand (len, &stmts2, true, NULL_TREE);
2432	gimple_seq_add_seq_without_update (&stmts, stmts2);
2433
2434	repl = gimple_build_call (memcpy_fn, 3, newdst, src, len);
2435	gimple_seq_add_stmt_without_update (&stmts, repl);
2436	if (gimple_call_lhs (gs: stmt))
2437	{
2438	repl = gimple_build_assign (gimple_call_lhs (gs: stmt), dst);
2439	gimple_seq_add_stmt_without_update (&stmts, repl);
2440	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2441	/* gsi now points at the assignment to the lhs, get a
2442	stmt iterator to the memcpy call.
2443	??? We can't use gsi_for_stmt as that doesn't work when the
2444	CFG isn't built yet. */
2445	gimple_stmt_iterator gsi2 = *gsi;
2446	gsi_prev (i: &gsi2);
2447	fold_stmt (&gsi2);
2448	}
2449	else
2450	{
2451	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2452	fold_stmt (gsi);
2453	}
2454	return true;
2455	}
2456
2457	/* Fold a call to the __strcat_chk builtin FNDECL. DEST, SRC, and SIZE
2458	are the arguments to the call. */
2459
2460	static bool
2461	gimple_fold_builtin_strcat_chk (gimple_stmt_iterator *gsi)
2462	{
2463	gimple *stmt = gsi_stmt (i: *gsi);
2464	tree dest = gimple_call_arg (gs: stmt, index: 0);
2465	tree src = gimple_call_arg (gs: stmt, index: 1);
2466	tree size = gimple_call_arg (gs: stmt, index: 2);
2467	tree fn;
2468	const char *p;
2469
2470	p = c_getstr (src);
2471	/* If the SRC parameter is "", return DEST. */
2472	if (p && *p == '\0')
2473	{
2474	replace_call_with_value (gsi, val: dest);
2475	return true;
2476	}
2477
2478	if (! tree_fits_uhwi_p (size) || ! integer_all_onesp (size))
2479	return false;
2480
2481	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
2482	return false;
2483
2484	/* If __builtin_strcat_chk is used, assume strcat is available. */
2485	fn = builtin_decl_explicit (fncode: BUILT_IN_STRCAT);
2486	if (!fn)
2487	return false;
2488
2489	gimple *repl = gimple_build_call (fn, 2, dest, src);
2490	replace_call_with_call_and_fold (gsi, repl);
2491	return true;
2492	}
2493
2494	/* Simplify a call to the strncat builtin. */
2495
2496	static bool
2497	gimple_fold_builtin_strncat (gimple_stmt_iterator *gsi)
2498	{
2499	gimple *stmt = gsi_stmt (i: *gsi);
2500	tree dst = gimple_call_arg (gs: stmt, index: 0);
2501	tree src = gimple_call_arg (gs: stmt, index: 1);
2502	tree len = gimple_call_arg (gs: stmt, index: 2);
2503	tree src_len = c_strlen (src, 1);
2504
2505	/* If the requested length is zero, or the src parameter string
2506	length is zero, return the dst parameter. */
2507	if (integer_zerop (len) || (src_len && integer_zerop (src_len)))
2508	{
2509	replace_call_with_value (gsi, val: dst);
2510	return true;
2511	}
2512
2513	/* Return early if the requested len is less than the string length.
2514	Warnings will be issued elsewhere later. */
2515	if (!src_len || known_lower (stmt, len, size: src_len, strict: true))
2516	return false;
2517
2518	/* Warn on constant LEN. */
2519	if (TREE_CODE (len) == INTEGER_CST)
2520	{
2521	bool nowarn = warning_suppressed_p (stmt, OPT_Wstringop_overflow_);
2522	tree dstsize;
2523
2524	if (!nowarn && compute_builtin_object_size (dst, 1, &dstsize)
2525	&& TREE_CODE (dstsize) == INTEGER_CST)
2526	{
2527	int cmpdst = tree_int_cst_compare (t1: len, t2: dstsize);
2528
2529	if (cmpdst >= 0)
2530	{
2531	tree fndecl = gimple_call_fndecl (gs: stmt);
2532
2533	/* Strncat copies (at most) LEN bytes and always appends
2534	the terminating NUL so the specified bound should never
2535	be equal to (or greater than) the size of the destination.
2536	If it is, the copy could overflow. */
2537	location_t loc = gimple_location (g: stmt);
2538	nowarn = warning_at (loc, OPT_Wstringop_overflow_,
2539	cmpdst == 0
2540	? G_("%qD specified bound %E equals "
2541	"destination size")
2542	: G_("%qD specified bound %E exceeds "
2543	"destination size %E"),
2544	fndecl, len, dstsize);
2545	if (nowarn)
2546	suppress_warning (stmt, OPT_Wstringop_overflow_);
2547	}
2548	}
2549
2550	if (!nowarn && TREE_CODE (src_len) == INTEGER_CST
2551	&& tree_int_cst_compare (t1: src_len, t2: len) == 0)
2552	{
2553	tree fndecl = gimple_call_fndecl (gs: stmt);
2554	location_t loc = gimple_location (g: stmt);
2555
2556	/* To avoid possible overflow the specified bound should also
2557	not be equal to the length of the source, even when the size
2558	of the destination is unknown (it's not an uncommon mistake
2559	to specify as the bound to strncpy the length of the source). */
2560	if (warning_at (loc, OPT_Wstringop_overflow_,
2561	"%qD specified bound %E equals source length",
2562	fndecl, len))
2563	suppress_warning (stmt, OPT_Wstringop_overflow_);
2564	}
2565	}
2566
2567	if (!known_lower (stmt, len: src_len, size: len))
2568	return false;
2569
2570	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCAT);
2571
2572	/* If the replacement _DECL isn't initialized, don't do the
2573	transformation. */
2574	if (!fn || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
2575	return false;
2576
2577	/* Otherwise, emit a call to strcat. */
2578	gcall *repl = gimple_build_call (fn, 2, dst, src);
2579	replace_call_with_call_and_fold (gsi, repl);
2580	return true;
2581	}
2582
2583	/* Fold a call to the __strncat_chk builtin with arguments DEST, SRC,
2584	LEN, and SIZE. */
2585
2586	static bool
2587	gimple_fold_builtin_strncat_chk (gimple_stmt_iterator *gsi)
2588	{
2589	gimple *stmt = gsi_stmt (i: *gsi);
2590	tree dest = gimple_call_arg (gs: stmt, index: 0);
2591	tree src = gimple_call_arg (gs: stmt, index: 1);
2592	tree len = gimple_call_arg (gs: stmt, index: 2);
2593	tree size = gimple_call_arg (gs: stmt, index: 3);
2594	tree fn;
2595	const char *p;
2596
2597	p = c_getstr (src);
2598	/* If the SRC parameter is "" or if LEN is 0, return DEST. */
2599	if ((p && *p == '\0')
2600	|| integer_zerop (len))
2601	{
2602	replace_call_with_value (gsi, val: dest);
2603	return true;
2604	}
2605
2606	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
2607	return false;
2608
2609	if (! integer_all_onesp (size))
2610	{
2611	tree src_len = c_strlen (src, 1);
2612	if (known_lower (stmt, len: src_len, size: len))
2613	{
2614	/* If LEN >= strlen (SRC), optimize into __strcat_chk. */
2615	fn = builtin_decl_explicit (fncode: BUILT_IN_STRCAT_CHK);
2616	if (!fn)
2617	return false;
2618
2619	gimple *repl = gimple_build_call (fn, 3, dest, src, size);
2620	replace_call_with_call_and_fold (gsi, repl);
2621	return true;
2622	}
2623	return false;
2624	}
2625
2626	/* If __builtin_strncat_chk is used, assume strncat is available. */
2627	fn = builtin_decl_explicit (fncode: BUILT_IN_STRNCAT);
2628	if (!fn)
2629	return false;
2630
2631	gimple *repl = gimple_build_call (fn, 3, dest, src, len);
2632	replace_call_with_call_and_fold (gsi, repl);
2633	return true;
2634	}
2635
2636	/* Build and append gimple statements to STMTS that would load a first
2637	character of a memory location identified by STR. LOC is location
2638	of the statement. */
2639
2640	static tree
2641	gimple_load_first_char (location_t loc, tree str, gimple_seq *stmts)
2642	{
2643	tree var;
2644
2645	tree cst_uchar_node = build_type_variant (unsigned_char_type_node, 1, 0);
2646	tree cst_uchar_ptr_node
2647	= build_pointer_type_for_mode (cst_uchar_node, ptr_mode, true);
2648	tree off0 = build_int_cst (cst_uchar_ptr_node, 0);
2649
2650	tree temp = fold_build2_loc (loc, MEM_REF, cst_uchar_node, str, off0);
2651	gassign *stmt = gimple_build_assign (NULL_TREE, temp);
2652	var = make_ssa_name (var: cst_uchar_node, stmt);
2653
2654	gimple_assign_set_lhs (gs: stmt, lhs: var);
2655	gimple_seq_add_stmt_without_update (stmts, stmt);
2656
2657	return var;
2658	}
2659
2660	/* Fold a call to the str{n}{case}cmp builtin pointed by GSI iterator. */
2661
2662	static bool
2663	gimple_fold_builtin_string_compare (gimple_stmt_iterator *gsi)
2664	{
2665	gimple *stmt = gsi_stmt (i: *gsi);
2666	tree callee = gimple_call_fndecl (gs: stmt);
2667	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: callee);
2668
2669	tree type = integer_type_node;
2670	tree str1 = gimple_call_arg (gs: stmt, index: 0);
2671	tree str2 = gimple_call_arg (gs: stmt, index: 1);
2672	tree lhs = gimple_call_lhs (gs: stmt);
2673
2674	tree bound_node = NULL_TREE;
2675	unsigned HOST_WIDE_INT bound = HOST_WIDE_INT_M1U;
2676
2677	/* Handle strncmp and strncasecmp functions. */
2678	if (gimple_call_num_args (gs: stmt) == 3)
2679	{
2680	bound_node = gimple_call_arg (gs: stmt, index: 2);
2681	if (tree_fits_uhwi_p (bound_node))
2682	bound = tree_to_uhwi (bound_node);
2683	}
2684
2685	/* If the BOUND parameter is zero, return zero. */
2686	if (bound == 0)
2687	{
2688	replace_call_with_value (gsi, integer_zero_node);
2689	return true;
2690	}
2691
2692	/* If ARG1 and ARG2 are the same (and not volatile), return zero. */
2693	if (operand_equal_p (str1, str2, flags: 0))
2694	{
2695	replace_call_with_value (gsi, integer_zero_node);
2696	return true;
2697	}
2698
2699	if (!gimple_vuse (g: stmt) && gimple_in_ssa_p (cfun))
2700	return false;
2701
2702	/* Initially set to the number of characters, including the terminating
2703	nul if each array has one. LENx == strnlen (Sx, LENx) implies that
2704	the array Sx is not terminated by a nul.
2705	For nul-terminated strings then adjusted to their length so that
2706	LENx == NULPOSx holds. */
2707	unsigned HOST_WIDE_INT len1 = HOST_WIDE_INT_MAX, len2 = len1;
2708	const char *p1 = getbyterep (str1, &len1);
2709	const char *p2 = getbyterep (str2, &len2);
2710
2711	/* The position of the terminating nul character if one exists, otherwise
2712	a value greater than LENx. */
2713	unsigned HOST_WIDE_INT nulpos1 = HOST_WIDE_INT_MAX, nulpos2 = nulpos1;
2714
2715	if (p1)
2716	{
2717	size_t n = strnlen (string: p1, maxlen: len1);
2718	if (n < len1)
2719	len1 = nulpos1 = n;
2720	}
2721
2722	if (p2)
2723	{
2724	size_t n = strnlen (string: p2, maxlen: len2);
2725	if (n < len2)
2726	len2 = nulpos2 = n;
2727	}
2728
2729	/* For known strings, return an immediate value. */
2730	if (p1 && p2)
2731	{
2732	int r = 0;
2733	bool known_result = false;
2734
2735	switch (fcode)
2736	{
2737	case BUILT_IN_STRCMP:
2738	case BUILT_IN_STRCMP_EQ:
2739	if (len1 != nulpos1 || len2 != nulpos2)
2740	break;
2741
2742	r = strcmp (s1: p1, s2: p2);
2743	known_result = true;
2744	break;
2745
2746	case BUILT_IN_STRNCMP:
2747	case BUILT_IN_STRNCMP_EQ:
2748	{
2749	if (bound == HOST_WIDE_INT_M1U)
2750	break;
2751
2752	/* Reduce the bound to be no more than the length
2753	of the shorter of the two strings, or the sizes
2754	of the unterminated arrays. */
2755	unsigned HOST_WIDE_INT n = bound;
2756
2757	if (len1 == nulpos1 && len1 < n)
2758	n = len1 + 1;
2759	if (len2 == nulpos2 && len2 < n)
2760	n = len2 + 1;
2761
2762	if (MIN (nulpos1, nulpos2) + 1 < n)
2763	break;
2764
2765	r = strncmp (s1: p1, s2: p2, n: n);
2766	known_result = true;
2767	break;
2768	}
2769	/* Only handleable situation is where the string are equal (result 0),
2770	which is already handled by operand_equal_p case. */
2771	case BUILT_IN_STRCASECMP:
2772	break;
2773	case BUILT_IN_STRNCASECMP:
2774	{
2775	if (bound == HOST_WIDE_INT_M1U)
2776	break;
2777	r = strncmp (s1: p1, s2: p2, n: bound);
2778	if (r == 0)
2779	known_result = true;
2780	break;
2781	}
2782	default:
2783	gcc_unreachable ();
2784	}
2785
2786	if (known_result)
2787	{
2788	replace_call_with_value (gsi, val: build_cmp_result (type, res: r));
2789	return true;
2790	}
2791	}
2792
2793	bool nonzero_bound = (bound >= 1 && bound < HOST_WIDE_INT_M1U)
2794	|| fcode == BUILT_IN_STRCMP
2795	|| fcode == BUILT_IN_STRCMP_EQ
2796	|| fcode == BUILT_IN_STRCASECMP;
2797
2798	location_t loc = gimple_location (g: stmt);
2799
2800	/* If the second arg is "", return *(const unsigned char*)arg1. */
2801	if (p2 && *p2 == '\0' && nonzero_bound)
2802	{
2803	gimple_seq stmts = NULL;
2804	tree var = gimple_load_first_char (loc, str: str1, stmts: &stmts);
2805	if (lhs)
2806	{
2807	stmt = gimple_build_assign (lhs, NOP_EXPR, var);
2808	gimple_seq_add_stmt_without_update (&stmts, stmt);
2809	}
2810
2811	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2812	return true;
2813	}
2814
2815	/* If the first arg is "", return -*(const unsigned char*)arg2. */
2816	if (p1 && *p1 == '\0' && nonzero_bound)
2817	{
2818	gimple_seq stmts = NULL;
2819	tree var = gimple_load_first_char (loc, str: str2, stmts: &stmts);
2820
2821	if (lhs)
2822	{
2823	tree c = make_ssa_name (integer_type_node);
2824	stmt = gimple_build_assign (c, NOP_EXPR, var);
2825	gimple_seq_add_stmt_without_update (&stmts, stmt);
2826
2827	stmt = gimple_build_assign (lhs, NEGATE_EXPR, c);
2828	gimple_seq_add_stmt_without_update (&stmts, stmt);
2829	}
2830
2831	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2832	return true;
2833	}
2834
2835	/* If BOUND is one, return an expression corresponding to
2836	(*(const unsigned char*)arg2 - *(const unsigned char*)arg1). */
2837	if (fcode == BUILT_IN_STRNCMP && bound == 1)
2838	{
2839	gimple_seq stmts = NULL;
2840	tree temp1 = gimple_load_first_char (loc, str: str1, stmts: &stmts);
2841	tree temp2 = gimple_load_first_char (loc, str: str2, stmts: &stmts);
2842
2843	if (lhs)
2844	{
2845	tree c1 = make_ssa_name (integer_type_node);
2846	gassign *convert1 = gimple_build_assign (c1, NOP_EXPR, temp1);
2847	gimple_seq_add_stmt_without_update (&stmts, convert1);
2848
2849	tree c2 = make_ssa_name (integer_type_node);
2850	gassign *convert2 = gimple_build_assign (c2, NOP_EXPR, temp2);
2851	gimple_seq_add_stmt_without_update (&stmts, convert2);
2852
2853	stmt = gimple_build_assign (lhs, MINUS_EXPR, c1, c2);
2854	gimple_seq_add_stmt_without_update (&stmts, stmt);
2855	}
2856
2857	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2858	return true;
2859	}
2860
2861	/* If BOUND is greater than the length of one constant string,
2862	and the other argument is also a nul-terminated string, replace
2863	strncmp with strcmp. */
2864	if (fcode == BUILT_IN_STRNCMP
2865	&& bound > 0 && bound < HOST_WIDE_INT_M1U
2866	&& ((p2 && len2 < bound && len2 == nulpos2)
2867	|| (p1 && len1 < bound && len1 == nulpos1)))
2868	{
2869	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCMP);
2870	if (!fn)
2871	return false;
2872	gimple *repl = gimple_build_call (fn, 2, str1, str2);
2873	replace_call_with_call_and_fold (gsi, repl);
2874	return true;
2875	}
2876
2877	return false;
2878	}
2879
2880	/* Fold a call to the memchr pointed by GSI iterator. */
2881
2882	static bool
2883	gimple_fold_builtin_memchr (gimple_stmt_iterator *gsi)
2884	{
2885	gimple *stmt = gsi_stmt (i: *gsi);
2886	tree lhs = gimple_call_lhs (gs: stmt);
2887	tree arg1 = gimple_call_arg (gs: stmt, index: 0);
2888	tree arg2 = gimple_call_arg (gs: stmt, index: 1);
2889	tree len = gimple_call_arg (gs: stmt, index: 2);
2890
2891	/* If the LEN parameter is zero, return zero. */
2892	if (integer_zerop (len))
2893	{
2894	replace_call_with_value (gsi, val: build_int_cst (ptr_type_node, 0));
2895	return true;
2896	}
2897
2898	char c;
2899	if (TREE_CODE (arg2) != INTEGER_CST
2900	|| !tree_fits_uhwi_p (len)
2901	|| !target_char_cst_p (t: arg2, p: &c))
2902	return false;
2903
2904	unsigned HOST_WIDE_INT length = tree_to_uhwi (len);
2905	unsigned HOST_WIDE_INT string_length;
2906	const char *p1 = getbyterep (arg1, &string_length);
2907
2908	if (p1)
2909	{
2910	const char *r = (const char *)memchr (s: p1, c: c, MIN (length, string_length));
2911	if (r == NULL)
2912	{
2913	tree mem_size, offset_node;
2914	byte_representation (arg1, &offset_node, &mem_size, NULL);
2915	unsigned HOST_WIDE_INT offset = (offset_node == NULL_TREE)
2916	? 0 : tree_to_uhwi (offset_node);
2917	/* MEM_SIZE is the size of the array the string literal
2918	is stored in. */
2919	unsigned HOST_WIDE_INT string_size = tree_to_uhwi (mem_size) - offset;
2920	gcc_checking_assert (string_length <= string_size);
2921	if (length <= string_size)
2922	{
2923	replace_call_with_value (gsi, val: build_int_cst (ptr_type_node, 0));
2924	return true;
2925	}
2926	}
2927	else
2928	{
2929	unsigned HOST_WIDE_INT offset = r - p1;
2930	gimple_seq stmts = NULL;
2931	if (lhs != NULL_TREE)
2932	{
2933	tree offset_cst = build_int_cst (sizetype, offset);
2934	gassign *stmt = gimple_build_assign (lhs, POINTER_PLUS_EXPR,
2935	arg1, offset_cst);
2936	gimple_seq_add_stmt_without_update (&stmts, stmt);
2937	}
2938	else
2939	gimple_seq_add_stmt_without_update (&stmts,
2940	gimple_build_nop ());
2941
2942	gsi_replace_with_seq_vops (si_p: gsi, stmts);
2943	return true;
2944	}
2945	}
2946
2947	return false;
2948	}
2949
2950	/* Fold a call to the fputs builtin. ARG0 and ARG1 are the arguments
2951	to the call. IGNORE is true if the value returned
2952	by the builtin will be ignored. UNLOCKED is true is true if this
2953	actually a call to fputs_unlocked. If LEN in non-NULL, it represents
2954	the known length of the string. Return NULL_TREE if no simplification
2955	was possible. */
2956
2957	static bool
2958	gimple_fold_builtin_fputs (gimple_stmt_iterator *gsi,
2959	tree arg0, tree arg1,
2960	bool unlocked)
2961	{
2962	gimple *stmt = gsi_stmt (i: *gsi);
2963
2964	/* If we're using an unlocked function, assume the other unlocked
2965	functions exist explicitly. */
2966	tree const fn_fputc = (unlocked
2967	? builtin_decl_explicit (fncode: BUILT_IN_FPUTC_UNLOCKED)
2968	: builtin_decl_implicit (fncode: BUILT_IN_FPUTC));
2969	tree const fn_fwrite = (unlocked
2970	? builtin_decl_explicit (fncode: BUILT_IN_FWRITE_UNLOCKED)
2971	: builtin_decl_implicit (fncode: BUILT_IN_FWRITE));
2972
2973	/* If the return value is used, don't do the transformation. */
2974	if (gimple_call_lhs (gs: stmt))
2975	return false;
2976
2977	/* Get the length of the string passed to fputs. If the length
2978	can't be determined, punt. */
2979	tree len = get_maxval_strlen (arg: arg0, rkind: SRK_STRLEN);
2980	if (!len || TREE_CODE (len) != INTEGER_CST)
2981	return false;
2982
2983	switch (compare_tree_int (len, 1))
2984	{
2985	case -1: /* length is 0, delete the call entirely . */
2986	replace_call_with_value (gsi, integer_zero_node);
2987	return true;
2988
2989	case 0: /* length is 1, call fputc. */
2990	{
2991	const char *p = c_getstr (arg0);
2992	if (p != NULL)
2993	{
2994	if (!fn_fputc || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
2995	return false;
2996
2997	gimple *repl
2998	= gimple_build_call (fn_fputc, 2,
2999	build_int_cst (integer_type_node, p[0]),
3000	arg1);
3001	replace_call_with_call_and_fold (gsi, repl);
3002	return true;
3003	}
3004	}
3005	/* FALLTHROUGH */
3006	case 1: /* length is greater than 1, call fwrite. */
3007	{
3008	/* If optimizing for size keep fputs. */
3009	if (optimize_function_for_size_p (cfun))
3010	return false;
3011	/* New argument list transforming fputs(string, stream) to
3012	fwrite(string, 1, len, stream). */
3013	if (!fn_fwrite || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
3014	return false;
3015
3016	gimple *repl
3017	= gimple_build_call (fn_fwrite, 4, arg0, size_one_node,
3018	fold_convert (size_type_node, len), arg1);
3019	replace_call_with_call_and_fold (gsi, repl);
3020	return true;
3021	}
3022	default:
3023	gcc_unreachable ();
3024	}
3025	}
3026
3027	/* Fold a call to the __mem{cpy,pcpy,move,set}_chk builtin.
3028	DEST, SRC, LEN, and SIZE are the arguments to the call.
3029	IGNORE is true, if return value can be ignored. FCODE is the BUILT_IN_*
3030	code of the builtin. If MAXLEN is not NULL, it is maximum length
3031	passed as third argument. */
3032
3033	static bool
3034	gimple_fold_builtin_memory_chk (gimple_stmt_iterator *gsi,
3035	tree dest, tree src, tree len, tree size,
3036	enum built_in_function fcode)
3037	{
3038	gimple *stmt = gsi_stmt (i: *gsi);
3039	location_t loc = gimple_location (g: stmt);
3040	bool ignore = gimple_call_lhs (gs: stmt) == NULL_TREE;
3041	tree fn;
3042
3043	/* If SRC and DEST are the same (and not volatile), return DEST
3044	(resp. DEST+LEN for __mempcpy_chk). */
3045	if (fcode != BUILT_IN_MEMSET_CHK && operand_equal_p (src, dest, flags: 0))
3046	{
3047	if (fcode != BUILT_IN_MEMPCPY_CHK)
3048	{
3049	replace_call_with_value (gsi, val: dest);
3050	return true;
3051	}
3052	else
3053	{
3054	gimple_seq stmts = NULL;
3055	len = gimple_convert_to_ptrofftype (seq: &stmts, loc, op: len);
3056	tree temp = gimple_build (seq: &stmts, loc, code: POINTER_PLUS_EXPR,
3057	TREE_TYPE (dest), ops: dest, ops: len);
3058	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
3059	replace_call_with_value (gsi, val: temp);
3060	return true;
3061	}
3062	}
3063
3064	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
3065	return false;
3066
3067	tree maxlen = get_maxval_strlen (arg: len, rkind: SRK_INT_VALUE);
3068	if (! integer_all_onesp (size)
3069	&& !known_lower (stmt, len, size)
3070	&& !known_lower (stmt, len: maxlen, size))
3071	{
3072	/* MAXLEN and LEN both cannot be proved to be less than SIZE, at
3073	least try to optimize (void) __mempcpy_chk () into
3074	(void) __memcpy_chk () */
3075	if (fcode == BUILT_IN_MEMPCPY_CHK && ignore)
3076	{
3077	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMCPY_CHK);
3078	if (!fn)
3079	return false;
3080
3081	gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
3082	replace_call_with_call_and_fold (gsi, repl);
3083	return true;
3084	}
3085	return false;
3086	}
3087
3088	fn = NULL_TREE;
3089	/* If __builtin_mem{cpy,pcpy,move,set}_chk is used, assume
3090	mem{cpy,pcpy,move,set} is available. */
3091	switch (fcode)
3092	{
3093	case BUILT_IN_MEMCPY_CHK:
3094	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMCPY);
3095	break;
3096	case BUILT_IN_MEMPCPY_CHK:
3097	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMPCPY);
3098	break;
3099	case BUILT_IN_MEMMOVE_CHK:
3100	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMMOVE);
3101	break;
3102	case BUILT_IN_MEMSET_CHK:
3103	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMSET);
3104	break;
3105	default:
3106	break;
3107	}
3108
3109	if (!fn)
3110	return false;
3111
3112	gimple *repl = gimple_build_call (fn, 3, dest, src, len);
3113	replace_call_with_call_and_fold (gsi, repl);
3114	return true;
3115	}
3116
3117	/* Fold a call to the __st[rp]cpy_chk builtin.
3118	DEST, SRC, and SIZE are the arguments to the call.
3119	IGNORE is true if return value can be ignored. FCODE is the BUILT_IN_*
3120	code of the builtin. If MAXLEN is not NULL, it is maximum length of
3121	strings passed as second argument. */
3122
3123	static bool
3124	gimple_fold_builtin_stxcpy_chk (gimple_stmt_iterator *gsi,
3125	tree dest,
3126	tree src, tree size,
3127	enum built_in_function fcode)
3128	{
3129	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3130	location_t loc = gimple_location (g: stmt);
3131	bool ignore = gimple_call_lhs (gs: stmt) == NULL_TREE;
3132	tree len, fn;
3133
3134	/* If SRC and DEST are the same (and not volatile), return DEST. */
3135	if (fcode == BUILT_IN_STRCPY_CHK && operand_equal_p (src, dest, flags: 0))
3136	{
3137	/* Issue -Wrestrict unless the pointers are null (those do
3138	not point to objects and so do not indicate an overlap;
3139	such calls could be the result of sanitization and jump
3140	threading). */
3141	if (!integer_zerop (dest)
3142	&& !warning_suppressed_p (stmt, OPT_Wrestrict))
3143	{
3144	tree func = gimple_call_fndecl (gs: stmt);
3145
3146	warning_at (loc, OPT_Wrestrict,
3147	"%qD source argument is the same as destination",
3148	func);
3149	}
3150
3151	replace_call_with_value (gsi, val: dest);
3152	return true;
3153	}
3154
3155	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
3156	return false;
3157
3158	tree maxlen = get_maxval_strlen (arg: src, rkind: SRK_STRLENMAX);
3159	if (! integer_all_onesp (size))
3160	{
3161	len = c_strlen (src, 1);
3162	if (!known_lower (stmt, len, size, strict: true)
3163	&& !known_lower (stmt, len: maxlen, size, strict: true))
3164	{
3165	if (fcode == BUILT_IN_STPCPY_CHK)
3166	{
3167	if (! ignore)
3168	return false;
3169
3170	/* If return value of __stpcpy_chk is ignored,
3171	optimize into __strcpy_chk. */
3172	fn = builtin_decl_explicit (fncode: BUILT_IN_STRCPY_CHK);
3173	if (!fn)
3174	return false;
3175
3176	gimple *repl = gimple_build_call (fn, 3, dest, src, size);
3177	replace_call_with_call_and_fold (gsi, repl);
3178	return true;
3179	}
3180
3181	if (! len || TREE_SIDE_EFFECTS (len))
3182	return false;
3183
3184	/* If c_strlen returned something, but not provably less than size,
3185	transform __strcpy_chk into __memcpy_chk. */
3186	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMCPY_CHK);
3187	if (!fn)
3188	return false;
3189
3190	gimple_seq stmts = NULL;
3191	len = force_gimple_operand (len, &stmts, true, NULL_TREE);
3192	len = gimple_convert (seq: &stmts, loc, size_type_node, op: len);
3193	len = gimple_build (seq: &stmts, loc, code: PLUS_EXPR, size_type_node, ops: len,
3194	ops: build_int_cst (size_type_node, 1));
3195	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
3196	gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
3197	replace_call_with_call_and_fold (gsi, repl);
3198	return true;
3199	}
3200	}
3201
3202	/* If __builtin_st{r,p}cpy_chk is used, assume st{r,p}cpy is available. */
3203	fn = builtin_decl_explicit (fncode: fcode == BUILT_IN_STPCPY_CHK && !ignore
3204	? BUILT_IN_STPCPY : BUILT_IN_STRCPY);
3205	if (!fn)
3206	return false;
3207
3208	gcall *repl = gimple_build_call (fn, 2, dest, src);
3209	replace_call_with_call_and_fold (gsi, repl);
3210	return true;
3211	}
3212
3213	/* Fold a call to the __st{r,p}ncpy_chk builtin. DEST, SRC, LEN, and SIZE
3214	are the arguments to the call. If MAXLEN is not NULL, it is maximum
3215	length passed as third argument. IGNORE is true if return value can be
3216	ignored. FCODE is the BUILT_IN_* code of the builtin. */
3217
3218	static bool
3219	gimple_fold_builtin_stxncpy_chk (gimple_stmt_iterator *gsi,
3220	tree dest, tree src,
3221	tree len, tree size,
3222	enum built_in_function fcode)
3223	{
3224	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3225	bool ignore = gimple_call_lhs (gs: stmt) == NULL_TREE;
3226	tree fn;
3227
3228	tree maxlen = get_maxval_strlen (arg: len, rkind: SRK_INT_VALUE);
3229	if (! integer_all_onesp (size)
3230	&& !known_lower (stmt, len, size) && !known_lower (stmt, len: maxlen, size))
3231	{
3232	if (fcode == BUILT_IN_STPNCPY_CHK && ignore)
3233	{
3234	/* If return value of __stpncpy_chk is ignored,
3235	optimize into __strncpy_chk. */
3236	fn = builtin_decl_explicit (fncode: BUILT_IN_STRNCPY_CHK);
3237	if (fn)
3238	{
3239	gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
3240	replace_call_with_call_and_fold (gsi, repl);
3241	return true;
3242	}
3243	}
3244	return false;
3245	}
3246
3247	/* If __builtin_st{r,p}ncpy_chk is used, assume st{r,p}ncpy is available. */
3248	fn = builtin_decl_explicit (fncode: fcode == BUILT_IN_STPNCPY_CHK && !ignore
3249	? BUILT_IN_STPNCPY : BUILT_IN_STRNCPY);
3250	if (!fn || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
3251	return false;
3252
3253	gcall *repl = gimple_build_call (fn, 3, dest, src, len);
3254	replace_call_with_call_and_fold (gsi, repl);
3255	return true;
3256	}
3257
3258	/* Fold function call to builtin stpcpy with arguments DEST and SRC.
3259	Return NULL_TREE if no simplification can be made. */
3260
3261	static bool
3262	gimple_fold_builtin_stpcpy (gimple_stmt_iterator *gsi)
3263	{
3264	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3265	location_t loc = gimple_location (g: stmt);
3266	tree dest = gimple_call_arg (gs: stmt, index: 0);
3267	tree src = gimple_call_arg (gs: stmt, index: 1);
3268	tree fn, lenp1;
3269
3270	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
3271	return false;
3272
3273	/* If the result is unused, replace stpcpy with strcpy. */
3274	if (gimple_call_lhs (gs: stmt) == NULL_TREE)
3275	{
3276	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCPY);
3277	if (!fn)
3278	return false;
3279	gimple_call_set_fndecl (gs: stmt, decl: fn);
3280	fold_stmt (gsi);
3281	return true;
3282	}
3283
3284	/* Set to non-null if ARG refers to an unterminated array. */
3285	c_strlen_data data = { };
3286	/* The size of the unterminated array if SRC referes to one. */
3287	tree size;
3288	/* True if the size is exact/constant, false if it's the lower bound
3289	of a range. */
3290	bool exact;
3291	tree len = c_strlen (src, 1, &data, 1);
3292	if (!len
3293	|| TREE_CODE (len) != INTEGER_CST)
3294	{
3295	data.decl = unterminated_array (src, &size, &exact);
3296	if (!data.decl)
3297	return false;
3298	}
3299
3300	if (data.decl)
3301	{
3302	/* Avoid folding calls with unterminated arrays. */
3303	if (!warning_suppressed_p (stmt, OPT_Wstringop_overread))
3304	warn_string_no_nul (loc, stmt, "stpcpy", src, data.decl, size,
3305	exact);
3306	suppress_warning (stmt, OPT_Wstringop_overread);
3307	return false;
3308	}
3309
3310	if (optimize_function_for_size_p (cfun)
3311	/* If length is zero it's small enough. */
3312	&& !integer_zerop (len))
3313	return false;
3314
3315	/* If the source has a known length replace stpcpy with memcpy. */
3316	fn = builtin_decl_implicit (fncode: BUILT_IN_MEMCPY);
3317	if (!fn)
3318	return false;
3319
3320	gimple_seq stmts = NULL;
3321	tree tem = gimple_convert (seq: &stmts, loc, size_type_node, op: len);
3322	lenp1 = gimple_build (seq: &stmts, loc, code: PLUS_EXPR, size_type_node,
3323	ops: tem, ops: build_int_cst (size_type_node, 1));
3324	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
3325	gcall *repl = gimple_build_call (fn, 3, dest, src, lenp1);
3326	gimple_move_vops (repl, stmt);
3327	gsi_insert_before (gsi, repl, GSI_SAME_STMT);
3328	/* Replace the result with dest + len. */
3329	stmts = NULL;
3330	tem = gimple_convert (seq: &stmts, loc, sizetype, op: len);
3331	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
3332	gassign *ret = gimple_build_assign (gimple_call_lhs (gs: stmt),
3333	POINTER_PLUS_EXPR, dest, tem);
3334	gsi_replace (gsi, ret, false);
3335	/* Finally fold the memcpy call. */
3336	gimple_stmt_iterator gsi2 = *gsi;
3337	gsi_prev (i: &gsi2);
3338	fold_stmt (&gsi2);
3339	return true;
3340	}
3341
3342	/* Fold a call EXP to {,v}snprintf having NARGS passed as ARGS. Return
3343	NULL_TREE if a normal call should be emitted rather than expanding
3344	the function inline. FCODE is either BUILT_IN_SNPRINTF_CHK or
3345	BUILT_IN_VSNPRINTF_CHK. If MAXLEN is not NULL, it is maximum length
3346	passed as second argument. */
3347
3348	static bool
3349	gimple_fold_builtin_snprintf_chk (gimple_stmt_iterator *gsi,
3350	enum built_in_function fcode)
3351	{
3352	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3353	tree dest, size, len, fn, fmt, flag;
3354	const char *fmt_str;
3355
3356	/* Verify the required arguments in the original call. */
3357	if (gimple_call_num_args (gs: stmt) < 5)
3358	return false;
3359
3360	dest = gimple_call_arg (gs: stmt, index: 0);
3361	len = gimple_call_arg (gs: stmt, index: 1);
3362	flag = gimple_call_arg (gs: stmt, index: 2);
3363	size = gimple_call_arg (gs: stmt, index: 3);
3364	fmt = gimple_call_arg (gs: stmt, index: 4);
3365
3366	tree maxlen = get_maxval_strlen (arg: len, rkind: SRK_INT_VALUE);
3367	if (! integer_all_onesp (size)
3368	&& !known_lower (stmt, len, size) && !known_lower (stmt, len: maxlen, size))
3369	return false;
3370
3371	if (!init_target_chars ())
3372	return false;
3373
3374	/* Only convert __{,v}snprintf_chk to {,v}snprintf if flag is 0
3375	or if format doesn't contain % chars or is "%s". */
3376	if (! integer_zerop (flag))
3377	{
3378	fmt_str = c_getstr (fmt);
3379	if (fmt_str == NULL)
3380	return false;
3381	if (strchr (s: fmt_str, c: target_percent) != NULL
3382	&& strcmp (s1: fmt_str, s2: target_percent_s))
3383	return false;
3384	}
3385
3386	/* If __builtin_{,v}snprintf_chk is used, assume {,v}snprintf is
3387	available. */
3388	fn = builtin_decl_explicit (fncode: fcode == BUILT_IN_VSNPRINTF_CHK
3389	? BUILT_IN_VSNPRINTF : BUILT_IN_SNPRINTF);
3390	if (!fn || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
3391	return false;
3392
3393	/* Replace the called function and the first 5 argument by 3 retaining
3394	trailing varargs. */
3395	gimple_call_set_fndecl (gs: stmt, decl: fn);
3396	gimple_call_set_fntype (call_stmt: stmt, TREE_TYPE (fn));
3397	gimple_call_set_arg (gs: stmt, index: 0, arg: dest);
3398	gimple_call_set_arg (gs: stmt, index: 1, arg: len);
3399	gimple_call_set_arg (gs: stmt, index: 2, arg: fmt);
3400	for (unsigned i = 3; i < gimple_call_num_args (gs: stmt) - 2; ++i)
3401	gimple_call_set_arg (gs: stmt, index: i, arg: gimple_call_arg (gs: stmt, index: i + 2));
3402	gimple_set_num_ops (gs: stmt, num_ops: gimple_num_ops (gs: stmt) - 2);
3403	fold_stmt (gsi);
3404	return true;
3405	}
3406
3407	/* Fold a call EXP to __{,v}sprintf_chk having NARGS passed as ARGS.
3408	Return NULL_TREE if a normal call should be emitted rather than
3409	expanding the function inline. FCODE is either BUILT_IN_SPRINTF_CHK
3410	or BUILT_IN_VSPRINTF_CHK. */
3411
3412	static bool
3413	gimple_fold_builtin_sprintf_chk (gimple_stmt_iterator *gsi,
3414	enum built_in_function fcode)
3415	{
3416	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3417	tree dest, size, len, fn, fmt, flag;
3418	const char *fmt_str;
3419	unsigned nargs = gimple_call_num_args (gs: stmt);
3420
3421	/* Verify the required arguments in the original call. */
3422	if (nargs < 4)
3423	return false;
3424	dest = gimple_call_arg (gs: stmt, index: 0);
3425	flag = gimple_call_arg (gs: stmt, index: 1);
3426	size = gimple_call_arg (gs: stmt, index: 2);
3427	fmt = gimple_call_arg (gs: stmt, index: 3);
3428
3429	len = NULL_TREE;
3430
3431	if (!init_target_chars ())
3432	return false;
3433
3434	/* Check whether the format is a literal string constant. */
3435	fmt_str = c_getstr (fmt);
3436	if (fmt_str != NULL)
3437	{
3438	/* If the format doesn't contain % args or %%, we know the size. */
3439	if (strchr (s: fmt_str, c: target_percent) == 0)
3440	{
3441	if (fcode != BUILT_IN_SPRINTF_CHK || nargs == 4)
3442	len = build_int_cstu (size_type_node, strlen (s: fmt_str));
3443	}
3444	/* If the format is "%s" and first ... argument is a string literal,
3445	we know the size too. */
3446	else if (fcode == BUILT_IN_SPRINTF_CHK
3447	&& strcmp (s1: fmt_str, s2: target_percent_s) == 0)
3448	{
3449	tree arg;
3450
3451	if (nargs == 5)
3452	{
3453	arg = gimple_call_arg (gs: stmt, index: 4);
3454	if (POINTER_TYPE_P (TREE_TYPE (arg)))
3455	len = c_strlen (arg, 1);
3456	}
3457	}
3458	}
3459
3460	if (! integer_all_onesp (size) && !known_lower (stmt, len, size, strict: true))
3461	return false;
3462
3463	/* Only convert __{,v}sprintf_chk to {,v}sprintf if flag is 0
3464	or if format doesn't contain % chars or is "%s". */
3465	if (! integer_zerop (flag))
3466	{
3467	if (fmt_str == NULL)
3468	return false;
3469	if (strchr (s: fmt_str, c: target_percent) != NULL
3470	&& strcmp (s1: fmt_str, s2: target_percent_s))
3471	return false;
3472	}
3473
3474	/* If __builtin_{,v}sprintf_chk is used, assume {,v}sprintf is available. */
3475	fn = builtin_decl_explicit (fncode: fcode == BUILT_IN_VSPRINTF_CHK
3476	? BUILT_IN_VSPRINTF : BUILT_IN_SPRINTF);
3477	if (!fn || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
3478	return false;
3479
3480	/* Replace the called function and the first 4 argument by 2 retaining
3481	trailing varargs. */
3482	gimple_call_set_fndecl (gs: stmt, decl: fn);
3483	gimple_call_set_fntype (call_stmt: stmt, TREE_TYPE (fn));
3484	gimple_call_set_arg (gs: stmt, index: 0, arg: dest);
3485	gimple_call_set_arg (gs: stmt, index: 1, arg: fmt);
3486	for (unsigned i = 2; i < gimple_call_num_args (gs: stmt) - 2; ++i)
3487	gimple_call_set_arg (gs: stmt, index: i, arg: gimple_call_arg (gs: stmt, index: i + 2));
3488	gimple_set_num_ops (gs: stmt, num_ops: gimple_num_ops (gs: stmt) - 2);
3489	fold_stmt (gsi);
3490	return true;
3491	}
3492
3493	/* Simplify a call to the sprintf builtin with arguments DEST, FMT, and ORIG.
3494	ORIG may be null if this is a 2-argument call. We don't attempt to
3495	simplify calls with more than 3 arguments.
3496
3497	Return true if simplification was possible, otherwise false. */
3498
3499	bool
3500	gimple_fold_builtin_sprintf (gimple_stmt_iterator *gsi)
3501	{
3502	gimple *stmt = gsi_stmt (i: *gsi);
3503
3504	/* Verify the required arguments in the original call. We deal with two
3505	types of sprintf() calls: 'sprintf (str, fmt)' and
3506	'sprintf (dest, "%s", orig)'. */
3507	if (gimple_call_num_args (gs: stmt) > 3)
3508	return false;
3509
3510	tree orig = NULL_TREE;
3511	if (gimple_call_num_args (gs: stmt) == 3)
3512	orig = gimple_call_arg (gs: stmt, index: 2);
3513
3514	/* Check whether the format is a literal string constant. */
3515	tree fmt = gimple_call_arg (gs: stmt, index: 1);
3516	const char *fmt_str = c_getstr (fmt);
3517	if (fmt_str == NULL)
3518	return false;
3519
3520	tree dest = gimple_call_arg (gs: stmt, index: 0);
3521
3522	if (!init_target_chars ())
3523	return false;
3524
3525	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCPY);
3526	if (!fn || (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
3527	return false;
3528
3529	/* If the format doesn't contain % args or %%, use strcpy. */
3530	if (strchr (s: fmt_str, c: target_percent) == NULL)
3531	{
3532	/* Don't optimize sprintf (buf, "abc", ptr++). */
3533	if (orig)
3534	return false;
3535
3536	/* Convert sprintf (str, fmt) into strcpy (str, fmt) when
3537	'format' is known to contain no % formats. */
3538	gimple_seq stmts = NULL;
3539	gimple *repl = gimple_build_call (fn, 2, dest, fmt);
3540
3541	/* Propagate the NO_WARNING bit to avoid issuing the same
3542	warning more than once. */
3543	copy_warning (repl, stmt);
3544
3545	gimple_seq_add_stmt_without_update (&stmts, repl);
3546	if (tree lhs = gimple_call_lhs (gs: stmt))
3547	{
3548	repl = gimple_build_assign (lhs, build_int_cst (TREE_TYPE (lhs),
3549	strlen (s: fmt_str)));
3550	gimple_seq_add_stmt_without_update (&stmts, repl);
3551	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3552	/* gsi now points at the assignment to the lhs, get a
3553	stmt iterator to the memcpy call.
3554	??? We can't use gsi_for_stmt as that doesn't work when the
3555	CFG isn't built yet. */
3556	gimple_stmt_iterator gsi2 = *gsi;
3557	gsi_prev (i: &gsi2);
3558	fold_stmt (&gsi2);
3559	}
3560	else
3561	{
3562	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3563	fold_stmt (gsi);
3564	}
3565	return true;
3566	}
3567
3568	/* If the format is "%s", use strcpy if the result isn't used. */
3569	else if (fmt_str && strcmp (s1: fmt_str, s2: target_percent_s) == 0)
3570	{
3571	/* Don't crash on sprintf (str1, "%s"). */
3572	if (!orig)
3573	return false;
3574
3575	/* Don't fold calls with source arguments of invalid (nonpointer)
3576	types. */
3577	if (!POINTER_TYPE_P (TREE_TYPE (orig)))
3578	return false;
3579
3580	tree orig_len = NULL_TREE;
3581	if (gimple_call_lhs (gs: stmt))
3582	{
3583	orig_len = get_maxval_strlen (arg: orig, rkind: SRK_STRLEN);
3584	if (!orig_len)
3585	return false;
3586	}
3587
3588	/* Convert sprintf (str1, "%s", str2) into strcpy (str1, str2). */
3589	gimple_seq stmts = NULL;
3590	gimple *repl = gimple_build_call (fn, 2, dest, orig);
3591
3592	/* Propagate the NO_WARNING bit to avoid issuing the same
3593	warning more than once. */
3594	copy_warning (repl, stmt);
3595
3596	gimple_seq_add_stmt_without_update (&stmts, repl);
3597	if (tree lhs = gimple_call_lhs (gs: stmt))
3598	{
3599	if (!useless_type_conversion_p (TREE_TYPE (lhs),
3600	TREE_TYPE (orig_len)))
3601	orig_len = fold_convert (TREE_TYPE (lhs), orig_len);
3602	repl = gimple_build_assign (lhs, orig_len);
3603	gimple_seq_add_stmt_without_update (&stmts, repl);
3604	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3605	/* gsi now points at the assignment to the lhs, get a
3606	stmt iterator to the memcpy call.
3607	??? We can't use gsi_for_stmt as that doesn't work when the
3608	CFG isn't built yet. */
3609	gimple_stmt_iterator gsi2 = *gsi;
3610	gsi_prev (i: &gsi2);
3611	fold_stmt (&gsi2);
3612	}
3613	else
3614	{
3615	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3616	fold_stmt (gsi);
3617	}
3618	return true;
3619	}
3620	return false;
3621	}
3622
3623	/* Simplify a call to the snprintf builtin with arguments DEST, DESTSIZE,
3624	FMT, and ORIG. ORIG may be null if this is a 3-argument call. We don't
3625	attempt to simplify calls with more than 4 arguments.
3626
3627	Return true if simplification was possible, otherwise false. */
3628
3629	bool
3630	gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
3631	{
3632	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3633	tree dest = gimple_call_arg (gs: stmt, index: 0);
3634	tree destsize = gimple_call_arg (gs: stmt, index: 1);
3635	tree fmt = gimple_call_arg (gs: stmt, index: 2);
3636	tree orig = NULL_TREE;
3637	const char *fmt_str = NULL;
3638
3639	if (gimple_call_num_args (gs: stmt) > 4
3640	|| (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun)))
3641	return false;
3642
3643	if (gimple_call_num_args (gs: stmt) == 4)
3644	orig = gimple_call_arg (gs: stmt, index: 3);
3645
3646	/* Check whether the format is a literal string constant. */
3647	fmt_str = c_getstr (fmt);
3648	if (fmt_str == NULL)
3649	return false;
3650
3651	if (!init_target_chars ())
3652	return false;
3653
3654	/* If the format doesn't contain % args or %%, use strcpy. */
3655	if (strchr (s: fmt_str, c: target_percent) == NULL)
3656	{
3657	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCPY);
3658	if (!fn)
3659	return false;
3660
3661	/* Don't optimize snprintf (buf, 4, "abc", ptr++). */
3662	if (orig)
3663	return false;
3664
3665	tree len = build_int_cstu (TREE_TYPE (destsize), strlen (s: fmt_str));
3666
3667	/* We could expand this as
3668	memcpy (str, fmt, cst - 1); str[cst - 1] = '\0';
3669	or to
3670	memcpy (str, fmt_with_nul_at_cstm1, cst);
3671	but in the former case that might increase code size
3672	and in the latter case grow .rodata section too much.
3673	So punt for now. */
3674	if (!known_lower (stmt, len, size: destsize, strict: true))
3675	return false;
3676
3677	gimple_seq stmts = NULL;
3678	gimple *repl = gimple_build_call (fn, 2, dest, fmt);
3679	gimple_seq_add_stmt_without_update (&stmts, repl);
3680	if (tree lhs = gimple_call_lhs (gs: stmt))
3681	{
3682	repl = gimple_build_assign (lhs,
3683	fold_convert (TREE_TYPE (lhs), len));
3684	gimple_seq_add_stmt_without_update (&stmts, repl);
3685	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3686	/* gsi now points at the assignment to the lhs, get a
3687	stmt iterator to the memcpy call.
3688	??? We can't use gsi_for_stmt as that doesn't work when the
3689	CFG isn't built yet. */
3690	gimple_stmt_iterator gsi2 = *gsi;
3691	gsi_prev (i: &gsi2);
3692	fold_stmt (&gsi2);
3693	}
3694	else
3695	{
3696	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3697	fold_stmt (gsi);
3698	}
3699	return true;
3700	}
3701
3702	/* If the format is "%s", use strcpy if the result isn't used. */
3703	else if (fmt_str && strcmp (s1: fmt_str, s2: target_percent_s) == 0)
3704	{
3705	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCPY);
3706	if (!fn)
3707	return false;
3708
3709	/* Don't crash on snprintf (str1, cst, "%s"). */
3710	if (!orig)
3711	return false;
3712
3713	tree orig_len = get_maxval_strlen (arg: orig, rkind: SRK_STRLEN);
3714
3715	/* We could expand this as
3716	memcpy (str1, str2, cst - 1); str1[cst - 1] = '\0';
3717	or to
3718	memcpy (str1, str2_with_nul_at_cstm1, cst);
3719	but in the former case that might increase code size
3720	and in the latter case grow .rodata section too much.
3721	So punt for now. */
3722	if (!known_lower (stmt, len: orig_len, size: destsize, strict: true))
3723	return false;
3724
3725	/* Convert snprintf (str1, cst, "%s", str2) into
3726	strcpy (str1, str2) if strlen (str2) < cst. */
3727	gimple_seq stmts = NULL;
3728	gimple *repl = gimple_build_call (fn, 2, dest, orig);
3729	gimple_seq_add_stmt_without_update (&stmts, repl);
3730	if (tree lhs = gimple_call_lhs (gs: stmt))
3731	{
3732	if (!useless_type_conversion_p (TREE_TYPE (lhs),
3733	TREE_TYPE (orig_len)))
3734	orig_len = fold_convert (TREE_TYPE (lhs), orig_len);
3735	repl = gimple_build_assign (lhs, orig_len);
3736	gimple_seq_add_stmt_without_update (&stmts, repl);
3737	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3738	/* gsi now points at the assignment to the lhs, get a
3739	stmt iterator to the memcpy call.
3740	??? We can't use gsi_for_stmt as that doesn't work when the
3741	CFG isn't built yet. */
3742	gimple_stmt_iterator gsi2 = *gsi;
3743	gsi_prev (i: &gsi2);
3744	fold_stmt (&gsi2);
3745	}
3746	else
3747	{
3748	gsi_replace_with_seq_vops (si_p: gsi, stmts);
3749	fold_stmt (gsi);
3750	}
3751	return true;
3752	}
3753	return false;
3754	}
3755
3756	/* Fold a call to the {,v}fprintf{,_unlocked} and __{,v}printf_chk builtins.
3757	FP, FMT, and ARG are the arguments to the call. We don't fold calls with
3758	more than 3 arguments, and ARG may be null in the 2-argument case.
3759
3760	Return NULL_TREE if no simplification was possible, otherwise return the
3761	simplified form of the call as a tree. FCODE is the BUILT_IN_*
3762	code of the function to be simplified. */
3763
3764	static bool
3765	gimple_fold_builtin_fprintf (gimple_stmt_iterator *gsi,
3766	tree fp, tree fmt, tree arg,
3767	enum built_in_function fcode)
3768	{
3769	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3770	tree fn_fputc, fn_fputs;
3771	const char *fmt_str = NULL;
3772
3773	/* If the return value is used, don't do the transformation. */
3774	if (gimple_call_lhs (gs: stmt) != NULL_TREE)
3775	return false;
3776
3777	/* Check whether the format is a literal string constant. */
3778	fmt_str = c_getstr (fmt);
3779	if (fmt_str == NULL)
3780	return false;
3781
3782	if (fcode == BUILT_IN_FPRINTF_UNLOCKED)
3783	{
3784	/* If we're using an unlocked function, assume the other
3785	unlocked functions exist explicitly. */
3786	fn_fputc = builtin_decl_explicit (fncode: BUILT_IN_FPUTC_UNLOCKED);
3787	fn_fputs = builtin_decl_explicit (fncode: BUILT_IN_FPUTS_UNLOCKED);
3788	}
3789	else
3790	{
3791	fn_fputc = builtin_decl_implicit (fncode: BUILT_IN_FPUTC);
3792	fn_fputs = builtin_decl_implicit (fncode: BUILT_IN_FPUTS);
3793	}
3794
3795	if (!init_target_chars ())
3796	return false;
3797
3798	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
3799	return false;
3800
3801	/* If the format doesn't contain % args or %%, use strcpy. */
3802	if (strchr (s: fmt_str, c: target_percent) == NULL)
3803	{
3804	if (fcode != BUILT_IN_VFPRINTF && fcode != BUILT_IN_VFPRINTF_CHK
3805	&& arg)
3806	return false;
3807
3808	/* If the format specifier was "", fprintf does nothing. */
3809	if (fmt_str[0] == '\0')
3810	{
3811	replace_call_with_value (gsi, NULL_TREE);
3812	return true;
3813	}
3814
3815	/* When "string" doesn't contain %, replace all cases of
3816	fprintf (fp, string) with fputs (string, fp). The fputs
3817	builtin will take care of special cases like length == 1. */
3818	if (fn_fputs)
3819	{
3820	gcall *repl = gimple_build_call (fn_fputs, 2, fmt, fp);
3821	replace_call_with_call_and_fold (gsi, repl);
3822	return true;
3823	}
3824	}
3825
3826	/* The other optimizations can be done only on the non-va_list variants. */
3827	else if (fcode == BUILT_IN_VFPRINTF || fcode == BUILT_IN_VFPRINTF_CHK)
3828	return false;
3829
3830	/* If the format specifier was "%s", call __builtin_fputs (arg, fp). */
3831	else if (strcmp (s1: fmt_str, s2: target_percent_s) == 0)
3832	{
3833	if (!arg || ! POINTER_TYPE_P (TREE_TYPE (arg)))
3834	return false;
3835	if (fn_fputs)
3836	{
3837	gcall *repl = gimple_build_call (fn_fputs, 2, arg, fp);
3838	replace_call_with_call_and_fold (gsi, repl);
3839	return true;
3840	}
3841	}
3842
3843	/* If the format specifier was "%c", call __builtin_fputc (arg, fp). */
3844	else if (strcmp (s1: fmt_str, s2: target_percent_c) == 0)
3845	{
3846	if (!arg
3847	|| ! useless_type_conversion_p (integer_type_node, TREE_TYPE (arg)))
3848	return false;
3849	if (fn_fputc)
3850	{
3851	gcall *repl = gimple_build_call (fn_fputc, 2, arg, fp);
3852	replace_call_with_call_and_fold (gsi, repl);
3853	return true;
3854	}
3855	}
3856
3857	return false;
3858	}
3859
3860	/* Fold a call to the {,v}printf{,_unlocked} and __{,v}printf_chk builtins.
3861	FMT and ARG are the arguments to the call; we don't fold cases with
3862	more than 2 arguments, and ARG may be null if this is a 1-argument case.
3863
3864	Return NULL_TREE if no simplification was possible, otherwise return the
3865	simplified form of the call as a tree. FCODE is the BUILT_IN_*
3866	code of the function to be simplified. */
3867
3868	static bool
3869	gimple_fold_builtin_printf (gimple_stmt_iterator *gsi, tree fmt,
3870	tree arg, enum built_in_function fcode)
3871	{
3872	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
3873	tree fn_putchar, fn_puts, newarg;
3874	const char *fmt_str = NULL;
3875
3876	/* If the return value is used, don't do the transformation. */
3877	if (gimple_call_lhs (gs: stmt) != NULL_TREE)
3878	return false;
3879
3880	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
3881	return false;
3882
3883	/* Check whether the format is a literal string constant. */
3884	fmt_str = c_getstr (fmt);
3885	if (fmt_str == NULL)
3886	return false;
3887
3888	if (fcode == BUILT_IN_PRINTF_UNLOCKED)
3889	{
3890	/* If we're using an unlocked function, assume the other
3891	unlocked functions exist explicitly. */
3892	fn_putchar = builtin_decl_explicit (fncode: BUILT_IN_PUTCHAR_UNLOCKED);
3893	fn_puts = builtin_decl_explicit (fncode: BUILT_IN_PUTS_UNLOCKED);
3894	}
3895	else
3896	{
3897	fn_putchar = builtin_decl_implicit (fncode: BUILT_IN_PUTCHAR);
3898	fn_puts = builtin_decl_implicit (fncode: BUILT_IN_PUTS);
3899	}
3900
3901	if (!init_target_chars ())
3902	return false;
3903
3904	if (strcmp (s1: fmt_str, s2: target_percent_s) == 0
3905	|| strchr (s: fmt_str, c: target_percent) == NULL)
3906	{
3907	const char *str;
3908
3909	if (strcmp (s1: fmt_str, s2: target_percent_s) == 0)
3910	{
3911	if (fcode == BUILT_IN_VPRINTF || fcode == BUILT_IN_VPRINTF_CHK)
3912	return false;
3913
3914	if (!arg || ! POINTER_TYPE_P (TREE_TYPE (arg)))
3915	return false;
3916
3917	str = c_getstr (arg);
3918	if (str == NULL)
3919	return false;
3920	}
3921	else
3922	{
3923	/* The format specifier doesn't contain any '%' characters. */
3924	if (fcode != BUILT_IN_VPRINTF && fcode != BUILT_IN_VPRINTF_CHK
3925	&& arg)
3926	return false;
3927	str = fmt_str;
3928	}
3929
3930	/* If the string was "", printf does nothing. */
3931	if (str[0] == '\0')
3932	{
3933	replace_call_with_value (gsi, NULL_TREE);
3934	return true;
3935	}
3936
3937	/* If the string has length of 1, call putchar. */
3938	if (str[1] == '\0')
3939	{
3940	/* Given printf("c"), (where c is any one character,)
3941	convert "c"[0] to an int and pass that to the replacement
3942	function. */
3943	newarg = build_int_cst (integer_type_node, str[0]);
3944	if (fn_putchar)
3945	{
3946	gcall *repl = gimple_build_call (fn_putchar, 1, newarg);
3947	replace_call_with_call_and_fold (gsi, repl);
3948	return true;
3949	}
3950	}
3951	else
3952	{
3953	/* If the string was "string\n", call puts("string"). */
3954	size_t len = strlen (s: str);
3955	if ((unsigned char)str[len - 1] == target_newline
3956	&& (size_t) (int) len == len
3957	&& (int) len > 0)
3958	{
3959	char *newstr;
3960
3961	/* Create a NUL-terminated string that's one char shorter
3962	than the original, stripping off the trailing '\n'. */
3963	newstr = xstrdup (str);
3964	newstr[len - 1] = '\0';
3965	newarg = build_string_literal (len, newstr);
3966	free (ptr: newstr);
3967	if (fn_puts)
3968	{
3969	gcall *repl = gimple_build_call (fn_puts, 1, newarg);
3970	replace_call_with_call_and_fold (gsi, repl);
3971	return true;
3972	}
3973	}
3974	else
3975	/* We'd like to arrange to call fputs(string,stdout) here,
3976	but we need stdout and don't have a way to get it yet. */
3977	return false;
3978	}
3979	}
3980
3981	/* The other optimizations can be done only on the non-va_list variants. */
3982	else if (fcode == BUILT_IN_VPRINTF || fcode == BUILT_IN_VPRINTF_CHK)
3983	return false;
3984
3985	/* If the format specifier was "%s\n", call __builtin_puts(arg). */
3986	else if (strcmp (s1: fmt_str, s2: target_percent_s_newline) == 0)
3987	{
3988	if (!arg || ! POINTER_TYPE_P (TREE_TYPE (arg)))
3989	return false;
3990	if (fn_puts)
3991	{
3992	gcall *repl = gimple_build_call (fn_puts, 1, arg);
3993	replace_call_with_call_and_fold (gsi, repl);
3994	return true;
3995	}
3996	}
3997
3998	/* If the format specifier was "%c", call __builtin_putchar(arg). */
3999	else if (strcmp (s1: fmt_str, s2: target_percent_c) == 0)
4000	{
4001	if (!arg || ! useless_type_conversion_p (integer_type_node,
4002	TREE_TYPE (arg)))
4003	return false;
4004	if (fn_putchar)
4005	{
4006	gcall *repl = gimple_build_call (fn_putchar, 1, arg);
4007	replace_call_with_call_and_fold (gsi, repl);
4008	return true;
4009	}
4010	}
4011
4012	return false;
4013	}
4014
4015
4016
4017	/* Fold a call to __builtin_strlen with known length LEN. */
4018
4019	static bool
4020	gimple_fold_builtin_strlen (gimple_stmt_iterator *gsi)
4021	{
4022	gimple *stmt = gsi_stmt (i: *gsi);
4023	tree arg = gimple_call_arg (gs: stmt, index: 0);
4024
4025	wide_int minlen;
4026	wide_int maxlen;
4027
4028	c_strlen_data lendata = { };
4029	if (get_range_strlen (arg, pdata: &lendata, /* eltsize = */ 1)
4030	&& !lendata.decl
4031	&& lendata.minlen && TREE_CODE (lendata.minlen) == INTEGER_CST
4032	&& lendata.maxlen && TREE_CODE (lendata.maxlen) == INTEGER_CST)
4033	{
4034	/* The range of lengths refers to either a single constant
4035	string or to the longest and shortest constant string
4036	referenced by the argument of the strlen() call, or to
4037	the strings that can possibly be stored in the arrays
4038	the argument refers to. */
4039	minlen = wi::to_wide (t: lendata.minlen);
4040	maxlen = wi::to_wide (t: lendata.maxlen);
4041	}
4042	else
4043	{
4044	unsigned prec = TYPE_PRECISION (sizetype);
4045
4046	minlen = wi::shwi (val: 0, precision: prec);
4047	maxlen = wi::to_wide (t: max_object_size (), prec) - 2;
4048	}
4049
4050	/* For -fsanitize=address, don't optimize the upper bound of the
4051	length to be able to diagnose UB on non-zero terminated arrays. */
4052	if (sanitize_flags_p (flag: SANITIZE_ADDRESS))
4053	maxlen = wi::max_value (TYPE_PRECISION (sizetype), UNSIGNED);
4054
4055	if (minlen == maxlen)
4056	{
4057	/* Fold the strlen call to a constant. */
4058	tree type = TREE_TYPE (lendata.minlen);
4059	tree len = force_gimple_operand_gsi (gsi,
4060	wide_int_to_tree (type, cst: minlen),
4061	true, NULL, true, GSI_SAME_STMT);
4062	replace_call_with_value (gsi, val: len);
4063	return true;
4064	}
4065
4066	/* Set the strlen() range to [0, MAXLEN]. */
4067	if (tree lhs = gimple_call_lhs (gs: stmt))
4068	set_strlen_range (lhs, minlen, maxlen);
4069
4070	return false;
4071	}
4072
4073	static bool
4074	gimple_fold_builtin_omp_is_initial_device (gimple_stmt_iterator *gsi)
4075	{
4076	#if ACCEL_COMPILER
4077	replace_call_with_value (gsi, integer_zero_node);
4078	return true;
4079	#else
4080	if (!ENABLE_OFFLOADING || symtab->state == EXPANSION)
4081	{
4082	replace_call_with_value (gsi, integer_one_node);
4083	return true;
4084	}
4085	#endif
4086	return false;
4087	}
4088
4089	/* omp_get_initial_device was in OpenMP 5.0/5.1 explicitly and in
4090	5.0 implicitly the same as omp_get_num_devices; since 6.0 it is
4091	unspecified whether -1 or omp_get_num_devices() is returned. For
4092	better backward compatibility, use omp_get_num_devices() on the
4093	host - and -1 on the device (where the result is unspecified). */
4094
4095	static bool
4096	gimple_fold_builtin_omp_get_initial_device (gimple_stmt_iterator *gsi)
4097	{
4098	#if ACCEL_COMPILER
4099	replace_call_with_value (gsi, build_int_cst (integer_type_node, -1));
4100	#else
4101	if (!ENABLE_OFFLOADING)
4102	replace_call_with_value (gsi, integer_zero_node);
4103	else
4104	{
4105	tree fn = builtin_decl_explicit (fncode: BUILT_IN_OMP_GET_NUM_DEVICES);
4106	gcall *repl = gimple_build_call (fn, 0);
4107	replace_call_with_call_and_fold (gsi, repl);
4108	}
4109	#endif
4110	return true;
4111	}
4112
4113	static bool
4114	gimple_fold_builtin_omp_get_num_devices (gimple_stmt_iterator *gsi)
4115	{
4116	if (!ENABLE_OFFLOADING)
4117	{
4118	replace_call_with_value (gsi, integer_zero_node);
4119	return true;
4120	}
4121	return false;
4122	}
4123
4124	/* Fold a call to __builtin_acc_on_device. */
4125
4126	static bool
4127	gimple_fold_builtin_acc_on_device (gimple_stmt_iterator *gsi, tree arg0)
4128	{
4129	/* Defer folding until we know which compiler we're in. */
4130	if (symtab->state != EXPANSION)
4131	return false;
4132
4133	unsigned val_host = GOMP_DEVICE_HOST;
4134	unsigned val_dev = GOMP_DEVICE_NONE;
4135
4136	#ifdef ACCEL_COMPILER
4137	val_host = GOMP_DEVICE_NOT_HOST;
4138	val_dev = ACCEL_COMPILER_acc_device;
4139	#endif
4140
4141	location_t loc = gimple_location (g: gsi_stmt (i: *gsi));
4142
4143	tree host_eq = make_ssa_name (boolean_type_node);
4144	gimple *host_ass = gimple_build_assign
4145	(host_eq, EQ_EXPR, arg0, build_int_cst (TREE_TYPE (arg0), val_host));
4146	gimple_set_location (g: host_ass, location: loc);
4147	gsi_insert_before (gsi, host_ass, GSI_SAME_STMT);
4148
4149	tree dev_eq = make_ssa_name (boolean_type_node);
4150	gimple *dev_ass = gimple_build_assign
4151	(dev_eq, EQ_EXPR, arg0, build_int_cst (TREE_TYPE (arg0), val_dev));
4152	gimple_set_location (g: dev_ass, location: loc);
4153	gsi_insert_before (gsi, dev_ass, GSI_SAME_STMT);
4154
4155	tree result = make_ssa_name (boolean_type_node);
4156	gimple *result_ass = gimple_build_assign
4157	(result, BIT_IOR_EXPR, host_eq, dev_eq);
4158	gimple_set_location (g: result_ass, location: loc);
4159	gsi_insert_before (gsi, result_ass, GSI_SAME_STMT);
4160
4161	replace_call_with_value (gsi, val: result);
4162
4163	return true;
4164	}
4165
4166	/* Fold realloc (0, n) -> malloc (n). */
4167
4168	static bool
4169	gimple_fold_builtin_realloc (gimple_stmt_iterator *gsi)
4170	{
4171	gimple *stmt = gsi_stmt (i: *gsi);
4172	tree arg = gimple_call_arg (gs: stmt, index: 0);
4173	tree size = gimple_call_arg (gs: stmt, index: 1);
4174
4175	if (!gimple_vdef (g: stmt) && gimple_in_ssa_p (cfun))
4176	return false;
4177
4178	if (operand_equal_p (arg, null_pointer_node, flags: 0))
4179	{
4180	tree fn_malloc = builtin_decl_implicit (fncode: BUILT_IN_MALLOC);
4181	if (fn_malloc)
4182	{
4183	gcall *repl = gimple_build_call (fn_malloc, 1, size);
4184	replace_call_with_call_and_fold (gsi, repl);
4185	return true;
4186	}
4187	}
4188	return false;
4189	}
4190
4191	/* Number of bytes into which any type but aggregate, vector or
4192	_BitInt types should fit. */
4193	static constexpr size_t clear_padding_unit
4194	= MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT;
4195	/* Buffer size on which __builtin_clear_padding folding code works. */
4196	static const size_t clear_padding_buf_size = 32 * clear_padding_unit;
4197
4198	/* Data passed through __builtin_clear_padding folding. */
4199	struct clear_padding_struct {
4200	location_t loc;
4201	/* 0 during __builtin_clear_padding folding, nonzero during
4202	clear_type_padding_in_mask. In that case, instead of clearing the
4203	non-padding bits in union_ptr array clear the padding bits in there. */
4204	bool clear_in_mask;
4205	tree base;
4206	tree alias_type;
4207	gimple_stmt_iterator *gsi;
4208	/* Alignment of buf->base + 0. */
4209	unsigned align;
4210	/* Offset from buf->base. Should be always a multiple of UNITS_PER_WORD. */
4211	HOST_WIDE_INT off;
4212	/* Number of padding bytes before buf->off that don't have padding clear
4213	code emitted yet. */
4214	HOST_WIDE_INT padding_bytes;
4215	/* The size of the whole object. Never emit code to touch
4216	buf->base + buf->sz or following bytes. */
4217	HOST_WIDE_INT sz;
4218	/* Number of bytes recorded in buf->buf. */
4219	size_t size;
4220	/* When inside union, instead of emitting code we and bits inside of
4221	the union_ptr array. */
4222	unsigned char *union_ptr;
4223	/* Set bits mean padding bits that need to be cleared by the builtin. */
4224	unsigned char buf[clear_padding_buf_size + clear_padding_unit];
4225	};
4226
4227	/* Emit code to clear padding requested in BUF->buf - set bits
4228	in there stand for padding that should be cleared. FULL is true
4229	if everything from the buffer should be flushed, otherwise
4230	it can leave up to 2 * clear_padding_unit bytes for further
4231	processing. */
4232
4233	static void
4234	clear_padding_flush (clear_padding_struct *buf, bool full)
4235	{
4236	gcc_assert ((clear_padding_unit % UNITS_PER_WORD) == 0);
4237	if (!full && buf->size < 2 * clear_padding_unit)
4238	return;
4239	gcc_assert ((buf->off % UNITS_PER_WORD) == 0);
4240	size_t end = buf->size;
4241	if (!full)
4242	end = ((end - clear_padding_unit - 1) / clear_padding_unit
4243	* clear_padding_unit);
4244	size_t padding_bytes = buf->padding_bytes;
4245	if (buf->union_ptr)
4246	{
4247	if (buf->clear_in_mask)
4248	{
4249	/* During clear_type_padding_in_mask, clear the padding
4250	bits set in buf->buf in the buf->union_ptr mask. */
4251	for (size_t i = 0; i < end; i++)
4252	{
4253	if (buf->buf[i] == (unsigned char) ~0)
4254	padding_bytes++;
4255	else
4256	{
4257	memset (s: &buf->union_ptr[buf->off + i - padding_bytes],
4258	c: 0, n: padding_bytes);
4259	padding_bytes = 0;
4260	buf->union_ptr[buf->off + i] &= ~buf->buf[i];
4261	}
4262	}
4263	if (full)
4264	{
4265	memset (s: &buf->union_ptr[buf->off + end - padding_bytes],
4266	c: 0, n: padding_bytes);
4267	buf->off = 0;
4268	buf->size = 0;
4269	buf->padding_bytes = 0;
4270	}
4271	else
4272	{
4273	memmove (dest: buf->buf, src: buf->buf + end, n: buf->size - end);
4274	buf->off += end;
4275	buf->size -= end;
4276	buf->padding_bytes = padding_bytes;
4277	}
4278	return;
4279	}
4280	/* Inside of a union, instead of emitting any code, instead
4281	clear all bits in the union_ptr buffer that are clear
4282	in buf. Whole padding bytes don't clear anything. */
4283	for (size_t i = 0; i < end; i++)
4284	{
4285	if (buf->buf[i] == (unsigned char) ~0)
4286	padding_bytes++;
4287	else
4288	{
4289	padding_bytes = 0;
4290	buf->union_ptr[buf->off + i] &= buf->buf[i];
4291	}
4292	}
4293	if (full)
4294	{
4295	buf->off = 0;
4296	buf->size = 0;
4297	buf->padding_bytes = 0;
4298	}
4299	else
4300	{
4301	memmove (dest: buf->buf, src: buf->buf + end, n: buf->size - end);
4302	buf->off += end;
4303	buf->size -= end;
4304	buf->padding_bytes = padding_bytes;
4305	}
4306	return;
4307	}
4308	size_t wordsize = UNITS_PER_WORD;
4309	for (size_t i = 0; i < end; i += wordsize)
4310	{
4311	size_t nonzero_first = wordsize;
4312	size_t nonzero_last = 0;
4313	size_t zero_first = wordsize;
4314	size_t zero_last = 0;
4315	bool all_ones = true, bytes_only = true;
4316	if ((unsigned HOST_WIDE_INT) (buf->off + i + wordsize)
4317	> (unsigned HOST_WIDE_INT) buf->sz)
4318	{
4319	gcc_assert (wordsize > 1);
4320	wordsize /= 2;
4321	i -= wordsize;
4322	continue;
4323	}
4324	size_t endsize = end - i > wordsize ? wordsize : end - i;
4325	for (size_t j = i; j < i + endsize; j++)
4326	{
4327	if (buf->buf[j])
4328	{
4329	if (nonzero_first == wordsize)
4330	{
4331	nonzero_first = j - i;
4332	nonzero_last = j - i;
4333	}
4334	if (nonzero_last != j - i)
4335	all_ones = false;
4336	nonzero_last = j + 1 - i;
4337	}
4338	else
4339	{
4340	if (zero_first == wordsize)
4341	zero_first = j - i;
4342	zero_last = j + 1 - i;
4343	}
4344	if (buf->buf[j] != 0 && buf->buf[j] != (unsigned char) ~0)
4345	{
4346	all_ones = false;
4347	bytes_only = false;
4348	}
4349	}
4350	size_t padding_end = i;
4351	if (padding_bytes)
4352	{
4353	if (nonzero_first == 0
4354	&& nonzero_last == endsize
4355	&& all_ones)
4356	{
4357	/* All bits are padding and we had some padding
4358	before too. Just extend it. */
4359	padding_bytes += endsize;
4360	continue;
4361	}
4362	if (all_ones && nonzero_first == 0)
4363	{
4364	padding_bytes += nonzero_last;
4365	padding_end += nonzero_last;
4366	nonzero_first = wordsize;
4367	nonzero_last = 0;
4368	}
4369	else if (bytes_only && nonzero_first == 0)
4370	{
4371	gcc_assert (zero_first && zero_first != wordsize);
4372	padding_bytes += zero_first;
4373	padding_end += zero_first;
4374	}
4375	tree atype, src;
4376	if (padding_bytes == 1)
4377	{
4378	atype = char_type_node;
4379	src = build_zero_cst (char_type_node);
4380	}
4381	else
4382	{
4383	atype = build_array_type_nelts (char_type_node, padding_bytes);
4384	src = build_constructor (atype, NULL);
4385	}
4386	tree dst = build2_loc (loc: buf->loc, code: MEM_REF, type: atype, arg0: buf->base,
4387	arg1: build_int_cst (buf->alias_type,
4388	buf->off + padding_end
4389	- padding_bytes));
4390	gimple *g = gimple_build_assign (dst, src);
4391	gimple_set_location (g, location: buf->loc);
4392	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4393	padding_bytes = 0;
4394	buf->padding_bytes = 0;
4395	}
4396	if (nonzero_first == wordsize)
4397	/* All bits in a word are 0, there are no padding bits. */
4398	continue;
4399	if (all_ones && nonzero_last == endsize)
4400	{
4401	/* All bits between nonzero_first and end of word are padding
4402	bits, start counting padding_bytes. */
4403	padding_bytes = nonzero_last - nonzero_first;
4404	continue;
4405	}
4406	if (bytes_only)
4407	{
4408	/* If bitfields aren't involved in this word, prefer storing
4409	individual bytes or groups of them over performing a RMW
4410	operation on the whole word. */
4411	gcc_assert (i + zero_last <= end);
4412	for (size_t j = padding_end; j < i + zero_last; j++)
4413	{
4414	if (buf->buf[j])
4415	{
4416	size_t k;
4417	for (k = j; k < i + zero_last; k++)
4418	if (buf->buf[k] == 0)
4419	break;
4420	HOST_WIDE_INT off = buf->off + j;
4421	tree atype, src;
4422	if (k - j == 1)
4423	{
4424	atype = char_type_node;
4425	src = build_zero_cst (char_type_node);
4426	}
4427	else
4428	{
4429	atype = build_array_type_nelts (char_type_node, k - j);
4430	src = build_constructor (atype, NULL);
4431	}
4432	tree dst = build2_loc (loc: buf->loc, code: MEM_REF, type: atype,
4433	arg0: buf->base,
4434	arg1: build_int_cst (buf->alias_type, off));
4435	gimple *g = gimple_build_assign (dst, src);
4436	gimple_set_location (g, location: buf->loc);
4437	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4438	j = k;
4439	}
4440	}
4441	if (nonzero_last == endsize)
4442	padding_bytes = nonzero_last - zero_last;
4443	continue;
4444	}
4445	for (size_t eltsz = 1; eltsz <= wordsize; eltsz <<= 1)
4446	{
4447	if (nonzero_last - nonzero_first <= eltsz
4448	&& ((nonzero_first & ~(eltsz - 1))
4449	== ((nonzero_last - 1) & ~(eltsz - 1))))
4450	{
4451	tree type;
4452	if (eltsz == 1)
4453	type = char_type_node;
4454	else
4455	type = lang_hooks.types.type_for_size (eltsz * BITS_PER_UNIT,
4456	0);
4457	size_t start = nonzero_first & ~(eltsz - 1);
4458	HOST_WIDE_INT off = buf->off + i + start;
4459	tree atype = type;
4460	if (eltsz > 1 && buf->align < TYPE_ALIGN (type))
4461	atype = build_aligned_type (type, buf->align);
4462	tree dst = build2_loc (loc: buf->loc, code: MEM_REF, type: atype, arg0: buf->base,
4463	arg1: build_int_cst (buf->alias_type, off));
4464	tree src;
4465	gimple *g;
4466	if (all_ones
4467	&& nonzero_first == start
4468	&& nonzero_last == start + eltsz)
4469	src = build_zero_cst (type);
4470	else
4471	{
4472	src = make_ssa_name (var: type);
4473	tree tmp_dst = unshare_expr (dst);
4474	/* The folding introduces a read from the tmp_dst, we should
4475	prevent uninitialized warning analysis from issuing warning
4476	for such fake read. In order to suppress warning only for
4477	this expr, we should set the location of tmp_dst to
4478	UNKNOWN_LOCATION first, then suppress_warning will call
4479	set_no_warning_bit to set the no_warning flag only for
4480	tmp_dst. */
4481	SET_EXPR_LOCATION (tmp_dst, UNKNOWN_LOCATION);
4482	suppress_warning (tmp_dst, OPT_Wuninitialized);
4483	g = gimple_build_assign (src, tmp_dst);
4484	gimple_set_location (g, location: buf->loc);
4485	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4486	tree mask = native_interpret_expr (type,
4487	buf->buf + i + start,
4488	eltsz);
4489	gcc_assert (mask && TREE_CODE (mask) == INTEGER_CST);
4490	mask = fold_build1 (BIT_NOT_EXPR, type, mask);
4491	tree src_masked = make_ssa_name (var: type);
4492	g = gimple_build_assign (src_masked, BIT_AND_EXPR,
4493	src, mask);
4494	gimple_set_location (g, location: buf->loc);
4495	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4496	src = src_masked;
4497	}
4498	g = gimple_build_assign (dst, src);
4499	gimple_set_location (g, location: buf->loc);
4500	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4501	break;
4502	}
4503	}
4504	}
4505	if (full)
4506	{
4507	if (padding_bytes)
4508	{
4509	tree atype, src;
4510	if (padding_bytes == 1)
4511	{
4512	atype = char_type_node;
4513	src = build_zero_cst (char_type_node);
4514	}
4515	else
4516	{
4517	atype = build_array_type_nelts (char_type_node, padding_bytes);
4518	src = build_constructor (atype, NULL);
4519	}
4520	tree dst = build2_loc (loc: buf->loc, code: MEM_REF, type: atype, arg0: buf->base,
4521	arg1: build_int_cst (buf->alias_type,
4522	buf->off + end
4523	- padding_bytes));
4524	gimple *g = gimple_build_assign (dst, src);
4525	gimple_set_location (g, location: buf->loc);
4526	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4527	}
4528	size_t end_rem = end % UNITS_PER_WORD;
4529	buf->off += end - end_rem;
4530	buf->size = end_rem;
4531	memset (s: buf->buf, c: 0, n: buf->size);
4532	buf->padding_bytes = 0;
4533	}
4534	else
4535	{
4536	memmove (dest: buf->buf, src: buf->buf + end, n: buf->size - end);
4537	buf->off += end;
4538	buf->size -= end;
4539	buf->padding_bytes = padding_bytes;
4540	}
4541	}
4542
4543	/* Append PADDING_BYTES padding bytes. */
4544
4545	static void
4546	clear_padding_add_padding (clear_padding_struct *buf,
4547	HOST_WIDE_INT padding_bytes)
4548	{
4549	if (padding_bytes == 0)
4550	return;
4551	if ((unsigned HOST_WIDE_INT) padding_bytes + buf->size
4552	> (unsigned HOST_WIDE_INT) clear_padding_buf_size)
4553	clear_padding_flush (buf, full: false);
4554	if ((unsigned HOST_WIDE_INT) padding_bytes + buf->size
4555	> (unsigned HOST_WIDE_INT) clear_padding_buf_size)
4556	{
4557	memset (s: buf->buf + buf->size, c: ~0, n: clear_padding_buf_size - buf->size);
4558	padding_bytes -= clear_padding_buf_size - buf->size;
4559	buf->size = clear_padding_buf_size;
4560	clear_padding_flush (buf, full: false);
4561	gcc_assert (buf->padding_bytes);
4562	/* At this point buf->buf[0] through buf->buf[buf->size - 1]
4563	is guaranteed to be all ones. */
4564	padding_bytes += buf->size;
4565	buf->size = padding_bytes % UNITS_PER_WORD;
4566	memset (s: buf->buf, c: ~0, n: buf->size);
4567	buf->off += padding_bytes - buf->size;
4568	buf->padding_bytes += padding_bytes - buf->size;
4569	}
4570	else
4571	{
4572	memset (s: buf->buf + buf->size, c: ~0, n: padding_bytes);
4573	buf->size += padding_bytes;
4574	}
4575	}
4576
4577	static void clear_padding_type (clear_padding_struct *, tree,
4578	HOST_WIDE_INT, bool);
4579
4580	/* Clear padding bits of union type TYPE. */
4581
4582	static void
4583	clear_padding_union (clear_padding_struct *buf, tree type,
4584	HOST_WIDE_INT sz, bool for_auto_init)
4585	{
4586	clear_padding_struct *union_buf;
4587	HOST_WIDE_INT start_off = 0, next_off = 0;
4588	size_t start_size = 0;
4589	if (buf->union_ptr)
4590	{
4591	start_off = buf->off + buf->size;
4592	next_off = start_off + sz;
4593	start_size = start_off % UNITS_PER_WORD;
4594	start_off -= start_size;
4595	clear_padding_flush (buf, full: true);
4596	union_buf = buf;
4597	}
4598	else
4599	{
4600	if (sz + buf->size > clear_padding_buf_size)
4601	clear_padding_flush (buf, full: false);
4602	union_buf = XALLOCA (clear_padding_struct);
4603	union_buf->loc = buf->loc;
4604	union_buf->clear_in_mask = buf->clear_in_mask;
4605	union_buf->base = NULL_TREE;
4606	union_buf->alias_type = NULL_TREE;
4607	union_buf->gsi = NULL;
4608	union_buf->align = 0;
4609	union_buf->off = 0;
4610	union_buf->padding_bytes = 0;
4611	union_buf->sz = sz;
4612	union_buf->size = 0;
4613	if (sz + buf->size <= clear_padding_buf_size)
4614	union_buf->union_ptr = buf->buf + buf->size;
4615	else
4616	union_buf->union_ptr = XNEWVEC (unsigned char, sz);
4617	memset (s: union_buf->union_ptr, c: ~0, n: sz);
4618	}
4619
4620	for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
4621	if (TREE_CODE (field) == FIELD_DECL && !DECL_PADDING_P (field))
4622	{
4623	if (DECL_SIZE_UNIT (field) == NULL_TREE)
4624	{
4625	if (TREE_TYPE (field) == error_mark_node)
4626	continue;
4627	gcc_assert (TREE_CODE (TREE_TYPE (field)) == ARRAY_TYPE
4628	&& !COMPLETE_TYPE_P (TREE_TYPE (field)));
4629	if (!buf->clear_in_mask && !for_auto_init)
4630	error_at (buf->loc, "flexible array member %qD does not have "
4631	"well defined padding bits for %qs",
4632	field, "__builtin_clear_padding");
4633	continue;
4634	}
4635	HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field));
4636	gcc_assert (union_buf->size == 0);
4637	union_buf->off = start_off;
4638	union_buf->size = start_size;
4639	memset (s: union_buf->buf, c: ~0, n: start_size);
4640	clear_padding_type (union_buf, TREE_TYPE (field), fldsz, for_auto_init);
4641	clear_padding_add_padding (buf: union_buf, padding_bytes: sz - fldsz);
4642	clear_padding_flush (buf: union_buf, full: true);
4643	}
4644
4645	if (buf == union_buf)
4646	{
4647	buf->off = next_off;
4648	buf->size = next_off % UNITS_PER_WORD;
4649	buf->off -= buf->size;
4650	memset (s: buf->buf, c: ~0, n: buf->size);
4651	}
4652	else if (sz + buf->size <= clear_padding_buf_size)
4653	buf->size += sz;
4654	else
4655	{
4656	unsigned char *union_ptr = union_buf->union_ptr;
4657	while (sz)
4658	{
4659	clear_padding_flush (buf, full: false);
4660	HOST_WIDE_INT this_sz
4661	= MIN ((unsigned HOST_WIDE_INT) sz,
4662	clear_padding_buf_size - buf->size);
4663	memcpy (dest: buf->buf + buf->size, src: union_ptr, n: this_sz);
4664	buf->size += this_sz;
4665	union_ptr += this_sz;
4666	sz -= this_sz;
4667	}
4668	XDELETE (union_buf->union_ptr);
4669	}
4670	}
4671
4672	/* The only known floating point formats with padding bits are the
4673	IEEE extended ones. */
4674
4675	static bool
4676	clear_padding_real_needs_padding_p (tree type)
4677	{
4678	const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type));
4679	return (fmt->b == 2
4680	&& fmt->signbit_ro == fmt->signbit_rw
4681	&& (fmt->signbit_ro == 79 || fmt->signbit_ro == 95));
4682	}
4683
4684	/* _BitInt has padding bits if it isn't extended in the ABI and has smaller
4685	precision than bits in limb or corresponding number of limbs. */
4686
4687	static bool
4688	clear_padding_bitint_needs_padding_p (tree type)
4689	{
4690	struct bitint_info info;
4691	bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
4692	gcc_assert (ok);
4693	if (info.extended)
4694	return false;
4695	scalar_int_mode limb_mode = as_a <scalar_int_mode> (m: info.abi_limb_mode);
4696	if (TYPE_PRECISION (type) < GET_MODE_PRECISION (mode: limb_mode))
4697	return true;
4698	else if (TYPE_PRECISION (type) == GET_MODE_PRECISION (mode: limb_mode))
4699	return false;
4700	else
4701	return (((unsigned) TYPE_PRECISION (type))
4702	% GET_MODE_PRECISION (mode: limb_mode)) != 0;
4703	}
4704
4705	/* Return true if TYPE might contain any padding bits. */
4706
4707	bool
4708	clear_padding_type_may_have_padding_p (tree type)
4709	{
4710	switch (TREE_CODE (type))
4711	{
4712	case RECORD_TYPE:
4713	case UNION_TYPE:
4714	return true;
4715	case ARRAY_TYPE:
4716	case COMPLEX_TYPE:
4717	case VECTOR_TYPE:
4718	return clear_padding_type_may_have_padding_p (TREE_TYPE (type));
4719	case REAL_TYPE:
4720	return clear_padding_real_needs_padding_p (type);
4721	case BITINT_TYPE:
4722	return clear_padding_bitint_needs_padding_p (type);
4723	default:
4724	return false;
4725	}
4726	}
4727
4728	/* Return true if TYPE has padding bits aside from those in fields,
4729	elements, etc. */
4730
4731	bool
4732	type_has_padding_at_level_p (tree type)
4733	{
4734	switch (TREE_CODE (type))
4735	{
4736	case RECORD_TYPE:
4737	{
4738	tree bitpos = size_zero_node;
4739	/* Expect fields to be sorted by bit position. */
4740	for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
4741	if (TREE_CODE (f) == FIELD_DECL)
4742	{
4743	if (DECL_PADDING_P (f))
4744	return true;
4745	tree pos = bit_position (f);
4746	if (simple_cst_equal (bitpos, pos) != 1)
4747	return true;
4748	if (!DECL_SIZE (f))
4749	return true;
4750	bitpos = int_const_binop (PLUS_EXPR, pos, DECL_SIZE (f));
4751	}
4752	if (simple_cst_equal (bitpos, TYPE_SIZE (type)) != 1)
4753	return true;
4754	return false;
4755	}
4756	case UNION_TYPE:
4757	case QUAL_UNION_TYPE:
4758	bool any_fields;
4759	any_fields = false;
4760	/* If any of the fields is smaller than the whole, there is padding. */
4761	for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
4762	if (TREE_CODE (f) != FIELD_DECL || TREE_TYPE (f) == error_mark_node)
4763	continue;
4764	else if (simple_cst_equal (TYPE_SIZE (TREE_TYPE (f)),
4765	TYPE_SIZE (type)) != 1)
4766	return true;
4767	else
4768	any_fields = true;
4769	/* If the union doesn't have any fields and still has non-zero size,
4770	all of it is padding. */
4771	if (!any_fields && !integer_zerop (TYPE_SIZE (type)))
4772	return true;
4773	return false;
4774	case ARRAY_TYPE:
4775	case COMPLEX_TYPE:
4776	case VECTOR_TYPE:
4777	/* No recursing here, no padding at this level. */
4778	return false;
4779	case REAL_TYPE:
4780	return clear_padding_real_needs_padding_p (type);
4781	case BITINT_TYPE:
4782	return clear_padding_bitint_needs_padding_p (type);
4783	default:
4784	return false;
4785	}
4786	}
4787
4788	/* Emit a runtime loop:
4789	for (; buf.base != end; buf.base += sz)
4790	__builtin_clear_padding (buf.base); */
4791
4792	static void
4793	clear_padding_emit_loop (clear_padding_struct *buf, tree type,
4794	tree end, bool for_auto_init)
4795	{
4796	tree l1 = create_artificial_label (buf->loc);
4797	tree l2 = create_artificial_label (buf->loc);
4798	tree l3 = create_artificial_label (buf->loc);
4799	gimple *g = gimple_build_goto (dest: l2);
4800	gimple_set_location (g, location: buf->loc);
4801	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4802	g = gimple_build_label (label: l1);
4803	gimple_set_location (g, location: buf->loc);
4804	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4805	clear_padding_type (buf, type, buf->sz, for_auto_init);
4806	clear_padding_flush (buf, full: true);
4807	g = gimple_build_assign (buf->base, POINTER_PLUS_EXPR, buf->base,
4808	size_int (buf->sz));
4809	gimple_set_location (g, location: buf->loc);
4810	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4811	g = gimple_build_label (label: l2);
4812	gimple_set_location (g, location: buf->loc);
4813	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4814	g = gimple_build_cond (NE_EXPR, buf->base, end, l1, l3);
4815	gimple_set_location (g, location: buf->loc);
4816	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4817	g = gimple_build_label (label: l3);
4818	gimple_set_location (g, location: buf->loc);
4819	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4820	}
4821
4822	/* Clear padding bits for TYPE. Called recursively from
4823	gimple_fold_builtin_clear_padding. If FOR_AUTO_INIT is true,
4824	the __builtin_clear_padding is not called by the end user,
4825	instead, it's inserted by the compiler to initialize the
4826	paddings of automatic variable. Therefore, we should not
4827	emit the error messages for flexible array members to confuse
4828	the end user. */
4829
4830	static void
4831	clear_padding_type (clear_padding_struct *buf, tree type,
4832	HOST_WIDE_INT sz, bool for_auto_init)
4833	{
4834	switch (TREE_CODE (type))
4835	{
4836	case RECORD_TYPE:
4837	HOST_WIDE_INT cur_pos;
4838	cur_pos = 0;
4839	for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
4840	if (TREE_CODE (field) == FIELD_DECL && !DECL_PADDING_P (field))
4841	{
4842	tree ftype = TREE_TYPE (field);
4843	if (DECL_BIT_FIELD (field))
4844	{
4845	HOST_WIDE_INT fldsz = TYPE_PRECISION (ftype);
4846	if (fldsz == 0)
4847	continue;
4848	HOST_WIDE_INT pos = int_byte_position (field);
4849	if (pos >= sz)
4850	continue;
4851	HOST_WIDE_INT bpos
4852	= tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field));
4853	bpos %= BITS_PER_UNIT;
4854	HOST_WIDE_INT end
4855	= ROUND_UP (bpos + fldsz, BITS_PER_UNIT) / BITS_PER_UNIT;
4856	if (pos + end > cur_pos)
4857	{
4858	clear_padding_add_padding (buf, padding_bytes: pos + end - cur_pos);
4859	cur_pos = pos + end;
4860	}
4861	gcc_assert (cur_pos > pos
4862	&& ((unsigned HOST_WIDE_INT) buf->size
4863	>= (unsigned HOST_WIDE_INT) cur_pos - pos));
4864	unsigned char *p = buf->buf + buf->size - (cur_pos - pos);
4865	if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
4866	sorry_at (buf->loc, "PDP11 bit-field handling unsupported"
4867	" in %qs", "__builtin_clear_padding");
4868	else if (BYTES_BIG_ENDIAN)
4869	{
4870	/* Big endian. */
4871	if (bpos + fldsz <= BITS_PER_UNIT)
4872	*p &= ~(((1 << fldsz) - 1)
4873	<< (BITS_PER_UNIT - bpos - fldsz));
4874	else
4875	{
4876	if (bpos)
4877	{
4878	*p &= ~(((1U << BITS_PER_UNIT) - 1) >> bpos);
4879	p++;
4880	fldsz -= BITS_PER_UNIT - bpos;
4881	}
4882	memset (s: p, c: 0, n: fldsz / BITS_PER_UNIT);
4883	p += fldsz / BITS_PER_UNIT;
4884	fldsz %= BITS_PER_UNIT;
4885	if (fldsz)
4886	*p &= ((1U << BITS_PER_UNIT) - 1) >> fldsz;
4887	}
4888	}
4889	else
4890	{
4891	/* Little endian. */
4892	if (bpos + fldsz <= BITS_PER_UNIT)
4893	*p &= ~(((1 << fldsz) - 1) << bpos);
4894	else
4895	{
4896	if (bpos)
4897	{
4898	*p &= ~(((1 << BITS_PER_UNIT) - 1) << bpos);
4899	p++;
4900	fldsz -= BITS_PER_UNIT - bpos;
4901	}
4902	memset (s: p, c: 0, n: fldsz / BITS_PER_UNIT);
4903	p += fldsz / BITS_PER_UNIT;
4904	fldsz %= BITS_PER_UNIT;
4905	if (fldsz)
4906	*p &= ~((1 << fldsz) - 1);
4907	}
4908	}
4909	}
4910	else if (DECL_SIZE_UNIT (field) == NULL_TREE)
4911	{
4912	if (ftype == error_mark_node)
4913	continue;
4914	gcc_assert (TREE_CODE (ftype) == ARRAY_TYPE
4915	&& !COMPLETE_TYPE_P (ftype));
4916	if (!buf->clear_in_mask && !for_auto_init)
4917	error_at (buf->loc, "flexible array member %qD does not "
4918	"have well defined padding bits for %qs",
4919	field, "__builtin_clear_padding");
4920	}
4921	else if (is_empty_type (ftype))
4922	continue;
4923	else
4924	{
4925	HOST_WIDE_INT pos = int_byte_position (field);
4926	if (pos >= sz)
4927	continue;
4928	HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field));
4929	gcc_assert (pos >= 0 && fldsz >= 0 && pos >= cur_pos);
4930	clear_padding_add_padding (buf, padding_bytes: pos - cur_pos);
4931	cur_pos = pos;
4932	if (tree asbase = lang_hooks.types.classtype_as_base (field))
4933	ftype = asbase;
4934	clear_padding_type (buf, type: ftype, sz: fldsz, for_auto_init);
4935	cur_pos += fldsz;
4936	}
4937	}
4938	gcc_assert (sz >= cur_pos);
4939	clear_padding_add_padding (buf, padding_bytes: sz - cur_pos);
4940	break;
4941	case ARRAY_TYPE:
4942	HOST_WIDE_INT nelts, fldsz;
4943	fldsz = int_size_in_bytes (TREE_TYPE (type));
4944	if (fldsz == 0)
4945	break;
4946	nelts = sz / fldsz;
4947	if (nelts > 1
4948	&& sz > 8 * UNITS_PER_WORD
4949	&& buf->union_ptr == NULL
4950	&& clear_padding_type_may_have_padding_p (TREE_TYPE (type)))
4951	{
4952	/* For sufficiently large array of more than one elements,
4953	emit a runtime loop to keep code size manageable. */
4954	tree base = buf->base;
4955	unsigned int prev_align = buf->align;
4956	HOST_WIDE_INT off = buf->off + buf->size;
4957	HOST_WIDE_INT prev_sz = buf->sz;
4958	clear_padding_flush (buf, full: true);
4959	tree elttype = TREE_TYPE (type);
4960	buf->base = create_tmp_var (build_pointer_type (elttype));
4961	tree end = make_ssa_name (TREE_TYPE (buf->base));
4962	gimple *g = gimple_build_assign (buf->base, POINTER_PLUS_EXPR,
4963	base, size_int (off));
4964	gimple_set_location (g, location: buf->loc);
4965	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4966	g = gimple_build_assign (end, POINTER_PLUS_EXPR, buf->base,
4967	size_int (sz));
4968	gimple_set_location (g, location: buf->loc);
4969	gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
4970	buf->sz = fldsz;
4971	buf->align = TYPE_ALIGN (elttype);
4972	buf->off = 0;
4973	buf->size = 0;
4974	clear_padding_emit_loop (buf, type: elttype, end, for_auto_init);
4975	off += sz;
4976	buf->base = base;
4977	buf->sz = prev_sz;
4978	buf->align = prev_align;
4979	buf->size = off % UNITS_PER_WORD;
4980	buf->off = off - buf->size;
4981	memset (s: buf->buf, c: 0, n: buf->size);
4982	break;
4983	}
4984	for (HOST_WIDE_INT i = 0; i < nelts; i++)
4985	clear_padding_type (buf, TREE_TYPE (type), sz: fldsz, for_auto_init);
4986	break;
4987	case UNION_TYPE:
4988	clear_padding_union (buf, type, sz, for_auto_init);
4989	break;
4990	case REAL_TYPE:
4991	gcc_assert ((size_t) sz <= clear_padding_unit);
4992	if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size)
4993	clear_padding_flush (buf, full: false);
4994	if (clear_padding_real_needs_padding_p (type))
4995	{
4996	/* Use native_interpret_real + native_encode_expr to figure out
4997	which bits are padding. */
4998	memset (s: buf->buf + buf->size, c: ~0, n: sz);
4999	tree cst = native_interpret_real (type, buf->buf + buf->size, sz);
5000	gcc_assert (cst && TREE_CODE (cst) == REAL_CST);
5001	int len = native_encode_expr (cst, buf->buf + buf->size, sz);
5002	gcc_assert (len > 0 && (size_t) len == (size_t) sz);
5003	for (size_t i = 0; i < (size_t) sz; i++)
5004	buf->buf[buf->size + i] ^= ~0;
5005	}
5006	else
5007	memset (s: buf->buf + buf->size, c: 0, n: sz);
5008	buf->size += sz;
5009	break;
5010	case COMPLEX_TYPE:
5011	fldsz = int_size_in_bytes (TREE_TYPE (type));
5012	clear_padding_type (buf, TREE_TYPE (type), sz: fldsz, for_auto_init);
5013	clear_padding_type (buf, TREE_TYPE (type), sz: fldsz, for_auto_init);
5014	break;
5015	case VECTOR_TYPE:
5016	nelts = TYPE_VECTOR_SUBPARTS (node: type).to_constant ();
5017	fldsz = int_size_in_bytes (TREE_TYPE (type));
5018	for (HOST_WIDE_INT i = 0; i < nelts; i++)
5019	clear_padding_type (buf, TREE_TYPE (type), sz: fldsz, for_auto_init);
5020	break;
5021	case NULLPTR_TYPE:
5022	gcc_assert ((size_t) sz <= clear_padding_unit);
5023	if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size)
5024	clear_padding_flush (buf, full: false);
5025	memset (s: buf->buf + buf->size, c: ~0, n: sz);
5026	buf->size += sz;
5027	break;
5028	case BITINT_TYPE:
5029	{
5030	struct bitint_info info;
5031	bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
5032	gcc_assert (ok);
5033	scalar_int_mode limb_mode
5034	= as_a <scalar_int_mode> (m: info.abi_limb_mode);
5035	if (TYPE_PRECISION (type) <= GET_MODE_PRECISION (mode: limb_mode))
5036	{
5037	gcc_assert ((size_t) sz <= clear_padding_unit);
5038	if ((unsigned HOST_WIDE_INT) sz + buf->size
5039	> clear_padding_buf_size)
5040	clear_padding_flush (buf, full: false);
5041	if (!info.extended
5042	&& TYPE_PRECISION (type) < GET_MODE_PRECISION (mode: limb_mode))
5043	{
5044	int tprec = GET_MODE_PRECISION (mode: limb_mode);
5045	int prec = TYPE_PRECISION (type);
5046	tree t = build_nonstandard_integer_type (tprec, 1);
5047	tree cst = wide_int_to_tree (type: t, cst: wi::mask (width: prec, negate_p: true, precision: tprec));
5048	int len = native_encode_expr (cst, buf->buf + buf->size, sz);
5049	gcc_assert (len > 0 && (size_t) len == (size_t) sz);
5050	}
5051	else
5052	memset (s: buf->buf + buf->size, c: 0, n: sz);
5053	buf->size += sz;
5054	break;
5055	}
5056	tree limbtype
5057	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode: limb_mode), 1);
5058	fldsz = int_size_in_bytes (limbtype);
5059	nelts = int_size_in_bytes (type) / fldsz;
5060	for (HOST_WIDE_INT i = 0; i < nelts; i++)
5061	{
5062	if (!info.extended
5063	&& i == (info.big_endian ? 0 : nelts - 1)
5064	&& (((unsigned) TYPE_PRECISION (type))
5065	% TYPE_PRECISION (limbtype)) != 0)
5066	{
5067	int tprec = GET_MODE_PRECISION (mode: limb_mode);
5068	int prec = (((unsigned) TYPE_PRECISION (type)) % tprec);
5069	tree cst = wide_int_to_tree (type: limbtype,
5070	cst: wi::mask (width: prec, negate_p: true, precision: tprec));
5071	int len = native_encode_expr (cst, buf->buf + buf->size,
5072	fldsz);
5073	gcc_assert (len > 0 && (size_t) len == (size_t) fldsz);
5074	buf->size += fldsz;
5075	}
5076	else
5077	clear_padding_type (buf, type: limbtype, sz: fldsz, for_auto_init);
5078	}
5079	break;
5080	}
5081	default:
5082	gcc_assert ((size_t) sz <= clear_padding_unit);
5083	if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size)
5084	clear_padding_flush (buf, full: false);
5085	memset (s: buf->buf + buf->size, c: 0, n: sz);
5086	buf->size += sz;
5087	break;
5088	}
5089	}
5090
5091	/* Clear padding bits of TYPE in MASK. */
5092
5093	void
5094	clear_type_padding_in_mask (tree type, unsigned char *mask)
5095	{
5096	clear_padding_struct buf;
5097	buf.loc = UNKNOWN_LOCATION;
5098	buf.clear_in_mask = true;
5099	buf.base = NULL_TREE;
5100	buf.alias_type = NULL_TREE;
5101	buf.gsi = NULL;
5102	buf.align = 0;
5103	buf.off = 0;
5104	buf.padding_bytes = 0;
5105	buf.sz = int_size_in_bytes (type);
5106	buf.size = 0;
5107	buf.union_ptr = mask;
5108	clear_padding_type (buf: &buf, type, sz: buf.sz, for_auto_init: false);
5109	clear_padding_flush (buf: &buf, full: true);
5110	}
5111
5112	/* Fold __builtin_clear_padding builtin. */
5113
5114	static bool
5115	gimple_fold_builtin_clear_padding (gimple_stmt_iterator *gsi)
5116	{
5117	gimple *stmt = gsi_stmt (i: *gsi);
5118	gcc_assert (gimple_call_num_args (stmt) == 2);
5119	tree ptr = gimple_call_arg (gs: stmt, index: 0);
5120	tree typearg = gimple_call_arg (gs: stmt, index: 1);
5121	/* The 2nd argument of __builtin_clear_padding's value is used to
5122	distinguish whether this call is made by the user or by the compiler
5123	for automatic variable initialization. */
5124	bool for_auto_init = (bool) TREE_INT_CST_LOW (typearg);
5125	tree type = TREE_TYPE (TREE_TYPE (typearg));
5126	location_t loc = gimple_location (g: stmt);
5127	clear_padding_struct buf;
5128	gimple_stmt_iterator gsiprev = *gsi;
5129	/* This should be folded during the lower pass. */
5130	gcc_assert (!gimple_in_ssa_p (cfun) && cfun->cfg == NULL);
5131	gcc_assert (COMPLETE_TYPE_P (type));
5132	gsi_prev (i: &gsiprev);
5133
5134	buf.loc = loc;
5135	buf.clear_in_mask = false;
5136	buf.base = ptr;
5137	buf.alias_type = NULL_TREE;
5138	buf.gsi = gsi;
5139	buf.align = get_pointer_alignment (ptr);
5140	unsigned int talign = min_align_of_type (type) * BITS_PER_UNIT;
5141	buf.align = MAX (buf.align, talign);
5142	buf.off = 0;
5143	buf.padding_bytes = 0;
5144	buf.size = 0;
5145	buf.sz = int_size_in_bytes (type);
5146	buf.union_ptr = NULL;
5147	if (buf.sz < 0 && int_size_in_bytes (strip_array_types (type)) < 0)
5148	sorry_at (loc, "%s not supported for variable length aggregates",
5149	"__builtin_clear_padding");
5150	/* The implementation currently assumes 8-bit host and target
5151	chars which is the case for all currently supported targets
5152	and hosts and is required e.g. for native_{encode,interpret}* APIs. */
5153	else if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
5154	sorry_at (loc, "%s not supported on this target",
5155	"__builtin_clear_padding");
5156	else if (!clear_padding_type_may_have_padding_p (type))
5157	;
5158	else if (TREE_CODE (type) == ARRAY_TYPE && buf.sz < 0)
5159	{
5160	tree sz = TYPE_SIZE_UNIT (type);
5161	tree elttype = type;
5162	/* Only supports C/C++ VLAs and flattens all the VLA levels. */
5163	while (TREE_CODE (elttype) == ARRAY_TYPE
5164	&& int_size_in_bytes (elttype) < 0)
5165	elttype = TREE_TYPE (elttype);
5166	HOST_WIDE_INT eltsz = int_size_in_bytes (elttype);
5167	gcc_assert (eltsz >= 0);
5168	if (eltsz)
5169	{
5170	buf.base = create_tmp_var (build_pointer_type (elttype));
5171	tree end = make_ssa_name (TREE_TYPE (buf.base));
5172	gimple *g = gimple_build_assign (buf.base, ptr);
5173	gimple_set_location (g, location: loc);
5174	gsi_insert_before (gsi, g, GSI_SAME_STMT);
5175	g = gimple_build_assign (end, POINTER_PLUS_EXPR, buf.base, sz);
5176	gimple_set_location (g, location: loc);
5177	gsi_insert_before (gsi, g, GSI_SAME_STMT);
5178	buf.sz = eltsz;
5179	buf.align = TYPE_ALIGN (elttype);
5180	buf.alias_type = build_pointer_type (elttype);
5181	clear_padding_emit_loop (buf: &buf, type: elttype, end, for_auto_init);
5182	}
5183	}
5184	else
5185	{
5186	if (!is_gimple_mem_ref_addr (buf.base))
5187	{
5188	buf.base = make_ssa_name (TREE_TYPE (ptr));
5189	gimple *g = gimple_build_assign (buf.base, ptr);
5190	gimple_set_location (g, location: loc);
5191	gsi_insert_before (gsi, g, GSI_SAME_STMT);
5192	}
5193	buf.alias_type = build_pointer_type (type);
5194	clear_padding_type (buf: &buf, type, sz: buf.sz, for_auto_init);
5195	clear_padding_flush (buf: &buf, full: true);
5196	}
5197
5198	gimple_stmt_iterator gsiprev2 = *gsi;
5199	gsi_prev (i: &gsiprev2);
5200	if (gsi_stmt (i: gsiprev) == gsi_stmt (i: gsiprev2))
5201	gsi_replace (gsi, gimple_build_nop (), true);
5202	else
5203	{
5204	gsi_remove (gsi, true);
5205	*gsi = gsiprev2;
5206	}
5207	return true;
5208	}
5209
5210	/* Fold __builtin_constant_p builtin. */
5211
5212	static bool
5213	gimple_fold_builtin_constant_p (gimple_stmt_iterator *gsi)
5214	{
5215	gcall *call = as_a<gcall*>(p: gsi_stmt (i: *gsi));
5216
5217	if (gimple_call_num_args (gs: call) != 1)
5218	return false;
5219
5220	tree arg = gimple_call_arg (gs: call, index: 0);
5221	tree result = fold_builtin_constant_p (arg);
5222
5223	/* Resolve __builtin_constant_p. If it hasn't been
5224	folded to integer_one_node by now, it's fairly
5225	certain that the value simply isn't constant. */
5226	if (!result && fold_before_rtl_expansion_p ())
5227	result = integer_zero_node;
5228
5229	if (!result)
5230	return false;
5231
5232	gimplify_and_update_call_from_tree (si_p: gsi, expr: result);
5233	return true;
5234	}
5235
5236	/* If va_list type is a simple pointer and nothing special is needed,
5237	optimize __builtin_va_start (&ap, 0) into ap = __builtin_next_arg (0),
5238	__builtin_va_end (&ap) out as NOP and __builtin_va_copy into a simple
5239	pointer assignment. Returns true if a change happened. */
5240
5241	static bool
5242	gimple_fold_builtin_stdarg (gimple_stmt_iterator *gsi, gcall *call)
5243	{
5244	/* These shouldn't be folded before pass_stdarg. */
5245	if (!fold_before_rtl_expansion_p ())
5246	return false;
5247
5248	tree callee, lhs, rhs, cfun_va_list;
5249	bool va_list_simple_ptr;
5250	location_t loc = gimple_location (g: call);
5251	gimple *nstmt0, *nstmt;
5252	tree tlhs, oldvdef, newvdef;
5253
5254	callee = gimple_call_fndecl (gs: call);
5255
5256	cfun_va_list = targetm.fn_abi_va_list (callee);
5257	va_list_simple_ptr = POINTER_TYPE_P (cfun_va_list)
5258	&& (TREE_TYPE (cfun_va_list) == void_type_node
5259	|| TREE_TYPE (cfun_va_list) == char_type_node);
5260
5261	switch (DECL_FUNCTION_CODE (decl: callee))
5262	{
5263	case BUILT_IN_VA_START:
5264	if (!va_list_simple_ptr
5265	|| targetm.expand_builtin_va_start != NULL
5266	|| !builtin_decl_explicit_p (fncode: BUILT_IN_NEXT_ARG))
5267	return false;
5268
5269	if (gimple_call_num_args (gs: call) != 2)
5270	return false;
5271
5272	lhs = gimple_call_arg (gs: call, index: 0);
5273	if (!POINTER_TYPE_P (TREE_TYPE (lhs))
5274	|| TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (lhs)))
5275	!= TYPE_MAIN_VARIANT (cfun_va_list))
5276	return false;
5277	/* Create `tlhs = __builtin_next_arg(0);`. */
5278	tlhs = make_ssa_name (var: cfun_va_list);
5279	nstmt0 = gimple_build_call (builtin_decl_explicit (fncode: BUILT_IN_NEXT_ARG), 1, integer_zero_node);
5280	lhs = fold_build2 (MEM_REF, cfun_va_list, lhs, build_zero_cst (TREE_TYPE (lhs)));
5281	gimple_call_set_lhs (gs: nstmt0, lhs: tlhs);
5282	gimple_set_location (g: nstmt0, location: loc);
5283	gimple_move_vops (nstmt0, call);
5284	gsi_replace (gsi, nstmt0, false);
5285	oldvdef = gimple_vdef (g: nstmt0);
5286	newvdef = make_ssa_name (var: gimple_vop (cfun), stmt: nstmt0);
5287	gimple_set_vdef (g: nstmt0, vdef: newvdef);
5288
5289	/* Create `*lhs = tlhs;`. */
5290	nstmt = gimple_build_assign (lhs, tlhs);
5291	gimple_set_location (g: nstmt, location: loc);
5292	gimple_set_vuse (g: nstmt, vuse: newvdef);
5293	gimple_set_vdef (g: nstmt, vdef: oldvdef);
5294	SSA_NAME_DEF_STMT (oldvdef) = nstmt;
5295	gsi_insert_after (gsi, nstmt, GSI_NEW_STMT);
5296
5297	if (dump_file && (dump_flags & TDF_DETAILS))
5298	{
5299	fprintf (stream: dump_file, format: "Simplified\n ");
5300	print_gimple_stmt (dump_file, call, 0, dump_flags);
5301	fprintf (stream: dump_file, format: "into\n ");
5302	print_gimple_stmt (dump_file, nstmt0, 0, dump_flags);
5303	fprintf (stream: dump_file, format: " ");
5304	print_gimple_stmt (dump_file, nstmt, 0, dump_flags);
5305	}
5306	return true;
5307
5308	case BUILT_IN_VA_COPY:
5309	if (!va_list_simple_ptr)
5310	return false;
5311
5312	if (gimple_call_num_args (gs: call) != 2)
5313	return false;
5314
5315	lhs = gimple_call_arg (gs: call, index: 0);
5316	if (!POINTER_TYPE_P (TREE_TYPE (lhs))
5317	|| TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (lhs)))
5318	!= TYPE_MAIN_VARIANT (cfun_va_list))
5319	return false;
5320	rhs = gimple_call_arg (gs: call, index: 1);
5321	if (TYPE_MAIN_VARIANT (TREE_TYPE (rhs))
5322	!= TYPE_MAIN_VARIANT (cfun_va_list))
5323	return false;
5324
5325	lhs = fold_build2 (MEM_REF, cfun_va_list, lhs, build_zero_cst (TREE_TYPE (lhs)));
5326	nstmt = gimple_build_assign (lhs, rhs);
5327	gimple_set_location (g: nstmt, location: loc);
5328	gimple_move_vops (nstmt, call);
5329	gsi_replace (gsi, nstmt, false);
5330
5331	if (dump_file && (dump_flags & TDF_DETAILS))
5332	{
5333	fprintf (stream: dump_file, format: "Simplified\n ");
5334	print_gimple_stmt (dump_file, call, 0, dump_flags);
5335	fprintf (stream: dump_file, format: "into\n ");
5336	print_gimple_stmt (dump_file, nstmt, 0, dump_flags);
5337	}
5338	return true;
5339
5340	case BUILT_IN_VA_END:
5341	/* No effect, so the statement will be deleted. */
5342	if (dump_file && (dump_flags & TDF_DETAILS))
5343	{
5344	fprintf (stream: dump_file, format: "Removed\n ");
5345	print_gimple_stmt (dump_file, call, 0, dump_flags);
5346	}
5347	unlink_stmt_vdef (call);
5348	release_defs (call);
5349	gsi_replace (gsi, gimple_build_nop (), true);
5350	return true;
5351
5352	default:
5353	gcc_unreachable ();
5354	}
5355	}
5356
5357	/* Fold the non-target builtin at *GSI and return whether any simplification
5358	was made. */
5359
5360	static bool
5361	gimple_fold_builtin (gimple_stmt_iterator *gsi)
5362	{
5363	gcall *stmt = as_a <gcall *>(p: gsi_stmt (i: *gsi));
5364	tree callee = gimple_call_fndecl (gs: stmt);
5365
5366	/* Give up for always_inline inline builtins until they are
5367	inlined. */
5368	if (avoid_folding_inline_builtin (callee))
5369	return false;
5370
5371	unsigned n = gimple_call_num_args (gs: stmt);
5372	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: callee);
5373	switch (fcode)
5374	{
5375	case BUILT_IN_VA_START:
5376	case BUILT_IN_VA_END:
5377	case BUILT_IN_VA_COPY:
5378	return gimple_fold_builtin_stdarg (gsi, call: stmt);
5379	case BUILT_IN_BCMP:
5380	return gimple_fold_builtin_bcmp (gsi);
5381	case BUILT_IN_BCOPY:
5382	return gimple_fold_builtin_bcopy (gsi);
5383	case BUILT_IN_BZERO:
5384	return gimple_fold_builtin_bzero (gsi);
5385
5386	case BUILT_IN_MEMSET:
5387	return gimple_fold_builtin_memset (gsi,
5388	c: gimple_call_arg (gs: stmt, index: 1),
5389	len: gimple_call_arg (gs: stmt, index: 2));
5390	case BUILT_IN_MEMCPY:
5391	case BUILT_IN_MEMPCPY:
5392	case BUILT_IN_MEMMOVE:
5393	return gimple_fold_builtin_memory_op (gsi, dest: gimple_call_arg (gs: stmt, index: 0),
5394	src: gimple_call_arg (gs: stmt, index: 1), code: fcode);
5395	case BUILT_IN_SPRINTF_CHK:
5396	case BUILT_IN_VSPRINTF_CHK:
5397	return gimple_fold_builtin_sprintf_chk (gsi, fcode);
5398	case BUILT_IN_STRCAT_CHK:
5399	return gimple_fold_builtin_strcat_chk (gsi);
5400	case BUILT_IN_STRNCAT_CHK:
5401	return gimple_fold_builtin_strncat_chk (gsi);
5402	case BUILT_IN_STRLEN:
5403	return gimple_fold_builtin_strlen (gsi);
5404	case BUILT_IN_STRCPY:
5405	return gimple_fold_builtin_strcpy (gsi,
5406	dest: gimple_call_arg (gs: stmt, index: 0),
5407	src: gimple_call_arg (gs: stmt, index: 1));
5408	case BUILT_IN_STRNCPY:
5409	return gimple_fold_builtin_strncpy (gsi,
5410	dest: gimple_call_arg (gs: stmt, index: 0),
5411	src: gimple_call_arg (gs: stmt, index: 1),
5412	len: gimple_call_arg (gs: stmt, index: 2));
5413	case BUILT_IN_STRCAT:
5414	return gimple_fold_builtin_strcat (gsi, dst: gimple_call_arg (gs: stmt, index: 0),
5415	src: gimple_call_arg (gs: stmt, index: 1));
5416	case BUILT_IN_STRNCAT:
5417	return gimple_fold_builtin_strncat (gsi);
5418	case BUILT_IN_INDEX:
5419	case BUILT_IN_STRCHR:
5420	return gimple_fold_builtin_strchr (gsi, is_strrchr: false);
5421	case BUILT_IN_RINDEX:
5422	case BUILT_IN_STRRCHR:
5423	return gimple_fold_builtin_strchr (gsi, is_strrchr: true);
5424	case BUILT_IN_STRSTR:
5425	return gimple_fold_builtin_strstr (gsi);
5426	case BUILT_IN_STRCMP:
5427	case BUILT_IN_STRCMP_EQ:
5428	case BUILT_IN_STRCASECMP:
5429	case BUILT_IN_STRNCMP:
5430	case BUILT_IN_STRNCMP_EQ:
5431	case BUILT_IN_STRNCASECMP:
5432	return gimple_fold_builtin_string_compare (gsi);
5433	case BUILT_IN_MEMCHR:
5434	return gimple_fold_builtin_memchr (gsi);
5435	case BUILT_IN_FPUTS:
5436	return gimple_fold_builtin_fputs (gsi, arg0: gimple_call_arg (gs: stmt, index: 0),
5437	arg1: gimple_call_arg (gs: stmt, index: 1), unlocked: false);
5438	case BUILT_IN_FPUTS_UNLOCKED:
5439	return gimple_fold_builtin_fputs (gsi, arg0: gimple_call_arg (gs: stmt, index: 0),
5440	arg1: gimple_call_arg (gs: stmt, index: 1), unlocked: true);
5441	case BUILT_IN_MEMCPY_CHK:
5442	case BUILT_IN_MEMPCPY_CHK:
5443	case BUILT_IN_MEMMOVE_CHK:
5444	case BUILT_IN_MEMSET_CHK:
5445	return gimple_fold_builtin_memory_chk (gsi,
5446	dest: gimple_call_arg (gs: stmt, index: 0),
5447	src: gimple_call_arg (gs: stmt, index: 1),
5448	len: gimple_call_arg (gs: stmt, index: 2),
5449	size: gimple_call_arg (gs: stmt, index: 3),
5450	fcode);
5451	case BUILT_IN_STPCPY:
5452	return gimple_fold_builtin_stpcpy (gsi);
5453	case BUILT_IN_STRCPY_CHK:
5454	case BUILT_IN_STPCPY_CHK:
5455	return gimple_fold_builtin_stxcpy_chk (gsi,
5456	dest: gimple_call_arg (gs: stmt, index: 0),
5457	src: gimple_call_arg (gs: stmt, index: 1),
5458	size: gimple_call_arg (gs: stmt, index: 2),
5459	fcode);
5460	case BUILT_IN_STRNCPY_CHK:
5461	case BUILT_IN_STPNCPY_CHK:
5462	return gimple_fold_builtin_stxncpy_chk (gsi,
5463	dest: gimple_call_arg (gs: stmt, index: 0),
5464	src: gimple_call_arg (gs: stmt, index: 1),
5465	len: gimple_call_arg (gs: stmt, index: 2),
5466	size: gimple_call_arg (gs: stmt, index: 3),
5467	fcode);
5468	case BUILT_IN_SNPRINTF_CHK:
5469	case BUILT_IN_VSNPRINTF_CHK:
5470	return gimple_fold_builtin_snprintf_chk (gsi, fcode);
5471
5472	case BUILT_IN_FPRINTF:
5473	case BUILT_IN_FPRINTF_UNLOCKED:
5474	case BUILT_IN_VFPRINTF:
5475	if (n == 2 || n == 3)
5476	return gimple_fold_builtin_fprintf (gsi,
5477	fp: gimple_call_arg (gs: stmt, index: 0),
5478	fmt: gimple_call_arg (gs: stmt, index: 1),
5479	arg: n == 3
5480	? gimple_call_arg (gs: stmt, index: 2)
5481	: NULL_TREE,
5482	fcode);
5483	break;
5484	case BUILT_IN_FPRINTF_CHK:
5485	case BUILT_IN_VFPRINTF_CHK:
5486	if (n == 3 || n == 4)
5487	return gimple_fold_builtin_fprintf (gsi,
5488	fp: gimple_call_arg (gs: stmt, index: 0),
5489	fmt: gimple_call_arg (gs: stmt, index: 2),
5490	arg: n == 4
5491	? gimple_call_arg (gs: stmt, index: 3)
5492	: NULL_TREE,
5493	fcode);
5494	break;
5495	case BUILT_IN_PRINTF:
5496	case BUILT_IN_PRINTF_UNLOCKED:
5497	case BUILT_IN_VPRINTF:
5498	if (n == 1 || n == 2)
5499	return gimple_fold_builtin_printf (gsi, fmt: gimple_call_arg (gs: stmt, index: 0),
5500	arg: n == 2
5501	? gimple_call_arg (gs: stmt, index: 1)
5502	: NULL_TREE, fcode);
5503	break;
5504	case BUILT_IN_PRINTF_CHK:
5505	case BUILT_IN_VPRINTF_CHK:
5506	if (n == 2 || n == 3)
5507	return gimple_fold_builtin_printf (gsi, fmt: gimple_call_arg (gs: stmt, index: 1),
5508	arg: n == 3
5509	? gimple_call_arg (gs: stmt, index: 2)
5510	: NULL_TREE, fcode);
5511	break;
5512	case BUILT_IN_ACC_ON_DEVICE:
5513	return gimple_fold_builtin_acc_on_device (gsi,
5514	arg0: gimple_call_arg (gs: stmt, index: 0));
5515	case BUILT_IN_OMP_IS_INITIAL_DEVICE:
5516	return gimple_fold_builtin_omp_is_initial_device (gsi);
5517
5518	case BUILT_IN_OMP_GET_INITIAL_DEVICE:
5519	return gimple_fold_builtin_omp_get_initial_device (gsi);
5520
5521	case BUILT_IN_OMP_GET_NUM_DEVICES:
5522	return gimple_fold_builtin_omp_get_num_devices (gsi);
5523
5524	case BUILT_IN_REALLOC:
5525	return gimple_fold_builtin_realloc (gsi);
5526
5527	case BUILT_IN_CLEAR_PADDING:
5528	return gimple_fold_builtin_clear_padding (gsi);
5529
5530	case BUILT_IN_CONSTANT_P:
5531	return gimple_fold_builtin_constant_p (gsi);
5532
5533	default:;
5534	}
5535
5536	/* Try the generic builtin folder. */
5537	bool ignore = (gimple_call_lhs (gs: stmt) == NULL);
5538	tree result = fold_call_stmt (stmt, ignore);
5539	if (result)
5540	{
5541	if (ignore)
5542	STRIP_NOPS (result);
5543	else
5544	result = fold_convert (gimple_call_return_type (stmt), result);
5545	gimplify_and_update_call_from_tree (si_p: gsi, expr: result);
5546	return true;
5547	}
5548
5549	return false;
5550	}
5551
5552	/* Transform IFN_GOACC_DIM_SIZE and IFN_GOACC_DIM_POS internal
5553	function calls to constants, where possible. */
5554
5555	static tree
5556	fold_internal_goacc_dim (const gimple *call)
5557	{
5558	int axis = oacc_get_ifn_dim_arg (stmt: call);
5559	int size = oacc_get_fn_dim_size (fn: current_function_decl, axis);
5560	tree result = NULL_TREE;
5561	tree type = TREE_TYPE (gimple_call_lhs (call));
5562
5563	switch (gimple_call_internal_fn (gs: call))
5564	{
5565	case IFN_GOACC_DIM_POS:
5566	/* If the size is 1, we know the answer. */
5567	if (size == 1)
5568	result = build_int_cst (type, 0);
5569	break;
5570	case IFN_GOACC_DIM_SIZE:
5571	/* If the size is not dynamic, we know the answer. */
5572	if (size)
5573	result = build_int_cst (type, size);
5574	break;
5575	default:
5576	break;
5577	}
5578
5579	return result;
5580	}
5581
5582	/* Return true if stmt is __atomic_compare_exchange_N call which is suitable
5583	for conversion into ATOMIC_COMPARE_EXCHANGE if the second argument is
5584	&var where var is only addressable because of such calls. */
5585
5586	bool
5587	optimize_atomic_compare_exchange_p (gimple *stmt)
5588	{
5589	if (gimple_call_num_args (gs: stmt) != 6
5590	|| !flag_inline_atomics
5591	|| !optimize
5592	|| sanitize_flags_p (flag: SANITIZE_THREAD | SANITIZE_ADDRESS)
5593	|| !gimple_call_builtin_p (stmt, BUILT_IN_NORMAL)
5594	|| !gimple_vdef (g: stmt)
5595	|| !gimple_vuse (g: stmt))
5596	return false;
5597
5598	tree fndecl = gimple_call_fndecl (gs: stmt);
5599	switch (DECL_FUNCTION_CODE (decl: fndecl))
5600	{
5601	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
5602	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
5603	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
5604	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
5605	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
5606	break;
5607	default:
5608	return false;
5609	}
5610
5611	tree expected = gimple_call_arg (gs: stmt, index: 1);
5612	if (TREE_CODE (expected) != ADDR_EXPR
5613	|| !SSA_VAR_P (TREE_OPERAND (expected, 0)))
5614	return false;
5615
5616	tree etype = TREE_TYPE (TREE_OPERAND (expected, 0));
5617	if (!is_gimple_reg_type (type: etype)
5618	|| !auto_var_in_fn_p (TREE_OPERAND (expected, 0), current_function_decl)
5619	|| TREE_THIS_VOLATILE (etype)
5620	|| VECTOR_TYPE_P (etype)
5621	|| TREE_CODE (etype) == COMPLEX_TYPE
5622	/* Don't optimize floating point expected vars, VIEW_CONVERT_EXPRs
5623	might not preserve all the bits. See PR71716. */
5624	|| SCALAR_FLOAT_TYPE_P (etype)
5625	|| maybe_ne (TYPE_PRECISION (etype),
5626	b: GET_MODE_BITSIZE (TYPE_MODE (etype))))
5627	return false;
5628
5629	tree weak = gimple_call_arg (gs: stmt, index: 3);
5630	if (!integer_zerop (weak) && !integer_onep (weak))
5631	return false;
5632
5633	tree parmt = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
5634	tree itype = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (parmt)));
5635	machine_mode mode = TYPE_MODE (itype);
5636
5637	if (direct_optab_handler (op: atomic_compare_and_swap_optab, mode)
5638	== CODE_FOR_nothing
5639	&& optab_handler (op: sync_compare_and_swap_optab, mode) == CODE_FOR_nothing)
5640	return false;
5641
5642	if (maybe_ne (a: int_size_in_bytes (etype), b: GET_MODE_SIZE (mode)))
5643	return false;
5644
5645	return true;
5646	}
5647
5648	/* Fold
5649	r = __atomic_compare_exchange_N (p, &e, d, w, s, f);
5650	into
5651	_Complex uintN_t t = ATOMIC_COMPARE_EXCHANGE (p, e, d, w * 256 + N, s, f);
5652	i = IMAGPART_EXPR <t>;
5653	r = (_Bool) i;
5654	e = REALPART_EXPR <t>; */
5655
5656	void
5657	fold_builtin_atomic_compare_exchange (gimple_stmt_iterator *gsi)
5658	{
5659	gimple *stmt = gsi_stmt (i: *gsi);
5660	tree fndecl = gimple_call_fndecl (gs: stmt);
5661	tree parmt = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
5662	tree itype = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (parmt)));
5663	tree ctype = build_complex_type (itype);
5664	tree expected = TREE_OPERAND (gimple_call_arg (stmt, 1), 0);
5665	bool throws = false;
5666	edge e = NULL;
5667	gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (expected)),
5668	expected);
5669	gsi_insert_before (gsi, g, GSI_SAME_STMT);
5670	gimple_stmt_iterator gsiret = gsi_for_stmt (g);
5671	if (!useless_type_conversion_p (itype, TREE_TYPE (expected)))
5672	{
5673	g = gimple_build_assign (make_ssa_name (var: itype), VIEW_CONVERT_EXPR,
5674	build1 (VIEW_CONVERT_EXPR, itype,
5675	gimple_assign_lhs (gs: g)));
5676	gsi_insert_before (gsi, g, GSI_SAME_STMT);
5677	}
5678	int flag = (integer_onep (gimple_call_arg (gs: stmt, index: 3)) ? 256 : 0)
5679	+ int_size_in_bytes (itype);
5680	g = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6,
5681	gimple_call_arg (gs: stmt, index: 0),
5682	gimple_assign_lhs (gs: g),
5683	gimple_call_arg (gs: stmt, index: 2),
5684	build_int_cst (integer_type_node, flag),
5685	gimple_call_arg (gs: stmt, index: 4),
5686	gimple_call_arg (gs: stmt, index: 5));
5687	tree lhs = make_ssa_name (var: ctype);
5688	gimple_call_set_lhs (gs: g, lhs);
5689	gimple_move_vops (g, stmt);
5690	tree oldlhs = gimple_call_lhs (gs: stmt);
5691	if (stmt_can_throw_internal (cfun, stmt))
5692	{
5693	throws = true;
5694	e = find_fallthru_edge (edges: gsi_bb (i: *gsi)->succs);
5695	}
5696	gimple_call_set_nothrow (s: as_a <gcall *> (p: g),
5697	nothrow_p: gimple_call_nothrow_p (s: as_a <gcall *> (p: stmt)));
5698	gimple_call_set_lhs (gs: stmt, NULL_TREE);
5699	gsi_replace (gsi, g, true);
5700	if (oldlhs)
5701	{
5702	g = gimple_build_assign (make_ssa_name (var: itype), IMAGPART_EXPR,
5703	build1 (IMAGPART_EXPR, itype, lhs));
5704	if (throws)
5705	{
5706	gsi_insert_on_edge_immediate (e, g);
5707	*gsi = gsi_for_stmt (g);
5708	}
5709	else
5710	gsi_insert_after (gsi, g, GSI_NEW_STMT);
5711	g = gimple_build_assign (oldlhs, NOP_EXPR, gimple_assign_lhs (gs: g));
5712	gsi_insert_after (gsi, g, GSI_NEW_STMT);
5713	}
5714	g = gimple_build_assign (make_ssa_name (var: itype), REALPART_EXPR,
5715	build1 (REALPART_EXPR, itype, lhs));
5716	if (throws && oldlhs == NULL_TREE)
5717	{
5718	gsi_insert_on_edge_immediate (e, g);
5719	*gsi = gsi_for_stmt (g);
5720	}
5721	else
5722	gsi_insert_after (gsi, g, GSI_NEW_STMT);
5723	if (!useless_type_conversion_p (TREE_TYPE (expected), itype))
5724	{
5725	g = gimple_build_assign (make_ssa_name (TREE_TYPE (expected)),
5726	VIEW_CONVERT_EXPR,
5727	build1 (VIEW_CONVERT_EXPR, TREE_TYPE (expected),
5728	gimple_assign_lhs (gs: g)));
5729	gsi_insert_after (gsi, g, GSI_NEW_STMT);
5730	}
5731	g = gimple_build_assign (expected, SSA_NAME, gimple_assign_lhs (gs: g));
5732	gsi_insert_after (gsi, g, GSI_NEW_STMT);
5733	*gsi = gsiret;
5734	}
5735
5736	/* Return true if ARG0 CODE ARG1 in infinite signed precision operation
5737	doesn't fit into TYPE. The test for overflow should be regardless of
5738	-fwrapv, and even for unsigned types. */
5739
5740	bool
5741	arith_overflowed_p (enum tree_code code, const_tree type,
5742	const_tree arg0, const_tree arg1)
5743	{
5744	widest2_int warg0 = widest2_int_cst (arg0);
5745	widest2_int warg1 = widest2_int_cst (arg1);
5746	widest2_int wres;
5747	switch (code)
5748	{
5749	case PLUS_EXPR: wres = wi::add (x: warg0, y: warg1); break;
5750	case MINUS_EXPR: wres = wi::sub (x: warg0, y: warg1); break;
5751	case MULT_EXPR: wres = wi::mul (x: warg0, y: warg1); break;
5752	default: gcc_unreachable ();
5753	}
5754	signop sign = TYPE_SIGN (type);
5755	if (sign == UNSIGNED && wi::neg_p (x: wres))
5756	return true;
5757	return wi::min_precision (x: wres, sgn: sign) > TYPE_PRECISION (type);
5758	}
5759
5760	/* Mask state for partial load/store operations (mask and length). */
5761	enum mask_load_store_state {
5762	MASK_ALL_INACTIVE, /* All lanes/elements are inactive (can be elided). */
5763	MASK_ALL_ACTIVE, /* All lanes/elements are active (unconditional). */
5764	MASK_UNKNOWN
5765	};
5766
5767	/* Check the mask/length state of IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL.
5768	Returns whether all elements are active, all inactive, or mixed.
5769	VECTYPE is the vector type of the operation. */
5770
5771	static enum mask_load_store_state
5772	partial_load_store_mask_state (gcall *call, tree vectype)
5773	{
5774	internal_fn ifn = gimple_call_internal_fn (gs: call);
5775	int mask_index = internal_fn_mask_index (ifn);
5776	int len_index = internal_fn_len_index (ifn);
5777
5778	/* Extract length and mask arguments up front. */
5779	tree len = len_index != -1 ? gimple_call_arg (gs: call, index: len_index) : NULL_TREE;
5780	tree bias = len ? gimple_call_arg (gs: call, index: len_index + 1) : NULL_TREE;
5781	tree mask = mask_index != -1 ? gimple_call_arg (gs: call, index: mask_index) : NULL_TREE;
5782
5783	poly_int64 nelts = GET_MODE_NUNITS (TYPE_MODE (vectype));
5784
5785	poly_widest_int wlen = -1;
5786	bool full_length_p = !len; /* No length means full length. */
5787
5788	/* Compute effective length. */
5789	if (len && poly_int_tree_p (t: len))
5790	{
5791	gcc_assert (TREE_CODE (bias) == INTEGER_CST);
5792	wlen = wi::to_poly_widest (t: len) + wi::to_widest (t: bias);
5793
5794	if (known_eq (wlen, 0))
5795	return MASK_ALL_INACTIVE;
5796
5797	if (known_eq (wlen, nelts))
5798	full_length_p = true;
5799	else
5800	full_length_p = false;
5801	}
5802
5803	/* Check mask for early return cases. */
5804	if (mask)
5805	{
5806	if (integer_zerop (mask))
5807	return MASK_ALL_INACTIVE;
5808
5809	if (full_length_p && integer_all_onesp (mask))
5810	return MASK_ALL_ACTIVE;
5811	}
5812	else if (full_length_p)
5813	/* No mask and full length means all active. */
5814	return MASK_ALL_ACTIVE;
5815
5816	/* For VLA vectors, we can't do much more. */
5817	if (!nelts.is_constant ())
5818	return MASK_UNKNOWN;
5819
5820	/* Same for VLS vectors with non-constant mask. */
5821	if (mask && TREE_CODE (mask) != VECTOR_CST)
5822	return MASK_UNKNOWN;
5823
5824	/* Check VLS vector elements. */
5825	gcc_assert (wlen.is_constant ());
5826
5827	HOST_WIDE_INT active_len = wlen.to_constant ().to_shwi ();
5828	if (active_len == -1)
5829	active_len = nelts.to_constant ();
5830
5831	/* Check if all elements in the active range match the mask. */
5832	for (HOST_WIDE_INT i = 0; i < active_len; i++)
5833	{
5834	bool elt_active = !mask || !integer_zerop (vector_cst_elt (mask, i));
5835	if (!elt_active)
5836	{
5837	/* Found an inactive element. Check if all are inactive. */
5838	for (HOST_WIDE_INT j = 0; j < active_len; j++)
5839	if (!mask || !integer_zerop (vector_cst_elt (mask, j)))
5840	return MASK_UNKNOWN; /* Mixed state. */
5841	return MASK_ALL_INACTIVE;
5842	}
5843	}
5844
5845	/* All elements in active range are active. */
5846	return full_length_p ? MASK_ALL_ACTIVE : MASK_UNKNOWN;
5847	}
5848
5849
5850	/* If IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL is unconditional
5851	(all lanes active), return a MEM_REF for the memory it references.
5852	Otherwise return NULL_TREE. VECTYPE is the type of the memory vector. */
5853
5854	static tree
5855	gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype)
5856	{
5857	/* Only fold if all lanes are active (unconditional). */
5858	if (partial_load_store_mask_state (call, vectype) != MASK_ALL_ACTIVE)
5859	return NULL_TREE;
5860
5861	tree ptr = gimple_call_arg (gs: call, index: 0);
5862	tree alias_align = gimple_call_arg (gs: call, index: 1);
5863	if (!tree_fits_uhwi_p (alias_align))
5864	return NULL_TREE;
5865
5866	unsigned HOST_WIDE_INT align = tree_to_uhwi (alias_align);
5867	if (TYPE_ALIGN (vectype) != align)
5868	vectype = build_aligned_type (vectype, align);
5869	tree offset = build_zero_cst (TREE_TYPE (alias_align));
5870	return fold_build2 (MEM_REF, vectype, ptr, offset);
5871	}
5872
5873	/* Try to fold IFN_{MASK,LEN}_LOAD/STORE call CALL. Return true on success. */
5874
5875	static bool
5876	gimple_fold_partial_load_store (gimple_stmt_iterator *gsi, gcall *call)
5877	{
5878	internal_fn ifn = gimple_call_internal_fn (gs: call);
5879	tree lhs = gimple_call_lhs (gs: call);
5880	bool is_load = (lhs != NULL_TREE);
5881	tree vectype;
5882
5883	if (is_load)
5884	vectype = TREE_TYPE (lhs);
5885	else
5886	{
5887	tree rhs = gimple_call_arg (gs: call, index: internal_fn_stored_value_index (ifn));
5888	vectype = TREE_TYPE (rhs);
5889	}
5890
5891	enum mask_load_store_state state
5892	= partial_load_store_mask_state (call, vectype);
5893
5894	/* Handle all-inactive case. */
5895	if (state == MASK_ALL_INACTIVE)
5896	{
5897	if (is_load)
5898	{
5899	/* Replace load with else value. */
5900	int else_index = internal_fn_else_index (ifn);
5901	tree else_value = gimple_call_arg (gs: call, index: else_index);
5902	if (!is_gimple_reg (lhs))
5903	{
5904	if (!zerop (else_value))
5905	return false;
5906	else_value = build_constructor (TREE_TYPE (lhs), NULL);
5907	}
5908	gassign *new_stmt = gimple_build_assign (lhs, else_value);
5909	gimple_set_location (g: new_stmt, location: gimple_location (g: call));
5910	/* When the lhs is an array for LANES version, then there is still
5911	a store, move the vops from the old stmt to the new one. */
5912	if (!is_gimple_reg (lhs))
5913	gimple_move_vops (new_stmt, call);
5914	gsi_replace (gsi, new_stmt, false);
5915	return true;
5916	}
5917	else
5918	{
5919	/* Remove inactive store altogether. */
5920	unlink_stmt_vdef (call);
5921	release_defs (call);
5922	gsi_replace (gsi, gimple_build_nop (), true);
5923	return true;
5924	}
5925	}
5926
5927	/* We cannot simplify a gather/scatter or load/store lanes further. */
5928	if (internal_gather_scatter_fn_p (ifn)
5929	|| TREE_CODE (vectype) == ARRAY_TYPE)
5930	return false;
5931
5932	/* Handle all-active case by folding to regular memory operation. */
5933	if (tree mem_ref = gimple_fold_partial_load_store_mem_ref (call, vectype))
5934	{
5935	gassign *new_stmt;
5936	if (is_load)
5937	new_stmt = gimple_build_assign (lhs, mem_ref);
5938	else
5939	{
5940	tree rhs
5941	= gimple_call_arg (gs: call, index: internal_fn_stored_value_index (ifn));
5942	new_stmt = gimple_build_assign (mem_ref, rhs);
5943	}
5944
5945	gimple_set_location (g: new_stmt, location: gimple_location (g: call));
5946	gimple_move_vops (new_stmt, call);
5947	gsi_replace (gsi, new_stmt, false);
5948	return true;
5949	}
5950	return false;
5951	}
5952
5953	/* Attempt to fold a call statement referenced by the statement iterator GSI.
5954	The statement may be replaced by another statement, e.g., if the call
5955	simplifies to a constant value. Return true if any changes were made.
5956	It is assumed that the operands have been previously folded. */
5957
5958	static bool
5959	gimple_fold_call (gimple_stmt_iterator *gsi, bool inplace)
5960	{
5961	gcall *stmt = as_a <gcall *> (p: gsi_stmt (i: *gsi));
5962	tree callee;
5963	bool changed = false;
5964
5965	/* Check for virtual calls that became direct calls. */
5966	callee = gimple_call_fn (gs: stmt);
5967	if (callee && TREE_CODE (callee) == OBJ_TYPE_REF)
5968	{
5969	if (gimple_call_addr_fndecl (OBJ_TYPE_REF_EXPR (callee)) != NULL_TREE)
5970	{
5971	if (dump_file && virtual_method_call_p (callee)
5972	&& !possible_polymorphic_call_target_p
5973	(ref: callee, stmt, n: cgraph_node::get (decl: gimple_call_addr_fndecl
5974	(OBJ_TYPE_REF_EXPR (callee)))))
5975	{
5976	fprintf (stream: dump_file,
5977	format: "Type inheritance inconsistent devirtualization of ");
5978	print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
5979	fprintf (stream: dump_file, format: " to ");
5980	print_generic_expr (dump_file, callee, TDF_SLIM);
5981	fprintf (stream: dump_file, format: "\n");
5982	}
5983
5984	gimple_call_set_fn (gs: stmt, OBJ_TYPE_REF_EXPR (callee));
5985	changed = true;
5986	}
5987	else if (flag_devirtualize && !inplace && virtual_method_call_p (callee))
5988	{
5989	bool final;
5990	vec <cgraph_node *>targets
5991	= possible_polymorphic_call_targets (ref: callee, call: stmt, completep: &final);
5992	if (final && targets.length () <= 1 && dbg_cnt (index: devirt))
5993	{
5994	tree lhs = gimple_call_lhs (gs: stmt);
5995	if (dump_enabled_p ())
5996	{
5997	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
5998	"folding virtual function call to %s\n",
5999	targets.length () == 1
6000	? targets[0]->name ()
6001	: "__builtin_unreachable");
6002	}
6003	if (targets.length () == 1)
6004	{
6005	tree fndecl = targets[0]->decl;
6006	gimple_call_set_fndecl (gs: stmt, decl: fndecl);
6007	changed = true;
6008	/* If changing the call to __cxa_pure_virtual
6009	or similar noreturn function, adjust gimple_call_fntype
6010	too. */
6011	if (gimple_call_noreturn_p (s: stmt)
6012	&& VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl)))
6013	&& TYPE_ARG_TYPES (TREE_TYPE (fndecl))
6014	&& (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
6015	== void_type_node))
6016	gimple_call_set_fntype (call_stmt: stmt, TREE_TYPE (fndecl));
6017	/* If the call becomes noreturn, remove the lhs. */
6018	if (lhs
6019	&& gimple_call_noreturn_p (s: stmt)
6020	&& (VOID_TYPE_P (TREE_TYPE (gimple_call_fntype (stmt)))
6021	|| should_remove_lhs_p (lhs)))
6022	{
6023	if (TREE_CODE (lhs) == SSA_NAME)
6024	{
6025	tree var = create_tmp_var (TREE_TYPE (lhs));
6026	tree def = get_or_create_ssa_default_def (cfun, var);
6027	gimple *new_stmt = gimple_build_assign (lhs, def);
6028	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6029	}
6030	gimple_call_set_lhs (gs: stmt, NULL_TREE);
6031	}
6032	maybe_remove_unused_call_args (cfun, stmt);
6033	}
6034	else
6035	{
6036	location_t loc = gimple_location (g: stmt);
6037	gimple *new_stmt = gimple_build_builtin_unreachable (loc);
6038	gimple_call_set_ctrl_altering (s: new_stmt, ctrl_altering_p: false);
6039	/* If the call had a SSA name as lhs morph that into
6040	an uninitialized value. */
6041	if (lhs && TREE_CODE (lhs) == SSA_NAME)
6042	{
6043	tree var = create_tmp_var (TREE_TYPE (lhs));
6044	SET_SSA_NAME_VAR_OR_IDENTIFIER (lhs, var);
6045	SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
6046	set_ssa_default_def (cfun, var, lhs);
6047	}
6048	gimple_move_vops (new_stmt, stmt);
6049	gsi_replace (gsi, new_stmt, false);
6050	return true;
6051	}
6052	}
6053	}
6054	}
6055
6056	/* Check for indirect calls that became direct calls, and then
6057	no longer require a static chain. */
6058	if (gimple_call_chain (gs: stmt))
6059	{
6060	tree fn = gimple_call_fndecl (gs: stmt);
6061	if (fn && !DECL_STATIC_CHAIN (fn))
6062	{
6063	gimple_call_set_chain (call_stmt: stmt, NULL);
6064	changed = true;
6065	}
6066	}
6067
6068	if (inplace)
6069	return changed;
6070
6071	/* Check for builtins that CCP can handle using information not
6072	available in the generic fold routines. */
6073	if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
6074	{
6075	if (gimple_fold_builtin (gsi))
6076	changed = true;
6077	}
6078	else if (gimple_call_builtin_p (stmt, BUILT_IN_MD))
6079	{
6080	changed |= targetm.gimple_fold_builtin (gsi);
6081	}
6082	else if (gimple_call_internal_p (gs: stmt))
6083	{
6084	enum tree_code subcode = ERROR_MARK;
6085	tree result = NULL_TREE;
6086	bool cplx_result = false;
6087	bool uaddc_usubc = false;
6088	tree overflow = NULL_TREE;
6089	switch (gimple_call_internal_fn (gs: stmt))
6090	{
6091	case IFN_ASSUME:
6092	/* Remove .ASSUME calls during the last fold since it is no
6093	longer needed. */
6094	if (fold_before_rtl_expansion_p ())
6095	replace_call_with_value (gsi, NULL_TREE);
6096	break;
6097	case IFN_BUILTIN_EXPECT:
6098	result = fold_builtin_expect (gimple_location (g: stmt),
6099	gimple_call_arg (gs: stmt, index: 0),
6100	gimple_call_arg (gs: stmt, index: 1),
6101	gimple_call_arg (gs: stmt, index: 2),
6102	NULL_TREE);
6103	break;
6104	case IFN_UBSAN_OBJECT_SIZE:
6105	{
6106	tree offset = gimple_call_arg (gs: stmt, index: 1);
6107	tree objsize = gimple_call_arg (gs: stmt, index: 2);
6108	if (integer_all_onesp (objsize)
6109	|| (TREE_CODE (offset) == INTEGER_CST
6110	&& TREE_CODE (objsize) == INTEGER_CST
6111	&& tree_int_cst_le (t1: offset, t2: objsize)))
6112	{
6113	replace_call_with_value (gsi, NULL_TREE);
6114	return true;
6115	}
6116	}
6117	break;
6118	case IFN_UBSAN_PTR:
6119	if (integer_zerop (gimple_call_arg (gs: stmt, index: 1)))
6120	{
6121	replace_call_with_value (gsi, NULL_TREE);
6122	return true;
6123	}
6124	break;
6125	case IFN_UBSAN_BOUNDS:
6126	{
6127	tree index = gimple_call_arg (gs: stmt, index: 1);
6128	tree bound = gimple_call_arg (gs: stmt, index: 2);
6129	if (TREE_CODE (index) == INTEGER_CST
6130	&& TREE_CODE (bound) == INTEGER_CST)
6131	{
6132	index = fold_convert (TREE_TYPE (bound), index);
6133	if (TREE_CODE (index) == INTEGER_CST
6134	&& tree_int_cst_lt (t1: index, t2: bound))
6135	{
6136	replace_call_with_value (gsi, NULL_TREE);
6137	return true;
6138	}
6139	}
6140	}
6141	break;
6142	case IFN_GOACC_DIM_SIZE:
6143	case IFN_GOACC_DIM_POS:
6144	result = fold_internal_goacc_dim (call: stmt);
6145	break;
6146	case IFN_UBSAN_CHECK_ADD:
6147	subcode = PLUS_EXPR;
6148	break;
6149	case IFN_UBSAN_CHECK_SUB:
6150	subcode = MINUS_EXPR;
6151	break;
6152	case IFN_UBSAN_CHECK_MUL:
6153	subcode = MULT_EXPR;
6154	break;
6155	case IFN_ADD_OVERFLOW:
6156	subcode = PLUS_EXPR;
6157	cplx_result = true;
6158	break;
6159	case IFN_SUB_OVERFLOW:
6160	subcode = MINUS_EXPR;
6161	cplx_result = true;
6162	break;
6163	case IFN_MUL_OVERFLOW:
6164	subcode = MULT_EXPR;
6165	cplx_result = true;
6166	break;
6167	case IFN_UADDC:
6168	subcode = PLUS_EXPR;
6169	cplx_result = true;
6170	uaddc_usubc = true;
6171	break;
6172	case IFN_USUBC:
6173	subcode = MINUS_EXPR;
6174	cplx_result = true;
6175	uaddc_usubc = true;
6176	break;
6177	case IFN_LEN_LOAD:
6178	case IFN_MASK_LOAD:
6179	case IFN_MASK_LEN_LOAD:
6180	case IFN_MASK_GATHER_LOAD:
6181	case IFN_MASK_LEN_GATHER_LOAD:
6182	case IFN_MASK_LOAD_LANES:
6183	case IFN_MASK_LEN_LOAD_LANES:
6184	case IFN_LEN_STORE:
6185	case IFN_MASK_STORE:
6186	case IFN_MASK_LEN_STORE:
6187	case IFN_MASK_SCATTER_STORE:
6188	case IFN_MASK_LEN_SCATTER_STORE:
6189	case IFN_MASK_STORE_LANES:
6190	case IFN_MASK_LEN_STORE_LANES:
6191	changed |= gimple_fold_partial_load_store (gsi, call: stmt);
6192	break;
6193	default:
6194	break;
6195	}
6196	if (subcode != ERROR_MARK)
6197	{
6198	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
6199	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
6200	tree arg2 = NULL_TREE;
6201	tree type = TREE_TYPE (arg0);
6202	if (cplx_result)
6203	{
6204	tree lhs = gimple_call_lhs (gs: stmt);
6205	if (lhs == NULL_TREE)
6206	type = NULL_TREE;
6207	else
6208	type = TREE_TYPE (TREE_TYPE (lhs));
6209	if (uaddc_usubc)
6210	arg2 = gimple_call_arg (gs: stmt, index: 2);
6211	}
6212	if (type == NULL_TREE)
6213	;
6214	else if (uaddc_usubc)
6215	{
6216	if (!integer_zerop (arg2))
6217	;
6218	/* x = y + 0 + 0; x = y - 0 - 0; */
6219	else if (integer_zerop (arg1))
6220	result = arg0;
6221	/* x = 0 + y + 0; */
6222	else if (subcode != MINUS_EXPR && integer_zerop (arg0))
6223	result = arg1;
6224	/* x = y - y - 0; */
6225	else if (subcode == MINUS_EXPR
6226	&& operand_equal_p (arg0, arg1, flags: 0))
6227	result = integer_zero_node;
6228	}
6229	/* x = y + 0; x = y - 0; x = y * 0; */
6230	else if (integer_zerop (arg1))
6231	result = subcode == MULT_EXPR ? integer_zero_node : arg0;
6232	/* x = 0 + y; x = 0 * y; */
6233	else if (subcode != MINUS_EXPR && integer_zerop (arg0))
6234	result = subcode == MULT_EXPR ? integer_zero_node : arg1;
6235	/* x = y - y; */
6236	else if (subcode == MINUS_EXPR && operand_equal_p (arg0, arg1, flags: 0))
6237	result = integer_zero_node;
6238	/* x = y * 1; x = 1 * y; */
6239	else if (subcode == MULT_EXPR && integer_onep (arg1))
6240	result = arg0;
6241	else if (subcode == MULT_EXPR && integer_onep (arg0))
6242	result = arg1;
6243	if (result)
6244	{
6245	if (result == integer_zero_node)
6246	result = build_zero_cst (type);
6247	else if (cplx_result && TREE_TYPE (result) != type)
6248	{
6249	if (TREE_CODE (result) == INTEGER_CST)
6250	{
6251	if (arith_overflowed_p (code: PLUS_EXPR, type, arg0: result,
6252	integer_zero_node))
6253	overflow = build_one_cst (type);
6254	}
6255	else if ((!TYPE_UNSIGNED (TREE_TYPE (result))
6256	&& TYPE_UNSIGNED (type))
6257	|| (TYPE_PRECISION (type)
6258	< (TYPE_PRECISION (TREE_TYPE (result))
6259	+ (TYPE_UNSIGNED (TREE_TYPE (result))
6260	&& !TYPE_UNSIGNED (type)))))
6261	result = NULL_TREE;
6262	if (result)
6263	result = fold_convert (type, result);
6264	}
6265	}
6266	}
6267
6268	if (result)
6269	{
6270	if (TREE_CODE (result) == INTEGER_CST && TREE_OVERFLOW (result))
6271	result = drop_tree_overflow (result);
6272	if (cplx_result)
6273	{
6274	if (overflow == NULL_TREE)
6275	overflow = build_zero_cst (TREE_TYPE (result));
6276	tree ctype = build_complex_type (TREE_TYPE (result));
6277	if (TREE_CODE (result) == INTEGER_CST
6278	&& TREE_CODE (overflow) == INTEGER_CST)
6279	result = build_complex (ctype, result, overflow);
6280	else
6281	result = build2_loc (loc: gimple_location (g: stmt), code: COMPLEX_EXPR,
6282	type: ctype, arg0: result, arg1: overflow);
6283	}
6284	gimplify_and_update_call_from_tree (si_p: gsi, expr: result);
6285	changed = true;
6286	}
6287	}
6288
6289	return changed;
6290	}
6291
6292
6293	/* Return true whether NAME has a use on STMT. Note this can return
6294	false even though there's a use on STMT if SSA operands are not
6295	up-to-date. */
6296
6297	static bool
6298	has_use_on_stmt (tree name, gimple *stmt)
6299	{
6300	ssa_op_iter iter;
6301	tree op;
6302	FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
6303	if (op == name)
6304	return true;
6305	return false;
6306	}
6307
6308	/* Add the lhs of each statement of SEQ to DCE_WORKLIST. */
6309
6310	void
6311	mark_lhs_in_seq_for_dce (bitmap dce_worklist, gimple_seq seq)
6312	{
6313	if (!dce_worklist)
6314	return;
6315
6316	for (gimple_stmt_iterator i = gsi_start (seq);
6317	!gsi_end_p (i); gsi_next (i: &i))
6318	{
6319	gimple *stmt = gsi_stmt (i);
6320	tree name = gimple_get_lhs (stmt);
6321	if (name && TREE_CODE (name) == SSA_NAME)
6322	bitmap_set_bit (dce_worklist, SSA_NAME_VERSION (name));
6323	}
6324	}
6325
6326	/* Worker for fold_stmt_1 dispatch to pattern based folding with
6327	gimple_simplify.
6328
6329	Replaces *GSI with the simplification result in RCODE and OPS
6330	and the associated statements in *SEQ. Does the replacement
6331	according to INPLACE and returns true if the operation succeeded. */
6332
6333	static bool
6334	replace_stmt_with_simplification (gimple_stmt_iterator *gsi,
6335	gimple_match_op *res_op,
6336	gimple_seq *seq, bool inplace,
6337	bitmap dce_worklist)
6338	{
6339	gimple *stmt = gsi_stmt (i: *gsi);
6340	tree *ops = res_op->ops;
6341	unsigned int num_ops = res_op->num_ops;
6342
6343	/* Play safe and do not allow abnormals to be mentioned in
6344	newly created statements. See also maybe_push_res_to_seq.
6345	As an exception allow such uses if there was a use of the
6346	same SSA name on the old stmt. */
6347	for (unsigned int i = 0; i < num_ops; ++i)
6348	if (TREE_CODE (ops[i]) == SSA_NAME
6349	&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ops[i])
6350	&& !has_use_on_stmt (name: ops[i], stmt))
6351	return false;
6352
6353	if (num_ops > 0 && COMPARISON_CLASS_P (ops[0]))
6354	for (unsigned int i = 0; i < 2; ++i)
6355	if (TREE_CODE (TREE_OPERAND (ops[0], i)) == SSA_NAME
6356	&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (ops[0], i))
6357	&& !has_use_on_stmt (TREE_OPERAND (ops[0], i), stmt))
6358	return false;
6359
6360	/* Don't insert new statements when INPLACE is true, even if we could
6361	reuse STMT for the final statement. */
6362	if (inplace && !gimple_seq_empty_p (s: *seq))
6363	return false;
6364
6365	if (gcond *cond_stmt = dyn_cast <gcond *> (p: stmt))
6366	{
6367	gcc_assert (res_op->code.is_tree_code ());
6368	auto code = tree_code (res_op->code);
6369	if (TREE_CODE_CLASS (code) == tcc_comparison
6370	/* GIMPLE_CONDs condition may not throw. */
6371	&& ((cfun
6372	&& (!flag_exceptions
6373	|| !cfun->can_throw_non_call_exceptions))
6374	|| !operation_could_trap_p (code,
6375	FLOAT_TYPE_P (TREE_TYPE (ops[0])),
6376	false, NULL_TREE)))
6377	gimple_cond_set_condition (stmt: cond_stmt, code, lhs: ops[0], rhs: ops[1]);
6378	else if (code == SSA_NAME)
6379	{
6380	/* If setting the gimple cond to the same thing,
6381	return false as nothing changed. */
6382	if (gimple_cond_code (gs: cond_stmt) == NE_EXPR
6383	&& operand_equal_p (gimple_cond_lhs (gs: cond_stmt), ops[0])
6384	&& integer_zerop (gimple_cond_rhs (gs: cond_stmt)))
6385	return false;
6386	gimple_cond_set_condition (stmt: cond_stmt, code: NE_EXPR, lhs: ops[0],
6387	rhs: build_zero_cst (TREE_TYPE (ops[0])));
6388	}
6389	else if (code == INTEGER_CST)
6390	{
6391	/* Make into the canonical form `1 != 0` and `0 != 0`.
6392	If already in the canonical form return false
6393	saying nothing has been done. */
6394	if (integer_zerop (ops[0]))
6395	{
6396	if (gimple_cond_false_canonical_p (gs: cond_stmt))
6397	return false;
6398	gimple_cond_make_false (gs: cond_stmt);
6399	}
6400	else
6401	{
6402	if (gimple_cond_true_canonical_p (gs: cond_stmt))
6403	return false;
6404	gimple_cond_make_true (gs: cond_stmt);
6405	}
6406	}
6407	else if (!inplace)
6408	{
6409	/* For throwing comparisons, see if the GIMPLE_COND is the same as
6410	the comparison would be.
6411	This can happen due to the match pattern for
6412	`(ne (cmp @0 @1) integer_zerop)` which creates a new expression
6413	for the comparison. */
6414	if (TREE_CODE_CLASS (code) == tcc_comparison
6415	&& (!cfun
6416	|| (flag_exceptions
6417	&& cfun->can_throw_non_call_exceptions))
6418	&& operation_could_trap_p (code,
6419	FLOAT_TYPE_P (TREE_TYPE (ops[0])),
6420	false, NULL_TREE))
6421	{
6422	tree lhs = gimple_cond_lhs (gs: cond_stmt);
6423	if (gimple_cond_code (gs: cond_stmt) == NE_EXPR
6424	&& TREE_CODE (lhs) == SSA_NAME
6425	&& INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6426	&& integer_zerop (gimple_cond_rhs (gs: cond_stmt)))
6427	{
6428	gimple *s = SSA_NAME_DEF_STMT (lhs);
6429	if (is_gimple_assign (gs: s)
6430	&& gimple_assign_rhs_code (gs: s) == code
6431	&& operand_equal_p (gimple_assign_rhs1 (gs: s), ops[0])
6432	&& operand_equal_p (gimple_assign_rhs2 (gs: s), ops[1]))
6433	return false;
6434	}
6435	}
6436	tree res = maybe_push_res_to_seq (res_op, seq);
6437	if (!res)
6438	return false;
6439	gimple_cond_set_condition (stmt: cond_stmt, code: NE_EXPR, lhs: res,
6440	rhs: build_zero_cst (TREE_TYPE (res)));
6441	}
6442	else
6443	return false;
6444	if (dump_file && (dump_flags & TDF_DETAILS))
6445	{
6446	fprintf (stream: dump_file, format: "gimple_simplified to ");
6447	if (!gimple_seq_empty_p (s: *seq))
6448	print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
6449	print_gimple_stmt (dump_file, gsi_stmt (i: *gsi),
6450	0, TDF_SLIM);
6451	}
6452	// Mark the lhs of the new statements maybe for dce
6453	mark_lhs_in_seq_for_dce (dce_worklist, seq: *seq);
6454	gsi_insert_seq_before (gsi, *seq, GSI_SAME_STMT);
6455	return true;
6456	}
6457	else if (is_gimple_assign (gs: stmt)
6458	&& res_op->code.is_tree_code ())
6459	{
6460	auto code = tree_code (res_op->code);
6461	if (!inplace
6462	|| gimple_num_ops (gs: stmt) > get_gimple_rhs_num_ops (code))
6463	{
6464	maybe_build_generic_op (res_op);
6465	gimple_assign_set_rhs_with_ops (gsi, code,
6466	res_op->op_or_null (i: 0),
6467	res_op->op_or_null (i: 1),
6468	res_op->op_or_null (i: 2));
6469	if (dump_file && (dump_flags & TDF_DETAILS))
6470	{
6471	fprintf (stream: dump_file, format: "gimple_simplified to ");
6472	if (!gimple_seq_empty_p (s: *seq))
6473	print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
6474	print_gimple_stmt (dump_file, gsi_stmt (i: *gsi),
6475	0, TDF_SLIM);
6476	}
6477	// Mark the lhs of the new statements maybe for dce
6478	mark_lhs_in_seq_for_dce (dce_worklist, seq: *seq);
6479	gsi_insert_seq_before (gsi, *seq, GSI_SAME_STMT);
6480	return true;
6481	}
6482	}
6483	else if (res_op->code.is_fn_code ()
6484	&& gimple_call_combined_fn (stmt) == combined_fn (res_op->code))
6485	{
6486	gcc_assert (num_ops == gimple_call_num_args (stmt));
6487	for (unsigned int i = 0; i < num_ops; ++i)
6488	gimple_call_set_arg (gs: stmt, index: i, arg: ops[i]);
6489	if (dump_file && (dump_flags & TDF_DETAILS))
6490	{
6491	fprintf (stream: dump_file, format: "gimple_simplified to ");
6492	if (!gimple_seq_empty_p (s: *seq))
6493	print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
6494	print_gimple_stmt (dump_file, gsi_stmt (i: *gsi), 0, TDF_SLIM);
6495	}
6496	// Mark the lhs of the new statements maybe for dce
6497	mark_lhs_in_seq_for_dce (dce_worklist, seq: *seq);
6498	gsi_insert_seq_before (gsi, *seq, GSI_SAME_STMT);
6499	return true;
6500	}
6501	else if (!inplace)
6502	{
6503	if (gimple_has_lhs (stmt))
6504	{
6505	tree lhs = gimple_get_lhs (stmt);
6506	if (!maybe_push_res_to_seq (res_op, seq, res: lhs))
6507	return false;
6508	if (dump_file && (dump_flags & TDF_DETAILS))
6509	{
6510	fprintf (stream: dump_file, format: "gimple_simplified to ");
6511	print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
6512	}
6513	// Mark the lhs of the new statements maybe for dce
6514	mark_lhs_in_seq_for_dce (dce_worklist, seq: *seq);
6515	gsi_replace_with_seq_vops (si_p: gsi, stmts: *seq);
6516	return true;
6517	}
6518	else
6519	gcc_unreachable ();
6520	}
6521
6522	return false;
6523	}
6524
6525	/* Canonicalize MEM_REFs invariant address operand after propagation. */
6526
6527	static bool
6528	maybe_canonicalize_mem_ref_addr (tree *t, bool is_debug = false)
6529	{
6530	bool res = false;
6531	tree *orig_t = t;
6532
6533	if (TREE_CODE (*t) == ADDR_EXPR)
6534	t = &TREE_OPERAND (*t, 0);
6535
6536	/* The C and C++ frontends use an ARRAY_REF for indexing with their
6537	generic vector extension. The actual vector referenced is
6538	view-converted to an array type for this purpose. If the index
6539	is constant the canonical representation in the middle-end is a
6540	BIT_FIELD_REF so re-write the former to the latter here. */
6541	if (TREE_CODE (*t) == ARRAY_REF
6542	&& TREE_CODE (TREE_OPERAND (*t, 0)) == VIEW_CONVERT_EXPR
6543	&& TREE_CODE (TREE_OPERAND (*t, 1)) == INTEGER_CST
6544	&& VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (TREE_OPERAND (*t, 0), 0))))
6545	{
6546	tree vtype = TREE_TYPE (TREE_OPERAND (TREE_OPERAND (*t, 0), 0));
6547	if (VECTOR_TYPE_P (vtype))
6548	{
6549	tree low = array_ref_low_bound (*t);
6550	if (TREE_CODE (low) == INTEGER_CST)
6551	{
6552	if (tree_int_cst_le (t1: low, TREE_OPERAND (*t, 1)))
6553	{
6554	widest_int idx = wi::sub (x: wi::to_widest (TREE_OPERAND (*t, 1)),
6555	y: wi::to_widest (t: low));
6556	idx = wi::mul (x: idx, y: wi::to_widest
6557	(TYPE_SIZE (TREE_TYPE (*t))));
6558	widest_int ext
6559	= wi::add (x: idx, y: wi::to_widest (TYPE_SIZE (TREE_TYPE (*t))));
6560	if (maybe_le (a: ext, b: wi::to_poly_widest (TYPE_SIZE (vtype))))
6561	{
6562	*t = build3_loc (EXPR_LOCATION (*t), code: BIT_FIELD_REF,
6563	TREE_TYPE (*t),
6564	TREE_OPERAND (TREE_OPERAND (*t, 0), 0),
6565	TYPE_SIZE (TREE_TYPE (*t)),
6566	arg2: wide_int_to_tree (bitsizetype, cst: idx));
6567	res = true;
6568	}
6569	}
6570	}
6571	}
6572	}
6573
6574	while (handled_component_p (t: *t))
6575	t = &TREE_OPERAND (*t, 0);
6576
6577	/* Canonicalize MEM [&foo.bar, 0] which appears after propagating
6578	of invariant addresses into a SSA name MEM_REF address. */
6579	if (TREE_CODE (*t) == MEM_REF
6580	|| TREE_CODE (*t) == TARGET_MEM_REF)
6581	{
6582	tree addr = TREE_OPERAND (*t, 0);
6583	if (TREE_CODE (addr) == ADDR_EXPR
6584	&& (TREE_CODE (TREE_OPERAND (addr, 0)) == MEM_REF
6585	|| handled_component_p (TREE_OPERAND (addr, 0))))
6586	{
6587	tree base;
6588	poly_int64 coffset;
6589	base = get_addr_base_and_unit_offset (TREE_OPERAND (addr, 0),
6590	&coffset);
6591	if (!base)
6592	{
6593	if (is_debug)
6594	return false;
6595	gcc_unreachable ();
6596	}
6597
6598	TREE_OPERAND (*t, 0) = build_fold_addr_expr (base);
6599	TREE_OPERAND (*t, 1) = int_const_binop (PLUS_EXPR,
6600	TREE_OPERAND (*t, 1),
6601	size_int (coffset));
6602	res = true;
6603	}
6604	gcc_checking_assert (TREE_CODE (TREE_OPERAND (*t, 0)) == DEBUG_EXPR_DECL
6605	|| is_gimple_mem_ref_addr (TREE_OPERAND (*t, 0)));
6606	}
6607
6608	/* Canonicalize back MEM_REFs to plain reference trees if the object
6609	accessed is a decl that has the same access semantics as the MEM_REF. */
6610	if (TREE_CODE (*t) == MEM_REF
6611	&& TREE_CODE (TREE_OPERAND (*t, 0)) == ADDR_EXPR
6612	&& integer_zerop (TREE_OPERAND (*t, 1))
6613	&& MR_DEPENDENCE_CLIQUE (*t) == 0)
6614	{
6615	tree decl = TREE_OPERAND (TREE_OPERAND (*t, 0), 0);
6616	tree alias_type = TREE_TYPE (TREE_OPERAND (*t, 1));
6617	if (/* Same volatile qualification. */
6618	TREE_THIS_VOLATILE (*t) == TREE_THIS_VOLATILE (decl)
6619	/* Same TBAA behavior with -fstrict-aliasing. */
6620	&& !TYPE_REF_CAN_ALIAS_ALL (alias_type)
6621	&& (TYPE_MAIN_VARIANT (TREE_TYPE (decl))
6622	== TYPE_MAIN_VARIANT (TREE_TYPE (alias_type)))
6623	/* Same alignment. */
6624	&& TYPE_ALIGN (TREE_TYPE (decl)) == TYPE_ALIGN (TREE_TYPE (*t))
6625	/* We have to look out here to not drop a required conversion
6626	from the rhs to the lhs if *t appears on the lhs or vice-versa
6627	if it appears on the rhs. Thus require strict type
6628	compatibility. */
6629	&& types_compatible_p (TREE_TYPE (*t), TREE_TYPE (decl)))
6630	{
6631	*t = TREE_OPERAND (TREE_OPERAND (*t, 0), 0);
6632	res = true;
6633	}
6634	}
6635
6636	else if (TREE_CODE (*orig_t) == ADDR_EXPR
6637	&& TREE_CODE (*t) == MEM_REF
6638	&& TREE_CODE (TREE_OPERAND (*t, 0)) == INTEGER_CST)
6639	{
6640	tree base;
6641	poly_int64 coffset;
6642	base = get_addr_base_and_unit_offset (TREE_OPERAND (*orig_t, 0),
6643	&coffset);
6644	if (base)
6645	{
6646	gcc_assert (TREE_CODE (base) == MEM_REF);
6647	poly_int64 moffset;
6648	if (mem_ref_offset (base).to_shwi (r: &moffset))
6649	{
6650	coffset += moffset;
6651	if (wi::to_poly_wide (TREE_OPERAND (base, 0)).to_shwi (r: &moffset))
6652	{
6653	coffset += moffset;
6654	*orig_t = build_int_cst (TREE_TYPE (*orig_t), coffset);
6655	return true;
6656	}
6657	}
6658	}
6659	}
6660
6661	/* Canonicalize TARGET_MEM_REF in particular with respect to
6662	the indexes becoming constant. */
6663	else if (TREE_CODE (*t) == TARGET_MEM_REF)
6664	{
6665	tree tem = maybe_fold_tmr (*t);
6666	if (tem)
6667	{
6668	*t = tem;
6669	if (TREE_CODE (*orig_t) == ADDR_EXPR)
6670	recompute_tree_invariant_for_addr_expr (*orig_t);
6671	res = true;
6672	}
6673	}
6674
6675	return res;
6676	}
6677
6678	/* Worker for both fold_stmt and fold_stmt_inplace. The INPLACE argument
6679	distinguishes both cases. */
6680
6681	static bool
6682	fold_stmt_1 (gimple_stmt_iterator *gsi, bool inplace, tree (*valueize) (tree),
6683	bitmap dce_worklist = nullptr)
6684	{
6685	bool changed = false;
6686	gimple *stmt = gsi_stmt (i: *gsi);
6687	bool nowarning = warning_suppressed_p (stmt, OPT_Wstrict_overflow);
6688	unsigned i;
6689	fold_defer_overflow_warnings ();
6690
6691	/* First do required canonicalization of [TARGET_]MEM_REF addresses
6692	after propagation.
6693	??? This shouldn't be done in generic folding but in the
6694	propagation helpers which also know whether an address was
6695	propagated.
6696	Also canonicalize operand order. */
6697	switch (gimple_code (g: stmt))
6698	{
6699	case GIMPLE_ASSIGN:
6700	if (gimple_assign_rhs_class (gs: stmt) == GIMPLE_SINGLE_RHS)
6701	{
6702	tree *rhs = gimple_assign_rhs1_ptr (gs: stmt);
6703	if ((REFERENCE_CLASS_P (*rhs)
6704	|| TREE_CODE (*rhs) == ADDR_EXPR)
6705	&& maybe_canonicalize_mem_ref_addr (t: rhs))
6706	changed = true;
6707	tree *lhs = gimple_assign_lhs_ptr (gs: stmt);
6708	if (REFERENCE_CLASS_P (*lhs)
6709	&& maybe_canonicalize_mem_ref_addr (t: lhs))
6710	changed = true;
6711	/* Canonicalize &MEM[ssa_n, CST] to ssa_n p+ CST.
6712	This cannot be done in maybe_canonicalize_mem_ref_addr
6713	as the gimple now has two operands rather than one.
6714	The same reason why this can't be done in
6715	maybe_canonicalize_mem_ref_addr is the same reason why
6716	this can't be done inplace. */
6717	if (!inplace && TREE_CODE (*rhs) == ADDR_EXPR)
6718	{
6719	tree inner = TREE_OPERAND (*rhs, 0);
6720	if (TREE_CODE (inner) == MEM_REF
6721	&& TREE_CODE (TREE_OPERAND (inner, 0)) == SSA_NAME
6722	&& TREE_CODE (TREE_OPERAND (inner, 1)) == INTEGER_CST)
6723	{
6724	tree ptr = TREE_OPERAND (inner, 0);
6725	tree addon = TREE_OPERAND (inner, 1);
6726	addon = fold_convert (sizetype, addon);
6727	gimple_assign_set_rhs_with_ops (gsi, code: POINTER_PLUS_EXPR,
6728	op1: ptr, op2: addon);
6729	changed = true;
6730	stmt = gsi_stmt (i: *gsi);
6731	}
6732	}
6733	}
6734	else
6735	{
6736	/* Canonicalize operand order. */
6737	enum tree_code code = gimple_assign_rhs_code (gs: stmt);
6738	if (TREE_CODE_CLASS (code) == tcc_comparison
6739	|| commutative_tree_code (code)
6740	|| commutative_ternary_tree_code (code))
6741	{
6742	tree rhs1 = gimple_assign_rhs1 (gs: stmt);
6743	tree rhs2 = gimple_assign_rhs2 (gs: stmt);
6744	if (tree_swap_operands_p (rhs1, rhs2))
6745	{
6746	gimple_assign_set_rhs1 (gs: stmt, rhs: rhs2);
6747	gimple_assign_set_rhs2 (gs: stmt, rhs: rhs1);
6748	if (TREE_CODE_CLASS (code) == tcc_comparison)
6749	gimple_assign_set_rhs_code (s: stmt,
6750	code: swap_tree_comparison (code));
6751	changed = true;
6752	}
6753	}
6754	}
6755	break;
6756	case GIMPLE_CALL:
6757	{
6758	gcall *call = as_a<gcall *> (p: stmt);
6759	for (i = 0; i < gimple_call_num_args (gs: call); ++i)
6760	{
6761	tree *arg = gimple_call_arg_ptr (gs: call, index: i);
6762	if (REFERENCE_CLASS_P (*arg)
6763	&& maybe_canonicalize_mem_ref_addr (t: arg))
6764	changed = true;
6765	}
6766	tree *lhs = gimple_call_lhs_ptr (gs: call);
6767	if (*lhs
6768	&& REFERENCE_CLASS_P (*lhs)
6769	&& maybe_canonicalize_mem_ref_addr (t: lhs))
6770	changed = true;
6771	if (*lhs)
6772	{
6773	combined_fn cfn = gimple_call_combined_fn (call);
6774	internal_fn ifn = associated_internal_fn (cfn, TREE_TYPE (*lhs));
6775	int opno = first_commutative_argument (ifn);
6776	if (opno >= 0)
6777	{
6778	tree arg1 = gimple_call_arg (gs: call, index: opno);
6779	tree arg2 = gimple_call_arg (gs: call, index: opno + 1);
6780	if (tree_swap_operands_p (arg1, arg2))
6781	{
6782	gimple_call_set_arg (gs: call, index: opno, arg: arg2);
6783	gimple_call_set_arg (gs: call, index: opno + 1, arg: arg1);
6784	changed = true;
6785	}
6786	}
6787	}
6788	break;
6789	}
6790	case GIMPLE_ASM:
6791	{
6792	gasm *asm_stmt = as_a <gasm *> (p: stmt);
6793	for (i = 0; i < gimple_asm_noutputs (asm_stmt); ++i)
6794	{
6795	tree link = gimple_asm_output_op (asm_stmt, index: i);
6796	tree op = TREE_VALUE (link);
6797	if (REFERENCE_CLASS_P (op)
6798	&& maybe_canonicalize_mem_ref_addr (t: &TREE_VALUE (link)))
6799	changed = true;
6800	}
6801	for (i = 0; i < gimple_asm_ninputs (asm_stmt); ++i)
6802	{
6803	tree link = gimple_asm_input_op (asm_stmt, index: i);
6804	tree op = TREE_VALUE (link);
6805	if ((REFERENCE_CLASS_P (op)
6806	|| TREE_CODE (op) == ADDR_EXPR)
6807	&& maybe_canonicalize_mem_ref_addr (t: &TREE_VALUE (link)))
6808	changed = true;
6809	}
6810	}
6811	break;
6812	case GIMPLE_DEBUG:
6813	if (gimple_debug_bind_p (s: stmt))
6814	{
6815	tree *val = gimple_debug_bind_get_value_ptr (dbg: stmt);
6816	if (*val
6817	&& (REFERENCE_CLASS_P (*val)
6818	|| TREE_CODE (*val) == ADDR_EXPR)
6819	&& maybe_canonicalize_mem_ref_addr (t: val, is_debug: true))
6820	changed = true;
6821	}
6822	break;
6823	case GIMPLE_COND:
6824	{
6825	/* Canonicalize operand order. */
6826	tree lhs = gimple_cond_lhs (gs: stmt);
6827	tree rhs = gimple_cond_rhs (gs: stmt);
6828	if (tree_swap_operands_p (lhs, rhs))
6829	{
6830	gcond *gc = as_a <gcond *> (p: stmt);
6831	gimple_cond_set_lhs (gs: gc, lhs: rhs);
6832	gimple_cond_set_rhs (gs: gc, rhs: lhs);
6833	gimple_cond_set_code (gs: gc,
6834	code: swap_tree_comparison (gimple_cond_code (gs: gc)));
6835	changed = true;
6836	}
6837	}
6838	default:;
6839	}
6840
6841	/* Dispatch to pattern-based folding. */
6842	if (!inplace
6843	|| is_gimple_assign (gs: stmt)
6844	|| gimple_code (g: stmt) == GIMPLE_COND)
6845	{
6846	gimple_seq seq = NULL;
6847	gimple_match_op res_op;
6848	if (gimple_simplify (stmt, &res_op, inplace ? NULL : &seq,
6849	valueize, valueize))
6850	{
6851	if (replace_stmt_with_simplification (gsi, res_op: &res_op, seq: &seq, inplace,
6852	dce_worklist))
6853	changed = true;
6854	else
6855	gimple_seq_discard (seq);
6856	}
6857	}
6858
6859	stmt = gsi_stmt (i: *gsi);
6860
6861	/* Fold the main computation performed by the statement. */
6862	switch (gimple_code (g: stmt))
6863	{
6864	case GIMPLE_ASSIGN:
6865	{
6866	/* Try to canonicalize for boolean-typed X the comparisons
6867	X == 0, X == 1, X != 0, and X != 1. */
6868	if (gimple_assign_rhs_code (gs: stmt) == EQ_EXPR
6869	|| gimple_assign_rhs_code (gs: stmt) == NE_EXPR)
6870	{
6871	tree lhs = gimple_assign_lhs (gs: stmt);
6872	tree op1 = gimple_assign_rhs1 (gs: stmt);
6873	tree op2 = gimple_assign_rhs2 (gs: stmt);
6874	tree type = TREE_TYPE (op1);
6875
6876	/* Check whether the comparison operands are of the same boolean
6877	type as the result type is.
6878	Check that second operand is an integer-constant with value
6879	one or zero. */
6880	if (TREE_CODE (op2) == INTEGER_CST
6881	&& (integer_zerop (op2) || integer_onep (op2))
6882	&& useless_type_conversion_p (TREE_TYPE (lhs), type))
6883	{
6884	enum tree_code cmp_code = gimple_assign_rhs_code (gs: stmt);
6885	bool is_logical_not = false;
6886
6887	/* X == 0 and X != 1 is a logical-not.of X
6888	X == 1 and X != 0 is X */
6889	if ((cmp_code == EQ_EXPR && integer_zerop (op2))
6890	|| (cmp_code == NE_EXPR && integer_onep (op2)))
6891	is_logical_not = true;
6892
6893	if (is_logical_not == false)
6894	gimple_assign_set_rhs_with_ops (gsi, TREE_CODE (op1), op1);
6895	/* Only for one-bit precision typed X the transformation
6896	!X -> ~X is valied. */
6897	else if (TYPE_PRECISION (type) == 1)
6898	gimple_assign_set_rhs_with_ops (gsi, code: BIT_NOT_EXPR, op1);
6899	/* Otherwise we use !X -> X ^ 1. */
6900	else
6901	gimple_assign_set_rhs_with_ops (gsi, code: BIT_XOR_EXPR, op1,
6902	op2: build_int_cst (type, 1));
6903	changed = true;
6904	break;
6905	}
6906	}
6907
6908	unsigned old_num_ops = gimple_num_ops (gs: stmt);
6909	tree lhs = gimple_assign_lhs (gs: stmt);
6910	tree new_rhs = fold_gimple_assign (si: gsi);
6911	if (new_rhs
6912	&& !useless_type_conversion_p (TREE_TYPE (lhs),
6913	TREE_TYPE (new_rhs)))
6914	new_rhs = fold_convert (TREE_TYPE (lhs), new_rhs);
6915	if (new_rhs
6916	&& (!inplace
6917	|| get_gimple_rhs_num_ops (TREE_CODE (new_rhs)) < old_num_ops))
6918	{
6919	gimple_assign_set_rhs_from_tree (gsi, new_rhs);
6920	changed = true;
6921	}
6922	break;
6923	}
6924
6925	case GIMPLE_CALL:
6926	changed |= gimple_fold_call (gsi, inplace);
6927	break;
6928
6929	case GIMPLE_DEBUG:
6930	if (gimple_debug_bind_p (s: stmt))
6931	{
6932	tree val = gimple_debug_bind_get_value (dbg: stmt);
6933	if (val && REFERENCE_CLASS_P (val))
6934	{
6935	tree tem = maybe_fold_reference (expr: val);
6936	if (tem)
6937	{
6938	gimple_debug_bind_set_value (dbg: stmt, value: tem);
6939	changed = true;
6940	}
6941	}
6942	}
6943	break;
6944
6945	case GIMPLE_RETURN:
6946	{
6947	greturn *ret_stmt = as_a<greturn *> (p: stmt);
6948	tree ret = gimple_return_retval(gs: ret_stmt);
6949
6950	if (ret && TREE_CODE (ret) == SSA_NAME && valueize)
6951	{
6952	tree val = valueize (ret);
6953	if (val && val != ret
6954	&& may_propagate_copy (ret, val))
6955	{
6956	gimple_return_set_retval (gs: ret_stmt, retval: val);
6957	changed = true;
6958	}
6959	}
6960	}
6961	break;
6962
6963	default:;
6964	}
6965
6966	stmt = gsi_stmt (i: *gsi);
6967
6968	fold_undefer_overflow_warnings (changed && !nowarning, stmt, 0);
6969	return changed;
6970	}
6971
6972	/* Valueziation callback that ends up not following SSA edges. */
6973
6974	tree
6975	no_follow_ssa_edges (tree)
6976	{
6977	return NULL_TREE;
6978	}
6979
6980	/* Valueization callback that ends up following single-use SSA edges only. */
6981
6982	tree
6983	follow_single_use_edges (tree val)
6984	{
6985	if (TREE_CODE (val) == SSA_NAME
6986	&& !has_single_use (var: val))
6987	return NULL_TREE;
6988	return val;
6989	}
6990
6991	/* Valueization callback that follows all SSA edges. */
6992
6993	tree
6994	follow_all_ssa_edges (tree val)
6995	{
6996	return val;
6997	}
6998
6999	/* Fold the statement pointed to by GSI. In some cases, this function may
7000	replace the whole statement with a new one. Returns true iff folding
7001	makes any changes.
7002	The statement pointed to by GSI should be in valid gimple form but may
7003	be in unfolded state as resulting from for example constant propagation
7004	which can produce *&x = 0. */
7005
7006	bool
7007	fold_stmt (gimple_stmt_iterator *gsi, bitmap dce_bitmap)
7008	{
7009	return fold_stmt_1 (gsi, inplace: false, valueize: no_follow_ssa_edges, dce_worklist: dce_bitmap);
7010	}
7011
7012	bool
7013	fold_stmt (gimple_stmt_iterator *gsi, tree (*valueize) (tree), bitmap dce_bitmap)
7014	{
7015	return fold_stmt_1 (gsi, inplace: false, valueize, dce_worklist: dce_bitmap);
7016	}
7017
7018	/* Perform the minimal folding on statement *GSI. Only operations like
7019	*&x created by constant propagation are handled. The statement cannot
7020	be replaced with a new one. Return true if the statement was
7021	changed, false otherwise.
7022	The statement *GSI should be in valid gimple form but may
7023	be in unfolded state as resulting from for example constant propagation
7024	which can produce *&x = 0. */
7025
7026	bool
7027	fold_stmt_inplace (gimple_stmt_iterator *gsi, tree (*valueize) (tree))
7028	{
7029	gimple *stmt = gsi_stmt (i: *gsi);
7030	bool changed = fold_stmt_1 (gsi, inplace: true, valueize);
7031	gcc_assert (gsi_stmt (*gsi) == stmt);
7032	return changed;
7033	}
7034
7035	/* Canonicalize and possibly invert the boolean EXPR; return NULL_TREE
7036	if EXPR is null or we don't know how.
7037	If non-null, the result always has boolean type. */
7038
7039	static tree
7040	canonicalize_bool (tree expr, bool invert)
7041	{
7042	if (!expr)
7043	return NULL_TREE;
7044	else if (invert)
7045	{
7046	if (integer_nonzerop (expr))
7047	return boolean_false_node;
7048	else if (integer_zerop (expr))
7049	return boolean_true_node;
7050	else if (TREE_CODE (expr) == SSA_NAME)
7051	return fold_build2 (EQ_EXPR, boolean_type_node, expr,
7052	build_int_cst (TREE_TYPE (expr), 0));
7053	else if (COMPARISON_CLASS_P (expr))
7054	return fold_build2 (invert_tree_comparison (TREE_CODE (expr), false),
7055	boolean_type_node,
7056	TREE_OPERAND (expr, 0),
7057	TREE_OPERAND (expr, 1));
7058	else
7059	return NULL_TREE;
7060	}
7061	else
7062	{
7063	if (TREE_CODE (TREE_TYPE (expr)) == BOOLEAN_TYPE)
7064	return expr;
7065	if (integer_nonzerop (expr))
7066	return boolean_true_node;
7067	else if (integer_zerop (expr))
7068	return boolean_false_node;
7069	else if (TREE_CODE (expr) == SSA_NAME)
7070	return fold_build2 (NE_EXPR, boolean_type_node, expr,
7071	build_int_cst (TREE_TYPE (expr), 0));
7072	else if (COMPARISON_CLASS_P (expr))
7073	return fold_build2 (TREE_CODE (expr),
7074	boolean_type_node,
7075	TREE_OPERAND (expr, 0),
7076	TREE_OPERAND (expr, 1));
7077	else
7078	return NULL_TREE;
7079	}
7080	}
7081
7082	/* Check to see if a boolean expression EXPR is logically equivalent to the
7083	comparison (OP1 CODE OP2). Check for various identities involving
7084	SSA_NAMEs. */
7085
7086	static bool
7087	same_bool_comparison_p (const_tree expr, enum tree_code code,
7088	const_tree op1, const_tree op2)
7089	{
7090	gimple *s;
7091
7092	/* The obvious case. */
7093	if (TREE_CODE (expr) == code
7094	&& operand_equal_p (TREE_OPERAND (expr, 0), op1, flags: 0)
7095	&& operand_equal_p (TREE_OPERAND (expr, 1), op2, flags: 0))
7096	return true;
7097
7098	/* Check for comparing (name, name != 0) and the case where expr
7099	is an SSA_NAME with a definition matching the comparison. */
7100	if (TREE_CODE (expr) == SSA_NAME
7101	&& TREE_CODE (TREE_TYPE (expr)) == BOOLEAN_TYPE)
7102	{
7103	if (operand_equal_p (expr, op1, flags: 0))
7104	return ((code == NE_EXPR && integer_zerop (op2))
7105	|| (code == EQ_EXPR && integer_nonzerop (op2)));
7106	s = SSA_NAME_DEF_STMT (expr);
7107	if (is_gimple_assign (gs: s)
7108	&& gimple_assign_rhs_code (gs: s) == code
7109	&& operand_equal_p (gimple_assign_rhs1 (gs: s), op1, flags: 0)
7110	&& operand_equal_p (gimple_assign_rhs2 (gs: s), op2, flags: 0))
7111	return true;
7112	}
7113
7114	/* If op1 is of the form (name != 0) or (name == 0), and the definition
7115	of name is a comparison, recurse. */
7116	if (TREE_CODE (op1) == SSA_NAME
7117	&& TREE_CODE (TREE_TYPE (op1)) == BOOLEAN_TYPE)
7118	{
7119	s = SSA_NAME_DEF_STMT (op1);
7120	if (is_gimple_assign (gs: s)
7121	&& TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison)
7122	{
7123	enum tree_code c = gimple_assign_rhs_code (gs: s);
7124	if ((c == NE_EXPR && integer_zerop (op2))
7125	|| (c == EQ_EXPR && integer_nonzerop (op2)))
7126	return same_bool_comparison_p (expr, code: c,
7127	op1: gimple_assign_rhs1 (gs: s),
7128	op2: gimple_assign_rhs2 (gs: s));
7129	if ((c == EQ_EXPR && integer_zerop (op2))
7130	|| (c == NE_EXPR && integer_nonzerop (op2)))
7131	return same_bool_comparison_p (expr,
7132	code: invert_tree_comparison (c, false),
7133	op1: gimple_assign_rhs1 (gs: s),
7134	op2: gimple_assign_rhs2 (gs: s));
7135	}
7136	}
7137	return false;
7138	}
7139
7140	/* Check to see if two boolean expressions OP1 and OP2 are logically
7141	equivalent. */
7142
7143	static bool
7144	same_bool_result_p (const_tree op1, const_tree op2)
7145	{
7146	/* Simple cases first. */
7147	if (operand_equal_p (op1, op2, flags: 0))
7148	return true;
7149
7150	/* Check the cases where at least one of the operands is a comparison.
7151	These are a bit smarter than operand_equal_p in that they apply some
7152	identifies on SSA_NAMEs. */
7153	if (COMPARISON_CLASS_P (op2)
7154	&& same_bool_comparison_p (expr: op1, TREE_CODE (op2),
7155	TREE_OPERAND (op2, 0),
7156	TREE_OPERAND (op2, 1)))
7157	return true;
7158	if (COMPARISON_CLASS_P (op1)
7159	&& same_bool_comparison_p (expr: op2, TREE_CODE (op1),
7160	TREE_OPERAND (op1, 0),
7161	TREE_OPERAND (op1, 1)))
7162	return true;
7163
7164	/* Default case. */
7165	return false;
7166	}
7167
7168	/* Forward declarations for some mutually recursive functions. */
7169
7170	static tree
7171	and_comparisons_1 (tree type, enum tree_code code1, tree op1a, tree op1b,
7172	enum tree_code code2, tree op2a, tree op2b, basic_block);
7173	static tree
7174	and_var_with_comparison (tree type, tree var, bool invert,
7175	enum tree_code code2, tree op2a, tree op2b,
7176	basic_block);
7177	static tree
7178	and_var_with_comparison_1 (tree type, gimple *stmt,
7179	enum tree_code code2, tree op2a, tree op2b,
7180	basic_block);
7181	static tree
7182	or_comparisons_1 (tree, enum tree_code code1, tree op1a, tree op1b,
7183	enum tree_code code2, tree op2a, tree op2b,
7184	basic_block);
7185	static tree
7186	or_var_with_comparison (tree, tree var, bool invert,
7187	enum tree_code code2, tree op2a, tree op2b,
7188	basic_block);
7189	static tree
7190	or_var_with_comparison_1 (tree, gimple *stmt,
7191	enum tree_code code2, tree op2a, tree op2b,
7192	basic_block);
7193
7194	/* Helper function for and_comparisons_1: try to simplify the AND of the
7195	ssa variable VAR with the comparison specified by (OP2A CODE2 OP2B).
7196	If INVERT is true, invert the value of the VAR before doing the AND.
7197	Return NULL_EXPR if we can't simplify this to a single expression. */
7198
7199	static tree
7200	and_var_with_comparison (tree type, tree var, bool invert,
7201	enum tree_code code2, tree op2a, tree op2b,
7202	basic_block outer_cond_bb)
7203	{
7204	tree t;
7205	gimple *stmt = SSA_NAME_DEF_STMT (var);
7206
7207	/* We can only deal with variables whose definitions are assignments. */
7208	if (!is_gimple_assign (gs: stmt))
7209	return NULL_TREE;
7210
7211	/* If we have an inverted comparison, apply DeMorgan's law and rewrite
7212	!var AND (op2a code2 op2b) => !(var OR !(op2a code2 op2b))
7213	Then we only have to consider the simpler non-inverted cases. */
7214	if (invert)
7215	t = or_var_with_comparison_1 (type, stmt,
7216	code2: invert_tree_comparison (code2, false),
7217	op2a, op2b, outer_cond_bb);
7218	else
7219	t = and_var_with_comparison_1 (type, stmt, code2, op2a, op2b,
7220	outer_cond_bb);
7221	return canonicalize_bool (expr: t, invert);
7222	}
7223
7224	/* Try to simplify the AND of the ssa variable defined by the assignment
7225	STMT with the comparison specified by (OP2A CODE2 OP2B).
7226	Return NULL_EXPR if we can't simplify this to a single expression. */
7227
7228	static tree
7229	and_var_with_comparison_1 (tree type, gimple *stmt,
7230	enum tree_code code2, tree op2a, tree op2b,
7231	basic_block outer_cond_bb)
7232	{
7233	tree var = gimple_assign_lhs (gs: stmt);
7234	tree true_test_var = NULL_TREE;
7235	tree false_test_var = NULL_TREE;
7236	enum tree_code innercode = gimple_assign_rhs_code (gs: stmt);
7237
7238	/* Check for identities like (var AND (var == 0)) => false. */
7239	if (TREE_CODE (op2a) == SSA_NAME
7240	&& TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE)
7241	{
7242	if ((code2 == NE_EXPR && integer_zerop (op2b))
7243	|| (code2 == EQ_EXPR && integer_nonzerop (op2b)))
7244	{
7245	true_test_var = op2a;
7246	if (var == true_test_var)
7247	return var;
7248	}
7249	else if ((code2 == EQ_EXPR && integer_zerop (op2b))
7250	|| (code2 == NE_EXPR && integer_nonzerop (op2b)))
7251	{
7252	false_test_var = op2a;
7253	if (var == false_test_var)
7254	return boolean_false_node;
7255	}
7256	}
7257
7258	/* If the definition is a comparison, recurse on it. */
7259	if (TREE_CODE_CLASS (innercode) == tcc_comparison)
7260	{
7261	tree t = and_comparisons_1 (type, code1: innercode,
7262	op1a: gimple_assign_rhs1 (gs: stmt),
7263	op1b: gimple_assign_rhs2 (gs: stmt),
7264	code2,
7265	op2a,
7266	op2b, outer_cond_bb);
7267	if (t)
7268	return t;
7269	}
7270
7271	/* If the definition is an AND or OR expression, we may be able to
7272	simplify by reassociating. */
7273	if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
7274	&& (innercode == BIT_AND_EXPR || innercode == BIT_IOR_EXPR))
7275	{
7276	tree inner1 = gimple_assign_rhs1 (gs: stmt);
7277	tree inner2 = gimple_assign_rhs2 (gs: stmt);
7278	gimple *s;
7279	tree t;
7280	tree partial = NULL_TREE;
7281	bool is_and = (innercode == BIT_AND_EXPR);
7282
7283	/* Check for boolean identities that don't require recursive examination
7284	of inner1/inner2:
7285	inner1 AND (inner1 AND inner2) => inner1 AND inner2 => var
7286	inner1 AND (inner1 OR inner2) => inner1
7287	!inner1 AND (inner1 AND inner2) => false
7288	!inner1 AND (inner1 OR inner2) => !inner1 AND inner2
7289	Likewise for similar cases involving inner2. */
7290	if (inner1 == true_test_var)
7291	return (is_and ? var : inner1);
7292	else if (inner2 == true_test_var)
7293	return (is_and ? var : inner2);
7294	else if (inner1 == false_test_var)
7295	return (is_and
7296	? boolean_false_node
7297	: and_var_with_comparison (type, var: inner2, invert: false, code2, op2a,
7298	op2b, outer_cond_bb));
7299	else if (inner2 == false_test_var)
7300	return (is_and
7301	? boolean_false_node
7302	: and_var_with_comparison (type, var: inner1, invert: false, code2, op2a,
7303	op2b, outer_cond_bb));
7304
7305	/* Next, redistribute/reassociate the AND across the inner tests.
7306	Compute the first partial result, (inner1 AND (op2a code op2b)) */
7307	if (TREE_CODE (inner1) == SSA_NAME
7308	&& is_gimple_assign (gs: s = SSA_NAME_DEF_STMT (inner1))
7309	&& TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
7310	&& (t = maybe_fold_and_comparisons (type, gimple_assign_rhs_code (gs: s),
7311	gimple_assign_rhs1 (gs: s),
7312	gimple_assign_rhs2 (gs: s),
7313	code2, op2a, op2b,
7314	outer_cond_bb)))
7315	{
7316	/* Handle the AND case, where we are reassociating:
7317	(inner1 AND inner2) AND (op2a code2 op2b)
7318	=> (t AND inner2)
7319	If the partial result t is a constant, we win. Otherwise
7320	continue on to try reassociating with the other inner test. */
7321	if (is_and)
7322	{
7323	if (integer_onep (t))
7324	return inner2;
7325	else if (integer_zerop (t))
7326	return boolean_false_node;
7327	}
7328
7329	/* Handle the OR case, where we are redistributing:
7330	(inner1 OR inner2) AND (op2a code2 op2b)
7331	=> (t OR (inner2 AND (op2a code2 op2b))) */
7332	else if (integer_onep (t))
7333	return boolean_true_node;
7334
7335	/* Save partial result for later. */
7336	partial = t;
7337	}
7338
7339	/* Compute the second partial result, (inner2 AND (op2a code op2b)) */
7340	if (TREE_CODE (inner2) == SSA_NAME
7341	&& is_gimple_assign (gs: s = SSA_NAME_DEF_STMT (inner2))
7342	&& TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
7343	&& (t = maybe_fold_and_comparisons (type, gimple_assign_rhs_code (gs: s),
7344	gimple_assign_rhs1 (gs: s),
7345	gimple_assign_rhs2 (gs: s),
7346	code2, op2a, op2b,
7347	outer_cond_bb)))
7348	{
7349	/* Handle the AND case, where we are reassociating:
7350	(inner1 AND inner2) AND (op2a code2 op2b)
7351	=> (inner1 AND t) */
7352	if (is_and)
7353	{
7354	if (integer_onep (t))
7355	return inner1;
7356	else if (integer_zerop (t))
7357	return boolean_false_node;
7358	/* If both are the same, we can apply the identity
7359	(x AND x) == x. */
7360	else if (partial && same_bool_result_p (op1: t, op2: partial))
7361	return t;
7362	}
7363
7364	/* Handle the OR case. where we are redistributing:
7365	(inner1 OR inner2) AND (op2a code2 op2b)
7366	=> (t OR (inner1 AND (op2a code2 op2b)))
7367	=> (t OR partial) */
7368	else
7369	{
7370	if (integer_onep (t))
7371	return boolean_true_node;
7372	else if (partial)
7373	{
7374	/* We already got a simplification for the other
7375	operand to the redistributed OR expression. The
7376	interesting case is when at least one is false.
7377	Or, if both are the same, we can apply the identity
7378	(x OR x) == x. */
7379	if (integer_zerop (partial))
7380	return t;
7381	else if (integer_zerop (t))
7382	return partial;
7383	else if (same_bool_result_p (op1: t, op2: partial))
7384	return t;
7385	}
7386	}
7387	}
7388	}
7389	return NULL_TREE;
7390	}
7391
7392	/* Try to simplify the AND of two comparisons defined by
7393	(OP1A CODE1 OP1B) and (OP2A CODE2 OP2B), respectively.
7394	If this can be done without constructing an intermediate value,
7395	return the resulting tree; otherwise NULL_TREE is returned.
7396	This function is deliberately asymmetric as it recurses on SSA_DEFs
7397	in the first comparison but not the second. */
7398
7399	static tree
7400	and_comparisons_1 (tree type, enum tree_code code1, tree op1a, tree op1b,
7401	enum tree_code code2, tree op2a, tree op2b,
7402	basic_block outer_cond_bb)
7403	{
7404	tree truth_type = truth_type_for (TREE_TYPE (op1a));
7405
7406	/* First check for ((x CODE1 y) AND (x CODE2 y)). */
7407	if (operand_equal_p (op1a, op2a, flags: 0)
7408	&& operand_equal_p (op1b, op2b, flags: 0))
7409	{
7410	/* Result will be either NULL_TREE, or a combined comparison. */
7411	tree t = combine_comparisons (UNKNOWN_LOCATION,
7412	TRUTH_ANDIF_EXPR, code1, code2,
7413	truth_type, op1a, op1b);
7414	if (t)
7415	return t;
7416	}
7417
7418	/* Likewise the swapped case of the above. */
7419	if (operand_equal_p (op1a, op2b, flags: 0)
7420	&& operand_equal_p (op1b, op2a, flags: 0))
7421	{
7422	/* Result will be either NULL_TREE, or a combined comparison. */
7423	tree t = combine_comparisons (UNKNOWN_LOCATION,
7424	TRUTH_ANDIF_EXPR, code1,
7425	swap_tree_comparison (code2),
7426	truth_type, op1a, op1b);
7427	if (t)
7428	return t;
7429	}
7430
7431	/* Perhaps the first comparison is (NAME != 0) or (NAME == 1) where
7432	NAME's definition is a truth value. See if there are any simplifications
7433	that can be done against the NAME's definition. */
7434	if (TREE_CODE (op1a) == SSA_NAME
7435	&& (code1 == NE_EXPR || code1 == EQ_EXPR)
7436	&& (integer_zerop (op1b) || integer_onep (op1b)))
7437	{
7438	bool invert = ((code1 == EQ_EXPR && integer_zerop (op1b))
7439	|| (code1 == NE_EXPR && integer_onep (op1b)));
7440	gimple *stmt = SSA_NAME_DEF_STMT (op1a);
7441	switch (gimple_code (g: stmt))
7442	{
7443	case GIMPLE_ASSIGN:
7444	/* Try to simplify by copy-propagating the definition. */
7445	return and_var_with_comparison (type, var: op1a, invert, code2, op2a,
7446	op2b, outer_cond_bb);
7447
7448	case GIMPLE_PHI:
7449	/* If every argument to the PHI produces the same result when
7450	ANDed with the second comparison, we win.
7451	Do not do this unless the type is bool since we need a bool
7452	result here anyway. */
7453	if (TREE_CODE (TREE_TYPE (op1a)) == BOOLEAN_TYPE)
7454	{
7455	tree result = NULL_TREE;
7456	unsigned i;
7457	for (i = 0; i < gimple_phi_num_args (gs: stmt); i++)
7458	{
7459	tree arg = gimple_phi_arg_def (gs: stmt, index: i);
7460
7461	/* If this PHI has itself as an argument, ignore it.
7462	If all the other args produce the same result,
7463	we're still OK. */
7464	if (arg == gimple_phi_result (gs: stmt))
7465	continue;
7466	else if (TREE_CODE (arg) == INTEGER_CST)
7467	{
7468	if (invert ? integer_nonzerop (arg) : integer_zerop (arg))
7469	{
7470	if (!result)
7471	result = boolean_false_node;
7472	else if (!integer_zerop (result))
7473	return NULL_TREE;
7474	}
7475	else if (!result)
7476	result = fold_build2 (code2, boolean_type_node,
7477	op2a, op2b);
7478	else if (!same_bool_comparison_p (expr: result,
7479	code: code2, op1: op2a, op2: op2b))
7480	return NULL_TREE;
7481	}
7482	else if (TREE_CODE (arg) == SSA_NAME
7483	&& !SSA_NAME_IS_DEFAULT_DEF (arg))
7484	{
7485	tree temp;
7486	gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
7487	/* In simple cases we can look through PHI nodes,
7488	but we have to be careful with loops.
7489	See PR49073. */
7490	if (! dom_info_available_p (CDI_DOMINATORS)
7491	|| gimple_bb (g: def_stmt) == gimple_bb (g: stmt)
7492	|| dominated_by_p (CDI_DOMINATORS,
7493	gimple_bb (g: def_stmt),
7494	gimple_bb (g: stmt)))
7495	return NULL_TREE;
7496	temp = and_var_with_comparison (type, var: arg, invert, code2,
7497	op2a, op2b,
7498	outer_cond_bb);
7499	if (!temp)
7500	return NULL_TREE;
7501	else if (!result)
7502	result = temp;
7503	else if (!same_bool_result_p (op1: result, op2: temp))
7504	return NULL_TREE;
7505	}
7506	else
7507	return NULL_TREE;
7508	}
7509	return result;
7510	}
7511
7512	default:
7513	break;
7514	}
7515	}
7516	return NULL_TREE;
7517	}
7518
7519	static basic_block fosa_bb;
7520	static vec<std::pair<tree, flow_sensitive_info_storage> > *fosa_unwind;
7521	static tree
7522	follow_outer_ssa_edges (tree val)
7523	{
7524	if (TREE_CODE (val) == SSA_NAME
7525	&& !SSA_NAME_IS_DEFAULT_DEF (val))
7526	{
7527	basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (val));
7528	if (!def_bb
7529	|| def_bb == fosa_bb
7530	|| (dom_info_available_p (CDI_DOMINATORS)
7531	&& (def_bb == fosa_bb
7532	|| dominated_by_p (CDI_DOMINATORS, fosa_bb, def_bb))))
7533	return val;
7534	/* We cannot temporarily rewrite stmts with undefined overflow
7535	behavior, so avoid expanding them. */
7536	if ((ANY_INTEGRAL_TYPE_P (TREE_TYPE (val))
7537	|| POINTER_TYPE_P (TREE_TYPE (val)))
7538	&& !TYPE_OVERFLOW_WRAPS (TREE_TYPE (val)))
7539	return NULL_TREE;
7540	flow_sensitive_info_storage storage;
7541	storage.save_and_clear (val);
7542	/* If the definition does not dominate fosa_bb temporarily reset
7543	flow-sensitive info. */
7544	fosa_unwind->safe_push (obj: std::make_pair (x&: val, y&: storage));
7545	return val;
7546	}
7547	return val;
7548	}
7549
7550	/* Helper function for maybe_fold_and_comparisons and maybe_fold_or_comparisons
7551	: try to simplify the AND/OR of the ssa variable VAR with the comparison
7552	specified by (OP2A CODE2 OP2B) from match.pd. Return NULL_EXPR if we can't
7553	simplify this to a single expression. As we are going to lower the cost
7554	of building SSA names / gimple stmts significantly, we need to allocate
7555	them ont the stack. This will cause the code to be a bit ugly. */
7556
7557	static tree
7558	maybe_fold_comparisons_from_match_pd (tree type, enum tree_code code,
7559	enum tree_code code1,
7560	tree op1a, tree op1b,
7561	enum tree_code code2, tree op2a,
7562	tree op2b,
7563	basic_block outer_cond_bb)
7564	{
7565	/* Allocate gimple stmt1 on the stack. */
7566	gassign *stmt1
7567	= (gassign *) XALLOCAVEC (char, gimple_size (GIMPLE_ASSIGN, 3));
7568	gimple_init (g: stmt1, code: GIMPLE_ASSIGN, num_ops: 3);
7569	gimple_assign_set_rhs_code (s: stmt1, code: code1);
7570	gimple_assign_set_rhs1 (gs: stmt1, rhs: op1a);
7571	gimple_assign_set_rhs2 (gs: stmt1, rhs: op1b);
7572	gimple_set_bb (stmt1, NULL);
7573
7574	/* Allocate gimple stmt2 on the stack. */
7575	gassign *stmt2
7576	= (gassign *) XALLOCAVEC (char, gimple_size (GIMPLE_ASSIGN, 3));
7577	gimple_init (g: stmt2, code: GIMPLE_ASSIGN, num_ops: 3);
7578	gimple_assign_set_rhs_code (s: stmt2, code: code2);
7579	gimple_assign_set_rhs1 (gs: stmt2, rhs: op2a);
7580	gimple_assign_set_rhs2 (gs: stmt2, rhs: op2b);
7581	gimple_set_bb (stmt2, NULL);
7582
7583	/* Allocate SSA names(lhs1) on the stack. */
7584	alignas (tree_node) unsigned char lhs1buf[sizeof (tree_ssa_name)];
7585	tree lhs1 = (tree) &lhs1buf[0];
7586	memset (s: lhs1, c: 0, n: sizeof (tree_ssa_name));
7587	TREE_SET_CODE (lhs1, SSA_NAME);
7588	TREE_TYPE (lhs1) = type;
7589	init_ssa_name_imm_use (lhs1);
7590
7591	/* Allocate SSA names(lhs2) on the stack. */
7592	alignas (tree_node) unsigned char lhs2buf[sizeof (tree_ssa_name)];
7593	tree lhs2 = (tree) &lhs2buf[0];
7594	memset (s: lhs2, c: 0, n: sizeof (tree_ssa_name));
7595	TREE_SET_CODE (lhs2, SSA_NAME);
7596	TREE_TYPE (lhs2) = type;
7597	init_ssa_name_imm_use (lhs2);
7598
7599	gimple_assign_set_lhs (gs: stmt1, lhs: lhs1);
7600	gimple_assign_set_lhs (gs: stmt2, lhs: lhs2);
7601
7602	gimple_match_op op (gimple_match_cond::UNCOND, code,
7603	type, gimple_assign_lhs (gs: stmt1),
7604	gimple_assign_lhs (gs: stmt2));
7605	fosa_bb = outer_cond_bb;
7606	auto_vec<std::pair<tree, flow_sensitive_info_storage>, 8> unwind_stack;
7607	fosa_unwind = &unwind_stack;
7608	if (op.resimplify (NULL, (!outer_cond_bb
7609	? follow_all_ssa_edges : follow_outer_ssa_edges)))
7610	{
7611	fosa_unwind = NULL;
7612	for (auto p : unwind_stack)
7613	p.second.restore (p.first);
7614	if (gimple_simplified_result_is_gimple_val (op: &op))
7615	{
7616	tree res = op.ops[0];
7617	if (res == lhs1)
7618	return build2 (code1, type, op1a, op1b);
7619	else if (res == lhs2)
7620	return build2 (code2, type, op2a, op2b);
7621	else
7622	return res;
7623	}
7624	else if (op.code.is_tree_code ()
7625	&& TREE_CODE_CLASS ((tree_code)op.code) == tcc_comparison)
7626	{
7627	tree op0 = op.ops[0];
7628	tree op1 = op.ops[1];
7629	if (op0 == lhs1 || op0 == lhs2 || op1 == lhs1 || op1 == lhs2)
7630	return NULL_TREE; /* not simple */
7631
7632	return build2 ((enum tree_code)op.code, op.type, op0, op1);
7633	}
7634	}
7635	fosa_unwind = NULL;
7636	for (auto p : unwind_stack)
7637	p.second.restore (p.first);
7638
7639	return NULL_TREE;
7640	}
7641
7642	/* Return TRUE and set op[0] if T, following all SSA edges, is a type
7643	conversion. Reject loads if LOAD is NULL, otherwise set *LOAD if a
7644	converting load is found. */
7645
7646	static bool
7647	gimple_convert_def_p (tree t, tree op[1], gimple **load = NULL)
7648	{
7649	bool ret = false;
7650
7651	if (TREE_CODE (t) == SSA_NAME
7652	&& !SSA_NAME_IS_DEFAULT_DEF (t))
7653	if (gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (t)))
7654	{
7655	bool load_p = gimple_assign_load_p (def);
7656	if (load_p && !load)
7657	return false;
7658	switch (gimple_assign_rhs_code (gs: def))
7659	{
7660	CASE_CONVERT:
7661	op[0] = gimple_assign_rhs1 (gs: def);
7662	ret = true;
7663	break;
7664
7665	case VIEW_CONVERT_EXPR:
7666	op[0] = TREE_OPERAND (gimple_assign_rhs1 (def), 0);
7667	ret = true;
7668	break;
7669
7670	default:
7671	break;
7672	}
7673
7674	if (ret && load_p)
7675	*load = def;
7676	}
7677
7678	return ret;
7679	}
7680
7681	/* Return TRUE and set op[*] if T, following all SSA edges, resolves to a
7682	binary expression with code CODE. */
7683
7684	static bool
7685	gimple_binop_def_p (enum tree_code code, tree t, tree op[2])
7686	{
7687	if (TREE_CODE (t) == SSA_NAME
7688	&& !SSA_NAME_IS_DEFAULT_DEF (t))
7689	if (gimple *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (t)))
7690	if (gimple_assign_rhs_code (gs: def) == code)
7691	{
7692	op[0] = gimple_assign_rhs1 (gs: def);
7693	op[1] = gimple_assign_rhs2 (gs: def);
7694	return true;
7695	}
7696	return false;
7697	}
7698	/* Subroutine for fold_truth_andor_1: decode a field reference.
7699
7700	If *PEXP is a comparison reference, we return the innermost reference.
7701
7702	*PBITSIZE is set to the number of bits in the reference, *PBITPOS is
7703	set to the starting bit number.
7704
7705	*PVOLATILEP is set to 1 if the any expression encountered is volatile;
7706	otherwise it is not changed.
7707
7708	*PUNSIGNEDP is set to the signedness of the field.
7709
7710	*PREVERSEP is set to the storage order of the field.
7711
7712	*PAND_MASK is set to the mask found in a BIT_AND_EXPR, if any. If
7713	*PAND_MASK is initially set to a mask with nonzero precision, that mask is
7714	combined with the found mask, or adjusted in precision to match.
7715
7716	*PSIGNBIT is set to TRUE if, before clipping to *PBITSIZE, the mask
7717	encompassed bits that corresponded to extensions of the sign bit.
7718
7719	*PXORP is to be FALSE if EXP might be a XOR used in a compare, in which
7720	case, if PXOR_CMP_OP is a zero constant, it will be overridden with *PEXP,
7721	*PXORP will be set to TRUE, *PXOR_AND_MASK will be copied from *PAND_MASK,
7722	and the left-hand operand of the XOR will be decoded. If *PXORP is TRUE,
7723	PXOR_CMP_OP and PXOR_AND_MASK are supposed to be NULL, and then the
7724	right-hand operand of the XOR will be decoded.
7725
7726	*LOAD is set to the load stmt of the innermost reference, if any,
7727	*and NULL otherwise.
7728
7729	LOC[0..3] are filled in as conversion, masking, shifting and loading
7730	operations are located.
7731
7732	Return 0 if this is not a component reference or is one that we can't
7733	do anything with. */
7734
7735	static tree
7736	decode_field_reference (tree *pexp, HOST_WIDE_INT *pbitsize,
7737	HOST_WIDE_INT *pbitpos,
7738	bool *punsignedp, bool *preversep, bool *pvolatilep,
7739	wide_int *pand_mask, bool *psignbit,
7740	bool *pxorp, tree *pxor_cmp_op, wide_int *pxor_and_mask,
7741	gimple **pload, location_t loc[4])
7742	{
7743	tree exp = *pexp;
7744	tree outer_type = 0;
7745	wide_int and_mask;
7746	tree inner, offset;
7747	int shiftrt = 0;
7748	tree res_ops[2];
7749	machine_mode mode;
7750	bool convert_before_shift = false;
7751	bool signbit = false;
7752	bool xorp = false;
7753	tree xor_cmp_op;
7754	wide_int xor_and_mask;
7755	gimple *load = NULL;
7756
7757	/* All the optimizations using this function assume integer fields.
7758	There are problems with FP fields since the type_for_size call
7759	below can fail for, e.g., XFmode. */
7760	if (! INTEGRAL_TYPE_P (TREE_TYPE (exp)))
7761	return NULL_TREE;
7762
7763	/* Drop casts, saving only the outermost type, effectively used in
7764	the compare. We can deal with at most one conversion, and it may
7765	appear at various points in the chain of recognized preparation
7766	statements. Earlier optimizers will often have already dropped
7767	unneeded extensions, but they may survive, as in PR118046. ???
7768	Can we do better and allow multiple conversions, perhaps taking
7769	note of the narrowest intermediate type, sign extensions and
7770	whatnot? */
7771	if (!outer_type && gimple_convert_def_p (t: exp, op: res_ops))
7772	{
7773	outer_type = TREE_TYPE (exp);
7774	loc[0] = gimple_location (SSA_NAME_DEF_STMT (exp));
7775	exp = res_ops[0];
7776	}
7777
7778	/* Recognize and save a masking operation. Combine it with an
7779	incoming mask. */
7780	if (gimple_binop_def_p (code: BIT_AND_EXPR, t: exp, op: res_ops)
7781	&& TREE_CODE (res_ops[1]) == INTEGER_CST)
7782	{
7783	loc[1] = gimple_location (SSA_NAME_DEF_STMT (exp));
7784	exp = res_ops[0];
7785	and_mask = wi::to_wide (t: res_ops[1]);
7786	unsigned prec_in = pand_mask->get_precision ();
7787	if (prec_in)
7788	{
7789	unsigned prec_op = and_mask.get_precision ();
7790	if (prec_in >= prec_op)
7791	{
7792	if (prec_in > prec_op)
7793	and_mask = wide_int::from (x: and_mask, precision: prec_in, sgn: UNSIGNED);
7794	and_mask &= *pand_mask;
7795	}
7796	else
7797	and_mask &= wide_int::from (x: *pand_mask, precision: prec_op, sgn: UNSIGNED);
7798	}
7799	}
7800	else
7801	and_mask = *pand_mask;
7802
7803	/* Turn (a ^ b) [!]= 0 into a [!]= b. */
7804	if (pxorp && gimple_binop_def_p (code: BIT_XOR_EXPR, t: exp, op: res_ops))
7805	{
7806	/* No location recorded for this one, it's entirely subsumed by the
7807	compare. */
7808	if (*pxorp)
7809	{
7810	exp = res_ops[1];
7811	gcc_checking_assert (!pxor_cmp_op && !pxor_and_mask);
7812	}
7813	else if (!pxor_cmp_op)
7814	/* Not much we can do when xor appears in the right-hand compare
7815	operand. */
7816	return NULL_TREE;
7817	else if (integer_zerop (*pxor_cmp_op))
7818	{
7819	xorp = true;
7820	exp = res_ops[0];
7821	xor_cmp_op = *pexp;
7822	xor_and_mask = *pand_mask;
7823	}
7824	}
7825
7826	/* Another chance to drop conversions. */
7827	if (!outer_type && gimple_convert_def_p (t: exp, op: res_ops))
7828	{
7829	outer_type = TREE_TYPE (exp);
7830	loc[0] = gimple_location (SSA_NAME_DEF_STMT (exp));
7831	exp = res_ops[0];
7832	}
7833
7834	/* Take note of shifts. */
7835	if (gimple_binop_def_p (code: RSHIFT_EXPR, t: exp, op: res_ops)
7836	&& TREE_CODE (res_ops[1]) == INTEGER_CST)
7837	{
7838	loc[2] = gimple_location (SSA_NAME_DEF_STMT (exp));
7839	exp = res_ops[0];
7840	if (!tree_fits_shwi_p (res_ops[1]))
7841	return NULL_TREE;
7842	shiftrt = tree_to_shwi (res_ops[1]);
7843	if (shiftrt <= 0)
7844	return NULL_TREE;
7845	}
7846
7847	/* Yet another chance to drop conversions. This one is allowed to
7848	match a converting load, subsuming the load identification block
7849	below. */
7850	if (!outer_type && gimple_convert_def_p (t: exp, op: res_ops, load: &load))
7851	{
7852	outer_type = TREE_TYPE (exp);
7853	loc[0] = gimple_location (SSA_NAME_DEF_STMT (exp));
7854	if (load)
7855	loc[3] = gimple_location (g: load);
7856	exp = res_ops[0];
7857	/* This looks backwards, but we're going back the def chain, so if we
7858	find the conversion here, after finding a shift, that's because the
7859	convert appears before the shift, and we should thus adjust the bit
7860	pos and size because of the shift after adjusting it due to type
7861	conversion. */
7862	convert_before_shift = true;
7863	}
7864
7865	/* Identify the load, if there is one. */
7866	if (!load && TREE_CODE (exp) == SSA_NAME && !SSA_NAME_IS_DEFAULT_DEF (exp))
7867	{
7868	gimple *def = SSA_NAME_DEF_STMT (exp);
7869	if (gimple_assign_load_p (def))
7870	{
7871	loc[3] = gimple_location (g: def);
7872	load = def;
7873	exp = gimple_assign_rhs1 (gs: def);
7874	}
7875	}
7876
7877	/* Identify the relevant bits. */
7878	poly_int64 poly_bitsize, poly_bitpos;
7879	int unsignedp, reversep = *preversep, volatilep = *pvolatilep;
7880	inner = get_inner_reference (exp, &poly_bitsize, &poly_bitpos, &offset,
7881	&mode, &unsignedp, &reversep, &volatilep);
7882
7883	HOST_WIDE_INT bs, bp;
7884	if (!poly_bitsize.is_constant (const_value: &bs)
7885	|| !poly_bitpos.is_constant (const_value: &bp)
7886	|| bs <= shiftrt
7887	|| offset != 0
7888	|| TREE_CODE (inner) == PLACEHOLDER_EXPR
7889	/* Reject out-of-bound accesses (PR79731, PR118514). */
7890	|| !access_in_bounds_of_type_p (TREE_TYPE (inner), bs, bp)
7891	|| (INTEGRAL_TYPE_P (TREE_TYPE (inner))
7892	&& !type_has_mode_precision_p (TREE_TYPE (inner))))
7893	return NULL_TREE;
7894
7895	/* Adjust shifts... */
7896	if (convert_before_shift
7897	&& outer_type && bs > TYPE_PRECISION (outer_type))
7898	{
7899	HOST_WIDE_INT excess = bs - TYPE_PRECISION (outer_type);
7900	if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
7901	bp += excess;
7902	bs -= excess;
7903	}
7904
7905	if (shiftrt)
7906	{
7907	/* Punt if we're shifting by more than the loaded bitfield (after
7908	adjustment), or if there's a shift after a change of signedness, punt.
7909	When comparing this field with a constant, we'll check that the
7910	constant is a proper sign- or zero-extension (depending on signedness)
7911	of a value that would fit in the selected portion of the bitfield. A
7912	shift after a change of signedness would make the extension
7913	non-uniform, and we can't deal with that (yet ???). See
7914	gcc.dg/field-merge-22.c for a test that would go wrong. */
7915	if (bs <= shiftrt
7916	|| (convert_before_shift
7917	&& outer_type && unsignedp != TYPE_UNSIGNED (outer_type)))
7918	return NULL_TREE;
7919	if (!reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
7920	bp += shiftrt;
7921	bs -= shiftrt;
7922	}
7923
7924	/* ... and bit position. */
7925	if (!convert_before_shift
7926	&& outer_type && bs > TYPE_PRECISION (outer_type))
7927	{
7928	HOST_WIDE_INT excess = bs - TYPE_PRECISION (outer_type);
7929	if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
7930	bp += excess;
7931	bs -= excess;
7932	}
7933
7934	/* If the number of bits in the reference is the same as the bitsize of
7935	the outer type, then the outer type gives the signedness. Otherwise
7936	(in case of a small bitfield) the signedness is unchanged. */
7937	if (outer_type && bs == TYPE_PRECISION (outer_type))
7938	unsignedp = TYPE_UNSIGNED (outer_type);
7939
7940	/* Make the mask the expected width. */
7941	if (and_mask.get_precision () != 0)
7942	{
7943	/* If the AND_MASK encompasses bits that would be extensions of
7944	the sign bit, set SIGNBIT. */
7945	if (!unsignedp
7946	&& and_mask.get_precision () > bs
7947	&& (and_mask & wi::mask (width: bs, negate_p: true, precision: and_mask.get_precision ())) != 0)
7948	signbit = true;
7949	and_mask = wide_int::from (x: and_mask, precision: bs, sgn: UNSIGNED);
7950	}
7951
7952	*pexp = exp;
7953	*pload = load;
7954	*pbitsize = bs;
7955	*pbitpos = bp;
7956	*punsignedp = unsignedp;
7957	*preversep = reversep;
7958	*pvolatilep = volatilep;
7959	*psignbit = signbit;
7960	*pand_mask = and_mask;
7961	if (xorp)
7962	{
7963	*pxorp = xorp;
7964	*pxor_cmp_op = xor_cmp_op;
7965	*pxor_and_mask = xor_and_mask;
7966	}
7967
7968	return inner;
7969	}
7970
7971	/* Return the one bitpos within bit extents L or R that is at an
7972	ALIGN-bit alignment boundary, or -1 if there is more than one such
7973	boundary, if there isn't any, or if there is any such boundary
7974	between the extents. L and R are given by bitpos and bitsize. If
7975	it doesn't return -1, there are two consecutive ALIGN-bit words
7976	that contain both extents, and at least one of the extents
7977	straddles across the returned alignment boundary. */
7978
7979	static inline HOST_WIDE_INT
7980	compute_split_boundary_from_align (HOST_WIDE_INT align,
7981	HOST_WIDE_INT l_bitpos,
7982	HOST_WIDE_INT l_bitsize,
7983	HOST_WIDE_INT r_bitpos,
7984	HOST_WIDE_INT r_bitsize)
7985	{
7986	HOST_WIDE_INT amask = ~(align - 1);
7987
7988	HOST_WIDE_INT first_bit = MIN (l_bitpos, r_bitpos);
7989	HOST_WIDE_INT end_bit = MAX (l_bitpos + l_bitsize, r_bitpos + r_bitsize);
7990
7991	HOST_WIDE_INT boundary = (end_bit - 1) & amask;
7992
7993	/* Make sure we're crossing no more than one alignment boundary.
7994
7995	??? We don't have logic to recombine loads of two adjacent
7996	fields that each crosses a different alignment boundary, so
7997	as to load the middle word only once, if other words can't be
7998	otherwise recombined. */
7999	if (boundary - first_bit > align)
8000	return -1;
8001
8002	HOST_WIDE_INT l_start_word = l_bitpos & amask;
8003	HOST_WIDE_INT l_end_word = (l_bitpos + l_bitsize - 1) & amask;
8004
8005	HOST_WIDE_INT r_start_word = r_bitpos & amask;
8006	HOST_WIDE_INT r_end_word = (r_bitpos + r_bitsize - 1) & amask;
8007
8008	/* If neither field straddles across an alignment boundary, it's no
8009	use to even try to merge them. */
8010	if (l_start_word == l_end_word && r_start_word == r_end_word)
8011	return -1;
8012
8013	return boundary;
8014	}
8015
8016	/* Make a bit_field_ref. If POINT is NULL, return the BIT_FIELD_REF.
8017	Otherwise, build and insert a load stmt before POINT, and return
8018	the SSA_NAME. ??? Rewrite LOAD in terms of the bitfield? */
8019
8020	static tree
8021	make_bit_field_load (location_t loc, tree inner, tree orig_inner, tree type,
8022	HOST_WIDE_INT bitsize, poly_int64 bitpos,
8023	bool unsignedp, bool reversep, gimple *point)
8024	{
8025	if (point && loc == UNKNOWN_LOCATION)
8026	loc = gimple_location (g: point);
8027
8028	tree ref = make_bit_field_ref (loc, unshare_expr (inner),
8029	unshare_expr (orig_inner),
8030	type, bitsize, bitpos,
8031	unsignedp, reversep);
8032	if (!point)
8033	return ref;
8034
8035	/* If we're remaking the same load, reuse the SSA NAME it is already loaded
8036	into. */
8037	if (gimple_assign_load_p (point)
8038	&& operand_equal_p (ref, gimple_assign_rhs1 (gs: point)))
8039	{
8040	gcc_checking_assert (TREE_CODE (gimple_assign_lhs (point)) == SSA_NAME);
8041	return gimple_assign_lhs (gs: point);
8042	}
8043
8044	gimple_seq stmts = NULL;
8045	tree ret = force_gimple_operand (ref, &stmts, true, NULL_TREE);
8046
8047	/* We know the vuse is supposed to end up being the same as that at the
8048	original load at the insertion point, but if we don't set it, it will be a
8049	generic placeholder that only the global SSA update at the end of the pass
8050	would make equal, too late for us to use in further combinations. So go
8051	ahead and copy the vuse. */
8052
8053	tree reaching_vuse = gimple_vuse (g: point);
8054	for (gimple_stmt_iterator i = gsi_start (seq&: stmts);
8055	!gsi_end_p (i); gsi_next (i: &i))
8056	{
8057	gimple *new_stmt = gsi_stmt (i);
8058	if (gimple_has_mem_ops (g: new_stmt))
8059	gimple_set_vuse (g: new_stmt, vuse: reaching_vuse);
8060	}
8061
8062	gimple_stmt_iterator gsi = gsi_for_stmt (point);
8063	gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
8064	return ret;
8065	}
8066
8067	/* Initialize ln_arg[0] and ln_arg[1] to a pair of newly-created (at
8068	LOC) loads from INNER (from ORIG_INNER), of modes MODE and MODE2,
8069	respectively, starting at BIT_POS, using reversed endianness if
8070	REVERSEP. Also initialize BITPOS (the starting position of each
8071	part into INNER), BITSIZ (the bit count starting at BITPOS),
8072	TOSHIFT[1] (the amount by which the part and its mask are to be
8073	shifted right to bring its least-significant bit to bit zero) and
8074	SHIFTED (the amount by which the part, by separate loading, has
8075	already been shifted right, but that the mask needs shifting to
8076	match). */
8077
8078	static inline void
8079	build_split_load (tree /* out */ ln_arg[2],
8080	HOST_WIDE_INT /* out */ bitpos[2],
8081	HOST_WIDE_INT /* out */ bitsiz[2],
8082	HOST_WIDE_INT /* in[0] out[0..1] */ toshift[2],
8083	HOST_WIDE_INT /* out */ shifted[2],
8084	location_t loc, tree inner, tree orig_inner,
8085	scalar_int_mode mode, scalar_int_mode mode2,
8086	HOST_WIDE_INT bit_pos, bool reversep,
8087	gimple *point[2])
8088	{
8089	scalar_int_mode modes[2] = { mode, mode2 };
8090	bitsiz[0] = GET_MODE_BITSIZE (mode);
8091	bitsiz[1] = GET_MODE_BITSIZE (mode: mode2);
8092
8093	for (int i = 0; i < 2; i++)
8094	{
8095	tree type = lang_hooks.types.type_for_mode (modes[i], 1);
8096	if (!type)
8097	{
8098	type = build_nonstandard_integer_type (bitsiz[0], 1);
8099	gcc_assert (type);
8100	}
8101	bitpos[i] = bit_pos;
8102	ln_arg[i] = make_bit_field_load (loc, inner, orig_inner,
8103	type, bitsize: bitsiz[i],
8104	bitpos: bit_pos, unsignedp: 1, reversep, point: point[i]);
8105	bit_pos += bitsiz[i];
8106	}
8107
8108	toshift[1] = toshift[0];
8109	if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
8110	{
8111	shifted[0] = bitsiz[1];
8112	shifted[1] = 0;
8113	toshift[0] = 0;
8114	}
8115	else
8116	{
8117	shifted[1] = bitsiz[0];
8118	shifted[0] = 0;
8119	toshift[1] = 0;
8120	}
8121	}
8122
8123	/* Make arrangements to split at bit BOUNDARY a single loaded word
8124	(with REVERSEP bit order) LN_ARG[0], to be shifted right by
8125	TOSHIFT[0] to bring the field of interest to the least-significant
8126	bit. The expectation is that the same loaded word will be
8127	propagated from part 0 to part 1, with just different shifting and
8128	masking to extract both parts. MASK is not expected to do more
8129	than masking out the bits that belong to the other part. See
8130	build_split_load for more information on the other fields. */
8131
8132	static inline void
8133	reuse_split_load (tree /* in[0] out[1] */ ln_arg[2],
8134	HOST_WIDE_INT /* in[0] out[1] */ bitpos[2],
8135	HOST_WIDE_INT /* in[0] out[1] */ bitsiz[2],
8136	HOST_WIDE_INT /* in[0] out[0..1] */ toshift[2],
8137	HOST_WIDE_INT /* out */ shifted[2],
8138	wide_int /* out */ mask[2],
8139	HOST_WIDE_INT boundary, bool reversep)
8140	{
8141	unsigned prec = TYPE_PRECISION (TREE_TYPE (ln_arg[0]));
8142
8143	ln_arg[1] = ln_arg[0];
8144	bitpos[1] = bitpos[0];
8145	bitsiz[1] = bitsiz[0];
8146	shifted[1] = shifted[0] = 0;
8147
8148	if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
8149	{
8150	toshift[1] = toshift[0];
8151	toshift[0] = bitpos[0] + bitsiz[0] - boundary;
8152	mask[0] = wi::mask (width: toshift[0], negate_p: true, precision: prec);
8153	mask[1] = wi::mask (width: toshift[0], negate_p: false, precision: prec);
8154	}
8155	else
8156	{
8157	toshift[1] = boundary - bitpos[1];
8158	mask[1] = wi::mask (width: toshift[1], negate_p: true, precision: prec);
8159	mask[0] = wi::mask (width: toshift[1], negate_p: false, precision: prec);
8160	}
8161	}
8162
8163	/* Find ways of folding logical expressions of LHS and RHS:
8164
8165	Try to merge two comparisons to nearby fields.
8166
8167	For example, if we have p->a == 2 && p->b == 4 and we can load both A and B
8168	at once, we can do this with a comparison against the object ANDed with the
8169	a mask.
8170
8171	If we have p->a == q->a && p->b == q->b, we may be able to use bit masking
8172	operations to do this with one comparison, loading both fields from P at
8173	once, and likewise from Q.
8174
8175	Herein, loading at once means loading from within the same alignment
8176	boundary for the enclosing object. If (packed) fields cross such alignment
8177	boundaries, we may still recombine the compares, so that loads do not cross
8178	the boundaries.
8179
8180	CODE is the logical operation being done. It can be TRUTH_ANDIF_EXPR,
8181	TRUTH_AND_EXPR, TRUTH_ORIF_EXPR, or TRUTH_OR_EXPR.
8182
8183	TRUTH_TYPE is the type of the logical operand.
8184
8185	LHS is denoted as LL_ARG LCODE LR_ARG.
8186
8187	RHS is denoted as RL_ARG RCODE RR_ARG.
8188
8189	LHS is assumed to dominate RHS.
8190
8191	Combined loads are inserted next to preexisting loads, once we determine
8192	that the combination is viable, and the combined condition references new
8193	SSA_NAMEs that hold the loaded values. Since the original loads are
8194	verified to have the same gimple_vuse, the insertion point doesn't matter
8195	for correctness. ??? The loads may be a lot earlier than the compares, and
8196	it's conceivable that one or two loads for RHS appear before those for LHS.
8197	It could be advantageous to try to place the loads optimally, taking
8198	advantage of knowing whether RHS is accessed before LHS, or that both are
8199	accessed before both compares, but we don't do that (yet?).
8200
8201	SEPARATEP should be NULL if the combined condition must be returned as a
8202	single expression, even if it is a compound condition. This must only be
8203	done if LHS and RHS are adjacent, without intervening conditions, and the
8204	combined condition is to replace RHS, while LHS is dropped altogether.
8205
8206	Otherwise, SEPARATEP must be a non-NULL pointer to a NULL_TREE, that may be
8207	replaced by a part of the compound condition that could replace RHS, while
8208	the returned expression replaces LHS. This works whether or not LHS and RHS
8209	are adjacent, as long as there aren't VDEFs or other side effects between
8210	them.
8211
8212	If the "words" accessed by RHS are already accessed by LHS, this won't
8213	matter, but if RHS accesses "words" that LHS doesn't, then *SEPARATEP will
8214	be set to the compares that should take RHS's place. By "words" we mean
8215	contiguous bits that do not cross a an TYPE_ALIGN boundary of the accessed
8216	object's type.
8217
8218	We return the simplified tree or 0 if no optimization is possible. */
8219
8220	tree
8221	fold_truth_andor_for_ifcombine (enum tree_code code, tree truth_type,
8222	location_t lloc, enum tree_code lcode,
8223	tree ll_arg, tree lr_arg,
8224	location_t rloc, enum tree_code rcode,
8225	tree rl_arg, tree rr_arg,
8226	tree *separatep)
8227	{
8228	/* If this is the "or" of two comparisons, we can do something if
8229	the comparisons are NE_EXPR. If this is the "and", we can do something
8230	if the comparisons are EQ_EXPR. I.e.,
8231	(a->b == 2 && a->c == 4) can become (a->new == NEW).
8232
8233	WANTED_CODE is this operation code. For single bit fields, we can
8234	convert EQ_EXPR to NE_EXPR so we need not reject the "wrong"
8235	comparison for one-bit fields. */
8236
8237	enum tree_code orig_code = code;
8238	enum tree_code wanted_code;
8239	tree ll_inner, lr_inner, rl_inner, rr_inner;
8240	gimple *ll_load, *lr_load, *rl_load, *rr_load;
8241	HOST_WIDE_INT ll_bitsize, ll_bitpos, lr_bitsize, lr_bitpos;
8242	HOST_WIDE_INT rl_bitsize, rl_bitpos, rr_bitsize, rr_bitpos;
8243	HOST_WIDE_INT xll_bitpos, xlr_bitpos, xrl_bitpos, xrr_bitpos;
8244	HOST_WIDE_INT lnbitsize, lnbitpos, lnprec;
8245	HOST_WIDE_INT rnbitsize, rnbitpos, rnprec;
8246	bool ll_unsignedp, lr_unsignedp, rl_unsignedp, rr_unsignedp;
8247	bool ll_reversep, lr_reversep, rl_reversep, rr_reversep;
8248	bool ll_signbit, lr_signbit, rl_signbit, rr_signbit;
8249	scalar_int_mode lnmode, lnmode2, rnmode;
8250	wide_int ll_and_mask, lr_and_mask, rl_and_mask, rr_and_mask;
8251	wide_int l_const, r_const;
8252	tree lntype, rntype, result;
8253	HOST_WIDE_INT first_bit, end_bit;
8254	bool volatilep;
8255	bool l_split_load;
8256
8257	/* These are indexed by: conv, mask, shft, load. */
8258	location_t ll_loc[4] = { lloc, lloc, lloc, UNKNOWN_LOCATION };
8259	location_t lr_loc[4] = { lloc, lloc, lloc, UNKNOWN_LOCATION };
8260	location_t rl_loc[4] = { rloc, rloc, rloc, UNKNOWN_LOCATION };
8261	location_t rr_loc[4] = { rloc, rloc, rloc, UNKNOWN_LOCATION };
8262
8263	gcc_checking_assert (!separatep || !*separatep);
8264
8265	/* Start by getting the comparison codes. Fail if anything is volatile.
8266	If one operand is a BIT_AND_EXPR with the constant one, treat it as if
8267	it were surrounded with a NE_EXPR. */
8268
8269	if (TREE_CODE_CLASS (lcode) != tcc_comparison
8270	|| TREE_CODE_CLASS (rcode) != tcc_comparison)
8271	return 0;
8272
8273	/* We don't normally find TRUTH_*IF_EXPR in gimple, but these codes may be
8274	given by our caller to denote conditions from different blocks. */
8275	switch (code)
8276	{
8277	case TRUTH_AND_EXPR:
8278	case TRUTH_ANDIF_EXPR:
8279	code = TRUTH_AND_EXPR;
8280	break;
8281
8282	case TRUTH_OR_EXPR:
8283	case TRUTH_ORIF_EXPR:
8284	code = TRUTH_OR_EXPR;
8285	break;
8286
8287	default:
8288	return 0;
8289	}
8290
8291	/* Prepare to turn compares of signed quantities with zero into sign-bit
8292	tests. We need not worry about *_reversep here for these compare
8293	rewrites: loads will have already been reversed before compares. Save the
8294	precision, because [lr]l_arg may change and we won't be able to tell how
8295	wide it was originally. */
8296	unsigned lsignbit = 0, rsignbit = 0;
8297	if ((lcode == LT_EXPR || lcode == GE_EXPR)
8298	&& integer_zerop (lr_arg)
8299	&& INTEGRAL_TYPE_P (TREE_TYPE (ll_arg))
8300	&& !TYPE_UNSIGNED (TREE_TYPE (ll_arg)))
8301	{
8302	lsignbit = TYPE_PRECISION (TREE_TYPE (ll_arg));
8303	lcode = (lcode == LT_EXPR ? NE_EXPR : EQ_EXPR);
8304	}
8305	/* Turn compares of unsigned quantities with powers of two into
8306	equality tests of masks. */
8307	else if ((lcode == LT_EXPR || lcode == GE_EXPR)
8308	&& INTEGRAL_TYPE_P (TREE_TYPE (ll_arg))
8309	&& TYPE_UNSIGNED (TREE_TYPE (ll_arg))
8310	&& TREE_CODE (lr_arg) == INTEGER_CST
8311	&& wi::popcount (wi::to_wide (t: lr_arg)) == 1)
8312	{
8313	ll_and_mask = ~(wi::to_wide (t: lr_arg) - 1);
8314	lcode = (lcode == GE_EXPR ? NE_EXPR : EQ_EXPR);
8315	lr_arg = wide_int_to_tree (TREE_TYPE (ll_arg), cst: ll_and_mask * 0);
8316	}
8317	/* Turn compares of unsigned quantities with powers of two minus one
8318	into equality tests of masks. */
8319	else if ((lcode == LE_EXPR || lcode == GT_EXPR)
8320	&& INTEGRAL_TYPE_P (TREE_TYPE (ll_arg))
8321	&& TYPE_UNSIGNED (TREE_TYPE (ll_arg))
8322	&& TREE_CODE (lr_arg) == INTEGER_CST
8323	&& wi::popcount (wi::to_wide (t: lr_arg) + 1) == 1)
8324	{
8325	ll_and_mask = ~wi::to_wide (t: lr_arg);
8326	lcode = (lcode == GT_EXPR ? NE_EXPR : EQ_EXPR);
8327	lr_arg = wide_int_to_tree (TREE_TYPE (ll_arg), cst: ll_and_mask * 0);
8328	}
8329	/* Likewise for the second compare. */
8330	if ((rcode == LT_EXPR || rcode == GE_EXPR)
8331	&& integer_zerop (rr_arg)
8332	&& INTEGRAL_TYPE_P (TREE_TYPE (rl_arg))
8333	&& !TYPE_UNSIGNED (TREE_TYPE (rl_arg)))
8334	{
8335	rsignbit = TYPE_PRECISION (TREE_TYPE (rl_arg));
8336	rcode = (rcode == LT_EXPR ? NE_EXPR : EQ_EXPR);
8337	}
8338	else if ((rcode == LT_EXPR || rcode == GE_EXPR)
8339	&& INTEGRAL_TYPE_P (TREE_TYPE (rl_arg))
8340	&& TYPE_UNSIGNED (TREE_TYPE (rl_arg))
8341	&& TREE_CODE (rr_arg) == INTEGER_CST
8342	&& wi::popcount (wi::to_wide (t: rr_arg)) == 1)
8343	{
8344	rl_and_mask = ~(wi::to_wide (t: rr_arg) - 1);
8345	rcode = (rcode == GE_EXPR ? NE_EXPR : EQ_EXPR);
8346	rr_arg = wide_int_to_tree (TREE_TYPE (rl_arg), cst: rl_and_mask * 0);
8347	}
8348	else if ((rcode == LE_EXPR || rcode == GT_EXPR)
8349	&& INTEGRAL_TYPE_P (TREE_TYPE (rl_arg))
8350	&& TYPE_UNSIGNED (TREE_TYPE (rl_arg))
8351	&& TREE_CODE (rr_arg) == INTEGER_CST
8352	&& wi::popcount (wi::to_wide (t: rr_arg) + 1) == 1)
8353	{
8354	rl_and_mask = ~wi::to_wide (t: rr_arg);
8355	rcode = (rcode == GT_EXPR ? NE_EXPR : EQ_EXPR);
8356	rr_arg = wide_int_to_tree (TREE_TYPE (rl_arg), cst: rl_and_mask * 0);
8357	}
8358
8359	/* See if the comparisons can be merged. Then get all the parameters for
8360	each side. */
8361
8362	if ((lcode != EQ_EXPR && lcode != NE_EXPR)
8363	|| (rcode != EQ_EXPR && rcode != NE_EXPR))
8364	return 0;
8365
8366	ll_reversep = lr_reversep = rl_reversep = rr_reversep = 0;
8367	volatilep = 0;
8368	bool l_xor = false, r_xor = false;
8369	ll_inner = decode_field_reference (pexp: &ll_arg, pbitsize: &ll_bitsize, pbitpos: &ll_bitpos,
8370	punsignedp: &ll_unsignedp, preversep: &ll_reversep, pvolatilep: &volatilep,
8371	pand_mask: &ll_and_mask, psignbit: &ll_signbit,
8372	pxorp: &l_xor, pxor_cmp_op: &lr_arg, pxor_and_mask: &lr_and_mask,
8373	pload: &ll_load, loc: ll_loc);
8374	if (!ll_inner)
8375	return 0;
8376	lr_inner = decode_field_reference (pexp: &lr_arg, pbitsize: &lr_bitsize, pbitpos: &lr_bitpos,
8377	punsignedp: &lr_unsignedp, preversep: &lr_reversep, pvolatilep: &volatilep,
8378	pand_mask: &lr_and_mask, psignbit: &lr_signbit, pxorp: &l_xor, pxor_cmp_op: 0, pxor_and_mask: 0,
8379	pload: &lr_load, loc: lr_loc);
8380	if (!lr_inner)
8381	return 0;
8382	rl_inner = decode_field_reference (pexp: &rl_arg, pbitsize: &rl_bitsize, pbitpos: &rl_bitpos,
8383	punsignedp: &rl_unsignedp, preversep: &rl_reversep, pvolatilep: &volatilep,
8384	pand_mask: &rl_and_mask, psignbit: &rl_signbit,
8385	pxorp: &r_xor, pxor_cmp_op: &rr_arg, pxor_and_mask: &rr_and_mask,
8386	pload: &rl_load, loc: rl_loc);
8387	if (!rl_inner)
8388	return 0;
8389	rr_inner = decode_field_reference (pexp: &rr_arg, pbitsize: &rr_bitsize, pbitpos: &rr_bitpos,
8390	punsignedp: &rr_unsignedp, preversep: &rr_reversep, pvolatilep: &volatilep,
8391	pand_mask: &rr_and_mask, psignbit: &rr_signbit, pxorp: &r_xor, pxor_cmp_op: 0, pxor_and_mask: 0,
8392	pload: &rr_load, loc: rr_loc);
8393	if (!rr_inner)
8394	return 0;
8395
8396	/* It must be true that the inner operation on the lhs of each
8397	comparison must be the same if we are to be able to do anything.
8398	Then see if we have constants. If not, the same must be true for
8399	the rhs's. If one is a load and the other isn't, we have to be
8400	conservative and avoid the optimization, otherwise we could get
8401	SRAed fields wrong. */
8402	if (volatilep)
8403	return 0;
8404
8405	if (ll_reversep != rl_reversep
8406	|| ! operand_equal_p (ll_inner, rl_inner, flags: 0))
8407	{
8408	/* Try swapping the operands. */
8409	if (ll_reversep != rr_reversep || rsignbit
8410	|| !operand_equal_p (ll_inner, rr_inner, flags: 0))
8411	return 0;
8412
8413	rcode = swap_tree_comparison (rcode);
8414	std::swap (a&: rl_arg, b&: rr_arg);
8415	std::swap (a&: rl_inner, b&: rr_inner);
8416	std::swap (a&: rl_bitsize, b&: rr_bitsize);
8417	std::swap (a&: rl_bitpos, b&: rr_bitpos);
8418	std::swap (a&: rl_unsignedp, b&: rr_unsignedp);
8419	std::swap (a&: rl_reversep, b&: rr_reversep);
8420	std::swap (a&: rl_and_mask, b&: rr_and_mask);
8421	std::swap (a&: rl_signbit, b&: rr_signbit);
8422	std::swap (a&: rl_load, b&: rr_load);
8423	std::swap (a&: rl_loc, b&: rr_loc);
8424	}
8425
8426	if ((ll_load && rl_load)
8427	? gimple_vuse (g: ll_load) != gimple_vuse (g: rl_load)
8428	: (!ll_load != !rl_load))
8429	return 0;
8430
8431	/* ??? Can we do anything with these? */
8432	if (lr_signbit || rr_signbit)
8433	return 0;
8434
8435	/* If the mask encompassed extensions of the sign bit before
8436	clipping, try to include the sign bit in the test. If we're not
8437	comparing with zero, don't even try to deal with it (for now?).
8438	If we've already commited to a sign test, the extended (before
8439	clipping) mask could already be messing with it. */
8440	if (ll_signbit)
8441	{
8442	if (!integer_zerop (lr_arg) || lsignbit)
8443	return 0;
8444	wide_int sign = wi::mask (width: ll_bitsize - 1, negate_p: true, precision: ll_bitsize);
8445	if (!ll_and_mask.get_precision ())
8446	ll_and_mask = sign;
8447	else
8448	ll_and_mask |= sign;
8449	}
8450
8451	if (rl_signbit)
8452	{
8453	if (!integer_zerop (rr_arg) || rsignbit)
8454	return 0;
8455	wide_int sign = wi::mask (width: rl_bitsize - 1, negate_p: true, precision: rl_bitsize);
8456	if (!rl_and_mask.get_precision ())
8457	rl_and_mask = sign;
8458	else
8459	rl_and_mask |= sign;
8460	}
8461
8462	if (TREE_CODE (lr_arg) == INTEGER_CST
8463	&& TREE_CODE (rr_arg) == INTEGER_CST)
8464	{
8465	l_const = wi::to_wide (t: lr_arg);
8466	/* We don't expect masks on constants, but if there are any, apply
8467	them now. */
8468	if (lr_and_mask.get_precision ())
8469	l_const &= wide_int::from (x: lr_and_mask,
8470	precision: l_const.get_precision (), sgn: UNSIGNED);
8471	r_const = wi::to_wide (t: rr_arg);
8472	if (rr_and_mask.get_precision ())
8473	r_const &= wide_int::from (x: rr_and_mask,
8474	precision: r_const.get_precision (), sgn: UNSIGNED);
8475	lr_reversep = ll_reversep;
8476	}
8477	else if (lr_reversep != rr_reversep
8478	|| ! operand_equal_p (lr_inner, rr_inner, flags: 0)
8479	|| ((lr_load && rr_load)
8480	? gimple_vuse (g: lr_load) != gimple_vuse (g: rr_load)
8481	: (!lr_load != !rr_load)))
8482	return 0;
8483
8484	/* If we found sign tests, finish turning them into bit tests. */
8485
8486	if (lsignbit)
8487	{
8488	wide_int sign = wi::mask (width: ll_bitsize - 1, negate_p: true, precision: ll_bitsize);
8489	/* If ll_arg is zero-extended and we're testing the sign bit, we know
8490	what the result should be. Shifting the sign bit out of sign will get
8491	us to mask the entire field out, yielding zero, i.e., the sign bit of
8492	the zero-extended value. We know the masked value is being compared
8493	with zero, so the compare will get us the result we're looking
8494	for: TRUE if EQ_EXPR, FALSE if NE_EXPR. */
8495	if (lsignbit > ll_bitsize && ll_unsignedp)
8496	sign <<= 1;
8497	if (!ll_and_mask.get_precision ())
8498	ll_and_mask = sign;
8499	else
8500	ll_and_mask &= sign;
8501	if (l_xor)
8502	{
8503	if (ll_bitsize != lr_bitsize)
8504	return 0;
8505	if (!lr_and_mask.get_precision ())
8506	lr_and_mask = sign;
8507	else
8508	lr_and_mask &= sign;
8509	if (l_const.get_precision ())
8510	l_const &= wide_int::from (x: lr_and_mask,
8511	precision: l_const.get_precision (), sgn: UNSIGNED);
8512	}
8513	}
8514
8515	if (rsignbit)
8516	{
8517	wide_int sign = wi::mask (width: rl_bitsize - 1, negate_p: true, precision: rl_bitsize);
8518	if (rsignbit > rl_bitsize && rl_unsignedp)
8519	sign <<= 1;
8520	if (!rl_and_mask.get_precision ())
8521	rl_and_mask = sign;
8522	else
8523	rl_and_mask &= sign;
8524	if (r_xor)
8525	{
8526	if (rl_bitsize != rr_bitsize)
8527	return 0;
8528	if (!rr_and_mask.get_precision ())
8529	rr_and_mask = sign;
8530	else
8531	rr_and_mask &= sign;
8532	if (r_const.get_precision ())
8533	r_const &= wide_int::from (x: rr_and_mask,
8534	precision: r_const.get_precision (), sgn: UNSIGNED);
8535	}
8536	}
8537
8538	/* If either comparison code is not correct for our logical operation,
8539	fail. However, we can convert a one-bit comparison against zero into
8540	the opposite comparison against that bit being set in the field. */
8541
8542	wanted_code = (code == TRUTH_AND_EXPR ? EQ_EXPR : NE_EXPR);
8543	if (lcode != wanted_code)
8544	{
8545	if (l_const.get_precision ()
8546	&& l_const == 0
8547	&& ll_and_mask.get_precision ()
8548	&& wi::popcount (ll_and_mask) == 1)
8549	{
8550	/* Make the left operand unsigned, since we are only interested
8551	in the value of one bit. Otherwise we are doing the wrong
8552	thing below. */
8553	ll_unsignedp = 1;
8554	l_const = ll_and_mask;
8555	}
8556	else
8557	return 0;
8558	}
8559
8560	/* This is analogous to the code for l_const above. */
8561	if (rcode != wanted_code)
8562	{
8563	if (r_const.get_precision ()
8564	&& r_const == 0
8565	&& rl_and_mask.get_precision ()
8566	&& wi::popcount (rl_and_mask) == 1)
8567	{
8568	rl_unsignedp = 1;
8569	r_const = rl_and_mask;
8570	}
8571	else
8572	return 0;
8573	}
8574
8575	/* This will be bumped to 2 if any of the field pairs crosses an
8576	alignment boundary, so the merged compare has to be done in two
8577	parts. */
8578	int parts = 1;
8579	/* Set to true if the second combined compare should come first,
8580	e.g., because the second original compare accesses a word that
8581	the first one doesn't, and the combined compares access those in
8582	cmp[0]. */
8583	bool first1 = false;
8584	/* Set to true if the first original compare is not the one being
8585	split. */
8586	bool maybe_separate = false;
8587
8588	/* The following 2-dimensional arrays use the first index to
8589	identify left(0)- vs right(1)-hand compare operands, and the
8590	second one to identify merged compare parts. */
8591	/* The memory loads or constants to be compared. */
8592	tree ld_arg[2][2];
8593	/* The first bit of the corresponding inner object that the
8594	corresponding LD_ARG covers. */
8595	HOST_WIDE_INT bitpos[2][2];
8596	/* The bit count starting at BITPOS that the corresponding LD_ARG
8597	covers. */
8598	HOST_WIDE_INT bitsiz[2][2];
8599	/* The number of bits by which LD_ARG has already been shifted
8600	right, WRT mask. */
8601	HOST_WIDE_INT shifted[2][2];
8602	/* The number of bits by which both LD_ARG and MASK need shifting to
8603	bring its least-significant bit to bit zero. */
8604	HOST_WIDE_INT toshift[2][2];
8605	/* An additional mask to be applied to LD_ARG, to remove any bits
8606	that may have been loaded for use in another compare, but that
8607	don't belong in the corresponding compare. */
8608	wide_int xmask[2][2] = {};
8609
8610	/* The combined compare or compares. */
8611	tree cmp[2];
8612
8613	/* Consider we're comparing two non-contiguous fields of packed
8614	structs, both aligned at 32-bit boundaries:
8615
8616	ll_arg: an 8-bit field at offset 0
8617	lr_arg: a 16-bit field at offset 2
8618
8619	rl_arg: an 8-bit field at offset 1
8620	rr_arg: a 16-bit field at offset 3
8621
8622	We'll have r_split_load, because rr_arg straddles across an
8623	alignment boundary.
8624
8625	We'll want to have:
8626
8627	bitpos = { { 0, 0 }, { 0, 32 } }
8628	bitsiz = { { 32, 32 }, { 32, 8 } }
8629
8630	And, for little-endian:
8631
8632	shifted = { { 0, 0 }, { 0, 32 } }
8633	toshift = { { 0, 24 }, { 0, 0 } }
8634
8635	Or, for big-endian:
8636
8637	shifted = { { 0, 0 }, { 8, 0 } }
8638	toshift = { { 8, 0 }, { 0, 0 } }
8639	*/
8640
8641	/* See if we can find a mode that contains both fields being compared on
8642	the left. If we can't, fail. Otherwise, update all constants and masks
8643	to be relative to a field of that size. */
8644	first_bit = MIN (ll_bitpos, rl_bitpos);
8645	end_bit = MAX (ll_bitpos + ll_bitsize, rl_bitpos + rl_bitsize);
8646	HOST_WIDE_INT ll_align = TYPE_ALIGN (TREE_TYPE (ll_inner));
8647	poly_uint64 ll_end_region = 0;
8648	if (TYPE_SIZE (TREE_TYPE (ll_inner))
8649	&& tree_fits_poly_uint64_p (TYPE_SIZE (TREE_TYPE (ll_inner))))
8650	ll_end_region = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (ll_inner)));
8651	if (get_best_mode (end_bit - first_bit, first_bit, 0, ll_end_region,
8652	ll_align, BITS_PER_WORD, volatilep, &lnmode))
8653	l_split_load = false;
8654	/* ??? If ll and rl share the same load, reuse that?
8655	See PR 118206 -> gcc.dg/field-merge-18.c */
8656	else
8657	{
8658	/* Consider the possibility of recombining loads if any of the
8659	fields straddles across an alignment boundary, so that either
8660	part can be loaded along with the other field. Since we
8661	limit access modes to BITS_PER_WORD, don't exceed that,
8662	otherwise on a 32-bit host and a 64-bit-aligned data
8663	structure, we'll fail the above for a field that straddles
8664	across two words, and would fail here for not even trying to
8665	split it at between 32-bit words. */
8666	HOST_WIDE_INT boundary = compute_split_boundary_from_align
8667	(MIN (ll_align, BITS_PER_WORD),
8668	l_bitpos: ll_bitpos, l_bitsize: ll_bitsize, r_bitpos: rl_bitpos, r_bitsize: rl_bitsize);
8669
8670	if (boundary < 0
8671	|| !get_best_mode (boundary - first_bit, first_bit, 0, ll_end_region,
8672	ll_align, BITS_PER_WORD, volatilep, &lnmode)
8673	|| !get_best_mode (end_bit - boundary, boundary, 0, ll_end_region,
8674	ll_align, BITS_PER_WORD, volatilep, &lnmode2))
8675	{
8676	if (ll_align <= BITS_PER_WORD)
8677	return 0;
8678
8679	/* As a last resort, try double-word access modes. This
8680	enables us to deal with misaligned double-word fields
8681	that straddle across 3 separate words. */
8682	boundary = compute_split_boundary_from_align
8683	(MIN (ll_align, 2 * BITS_PER_WORD),
8684	l_bitpos: ll_bitpos, l_bitsize: ll_bitsize, r_bitpos: rl_bitpos, r_bitsize: rl_bitsize);
8685	if (boundary < 0
8686	|| !get_best_mode (boundary - first_bit, first_bit,
8687	0, ll_end_region, ll_align, 2 * BITS_PER_WORD,
8688	volatilep, &lnmode)
8689	|| !get_best_mode (end_bit - boundary, boundary,
8690	0, ll_end_region, ll_align, 2 * BITS_PER_WORD,
8691	volatilep, &lnmode2))
8692	return 0;
8693	}
8694
8695	/* If we can't have a single load, but can with two, figure out whether
8696	the two compares can be separated, i.e., whether the entirety of the
8697	first original compare is encompassed by the entirety of the first
8698	combined compare. If the first original compare is past the alignment
8699	boundary, arrange to compare that range first, by setting first1
8700	(meaning make cmp[1] first, instead of cmp[0]). */
8701	l_split_load = true;
8702	parts = 2;
8703	if (ll_bitpos >= boundary)
8704	maybe_separate = first1 = true;
8705	else if (ll_bitpos + ll_bitsize <= boundary)
8706	maybe_separate = true;
8707	}
8708
8709	lnbitsize = GET_MODE_BITSIZE (mode: lnmode);
8710	lnbitpos = first_bit & ~ (lnbitsize - 1);
8711	/* Avoid situations that the code below can't handle. */
8712	if (lnbitpos < 0)
8713	return 0;
8714
8715	/* Choose the type for the combined compare. Even if we're splitting loads,
8716	make it wide enough to hold both. */
8717	if (l_split_load)
8718	lnbitsize += GET_MODE_BITSIZE (mode: lnmode2);
8719	lntype = build_nonstandard_integer_type (lnbitsize, 1);
8720	if (!lntype)
8721	return NULL_TREE;
8722	lnprec = TYPE_PRECISION (lntype);
8723	xll_bitpos = ll_bitpos - lnbitpos, xrl_bitpos = rl_bitpos - lnbitpos;
8724
8725	/* Adjust bit ranges for reverse endianness. */
8726	if (ll_reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
8727	{
8728	xll_bitpos = lnbitsize - xll_bitpos - ll_bitsize;
8729	xrl_bitpos = lnbitsize - xrl_bitpos - rl_bitsize;
8730	}
8731
8732	/* Adjust masks to match the positions in the combined lntype. */
8733	wide_int ll_mask, rl_mask, r_mask;
8734	if (ll_and_mask.get_precision ())
8735	ll_mask = wi::lshift (x: wide_int::from (x: ll_and_mask, precision: lnprec, sgn: UNSIGNED),
8736	y: xll_bitpos);
8737	else
8738	ll_mask = wi::shifted_mask (start: xll_bitpos, width: ll_bitsize, negate_p: false, precision: lnprec);
8739	if (rl_and_mask.get_precision ())
8740	rl_mask = wi::lshift (x: wide_int::from (x: rl_and_mask, precision: lnprec, sgn: UNSIGNED),
8741	y: xrl_bitpos);
8742	else
8743	rl_mask = wi::shifted_mask (start: xrl_bitpos, width: rl_bitsize, negate_p: false, precision: lnprec);
8744
8745	/* When we set l_const, we also set r_const. */
8746	gcc_checking_assert (!l_const.get_precision () == !r_const.get_precision ());
8747
8748	/* Adjust right-hand constants in both original comparisons to match width
8749	and bit position. */
8750	if (l_const.get_precision ())
8751	{
8752	/* Before clipping upper bits of the right-hand operand of the compare,
8753	check that they're sign or zero extensions, depending on how the
8754	left-hand operand would be extended. If it is unsigned, or if there's
8755	a mask that zeroes out extension bits, whether because we've checked
8756	for upper bits in the mask and did not set ll_signbit, or because the
8757	sign bit itself is masked out, check that the right-hand operand is
8758	zero-extended. */
8759	bool l_non_ext_bits = false;
8760	if (ll_bitsize < lr_bitsize)
8761	{
8762	wide_int zext = wi::zext (x: l_const, offset: ll_bitsize);
8763	if ((ll_unsignedp
8764	|| (ll_and_mask.get_precision ()
8765	&& (!ll_signbit
8766	|| ((ll_and_mask & wi::mask (width: ll_bitsize - 1, negate_p: true, precision: ll_bitsize))
8767	== 0)))
8768	? zext : wi::sext (x: l_const, offset: ll_bitsize)) == l_const)
8769	l_const = zext;
8770	else
8771	l_non_ext_bits = true;
8772	}
8773	/* We're doing bitwise equality tests, so don't bother with sign
8774	extensions. */
8775	l_const = wide_int::from (x: l_const, precision: lnprec, sgn: UNSIGNED);
8776	if (ll_and_mask.get_precision ())
8777	l_const &= wide_int::from (x: ll_and_mask, precision: lnprec, sgn: UNSIGNED);
8778	l_const <<= xll_bitpos;
8779	if (l_non_ext_bits || (l_const & ~ll_mask) != 0)
8780	{
8781	warning_at (lloc, OPT_Wtautological_compare,
8782	"comparison is always %d", wanted_code == NE_EXPR);
8783
8784	return constant_boolean_node (wanted_code == NE_EXPR, truth_type);
8785	}
8786
8787	/* Before clipping upper bits of the right-hand operand of the compare,
8788	check that they're sign or zero extensions, depending on how the
8789	left-hand operand would be extended. */
8790	bool r_non_ext_bits = false;
8791	if (rl_bitsize < rr_bitsize)
8792	{
8793	wide_int zext = wi::zext (x: r_const, offset: rl_bitsize);
8794	if ((rl_unsignedp
8795	|| (rl_and_mask.get_precision ()
8796	&& (!rl_signbit
8797	|| ((rl_and_mask & wi::mask (width: rl_bitsize - 1, negate_p: true, precision: rl_bitsize))
8798	== 0)))
8799	? zext : wi::sext (x: r_const, offset: rl_bitsize)) == r_const)
8800	r_const = zext;
8801	else
8802	r_non_ext_bits = true;
8803	}
8804	r_const = wide_int::from (x: r_const, precision: lnprec, sgn: UNSIGNED);
8805	if (rl_and_mask.get_precision ())
8806	r_const &= wide_int::from (x: rl_and_mask, precision: lnprec, sgn: UNSIGNED);
8807	r_const <<= xrl_bitpos;
8808	if (r_non_ext_bits || (r_const & ~rl_mask) != 0)
8809	{
8810	warning_at (rloc, OPT_Wtautological_compare,
8811	"comparison is always %d", wanted_code == NE_EXPR);
8812
8813	return constant_boolean_node (wanted_code == NE_EXPR, truth_type);
8814	}
8815
8816	/* If there is something in common between the masks, those bits of the
8817	constants must be the same. If not, the combined condition cannot be
8818	met, and the result is known. Test for this to avoid generating
8819	incorrect code below. */
8820	wide_int mask = ll_mask & rl_mask;
8821	if (mask != 0
8822	&& (l_const & mask) != (r_const & mask))
8823	{
8824	if (wanted_code == NE_EXPR)
8825	return constant_boolean_node (true, truth_type);
8826	else
8827	return constant_boolean_node (false, truth_type);
8828	}
8829
8830	/* The constants are combined so as to line up with the loaded field, so
8831	tentatively use the same parameters for the second combined
8832	compare. */
8833	ld_arg[1][0] = wide_int_to_tree (type: lntype, cst: l_const | r_const);
8834	toshift[1][0] = MIN (xll_bitpos, xrl_bitpos);
8835	shifted[1][0] = 0;
8836	bitpos[1][0] = lnbitpos;
8837	bitsiz[1][0] = lnbitsize;
8838
8839	if (parts > 1)
8840	reuse_split_load (ln_arg: ld_arg[1], bitpos: bitpos[1], bitsiz: bitsiz[1], toshift: toshift[1],
8841	shifted: shifted[1], mask: xmask[1],
8842	boundary: lnbitpos + GET_MODE_BITSIZE (mode: lnmode),
8843	reversep: lr_reversep);
8844
8845	/* No masking needed, we know the full constants. */
8846	r_mask = wi::mask (width: 0, negate_p: true, precision: lnprec);
8847
8848	/* If the compiler thinks this is used uninitialized below, it's
8849	because it can't realize that parts can only be 2 when
8850	comparing with constants if l_split_load is also true. This
8851	just silences the warning. */
8852	rnbitpos = 0;
8853	}
8854
8855	/* Likewise, if the right sides are not constant, align them for the combined
8856	compare. Also, disallow this optimization if a size, signedness or
8857	storage order mismatch occurs between the left and right sides. */
8858	else
8859	{
8860	if (ll_bitsize != lr_bitsize || rl_bitsize != rr_bitsize
8861	|| ll_unsignedp != lr_unsignedp || rl_unsignedp != rr_unsignedp
8862	|| ll_reversep != lr_reversep
8863	/* Make sure the two fields on the right
8864	correspond to the left without being swapped. */
8865	|| ll_bitpos - rl_bitpos != lr_bitpos - rr_bitpos)
8866	return 0;
8867
8868	bool r_split_load;
8869	scalar_int_mode rnmode2;
8870
8871	/* Figure out how to load the bits for the right-hand size of the
8872	combined compare. As in the left-hand size, we may have to split it,
8873	and then we use two separate compares. */
8874	first_bit = MIN (lr_bitpos, rr_bitpos);
8875	end_bit = MAX (lr_bitpos + lr_bitsize, rr_bitpos + rr_bitsize);
8876	HOST_WIDE_INT lr_align = TYPE_ALIGN (TREE_TYPE (lr_inner));
8877	poly_uint64 lr_end_region = 0;
8878	if (TYPE_SIZE (TREE_TYPE (lr_inner))
8879	&& tree_fits_poly_uint64_p (TYPE_SIZE (TREE_TYPE (lr_inner))))
8880	lr_end_region = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (lr_inner)));
8881	if (!get_best_mode (end_bit - first_bit, first_bit, 0, lr_end_region,
8882	lr_align, BITS_PER_WORD, volatilep, &rnmode))
8883	{
8884	/* Consider the possibility of recombining loads if any of the
8885	fields straddles across an alignment boundary, so that either
8886	part can be loaded along with the other field. */
8887	HOST_WIDE_INT boundary = compute_split_boundary_from_align
8888	(align: lr_align, l_bitpos: lr_bitpos, l_bitsize: lr_bitsize, r_bitpos: rr_bitpos, r_bitsize: rr_bitsize);
8889
8890	if (boundary < 0
8891	/* If we're to split both, make sure the split point is
8892	the same. */
8893	|| (l_split_load
8894	&& (boundary - lr_bitpos
8895	!= (lnbitpos + GET_MODE_BITSIZE (mode: lnmode)) - ll_bitpos))
8896	|| !get_best_mode (boundary - first_bit, first_bit,
8897	0, lr_end_region,
8898	lr_align, BITS_PER_WORD, volatilep, &rnmode)
8899	|| !get_best_mode (end_bit - boundary, boundary, 0, lr_end_region,
8900	lr_align, BITS_PER_WORD, volatilep, &rnmode2))
8901	return 0;
8902
8903	r_split_load = true;
8904	parts = 2;
8905	if (lr_bitpos >= boundary)
8906	maybe_separate = first1 = true;
8907	else if (lr_bitpos + lr_bitsize <= boundary)
8908	maybe_separate = true;
8909	}
8910	else
8911	r_split_load = false;
8912
8913	/* Find a type that can hold the entire right-hand operand. */
8914	rnbitsize = GET_MODE_BITSIZE (mode: rnmode);
8915	rnbitpos = first_bit & ~ (rnbitsize - 1);
8916	if (r_split_load)
8917	rnbitsize += GET_MODE_BITSIZE (mode: rnmode2);
8918	rntype = build_nonstandard_integer_type (rnbitsize, 1);
8919	if (!rntype)
8920	return 0;
8921	rnprec = TYPE_PRECISION (rntype);
8922	xlr_bitpos = lr_bitpos - rnbitpos, xrr_bitpos = rr_bitpos - rnbitpos;
8923
8924	/* Adjust for reversed endianness. */
8925	if (lr_reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
8926	{
8927	xlr_bitpos = rnbitsize - xlr_bitpos - lr_bitsize;
8928	xrr_bitpos = rnbitsize - xrr_bitpos - rr_bitsize;
8929	}
8930
8931	/* Adjust the masks to match the combined type, and combine them. */
8932	wide_int lr_mask, rr_mask;
8933	if (lr_and_mask.get_precision ())
8934	lr_mask = wi::lshift (x: wide_int::from (x: lr_and_mask, precision: rnprec, sgn: UNSIGNED),
8935	y: xlr_bitpos);
8936	else
8937	lr_mask = wi::shifted_mask (start: xlr_bitpos, width: lr_bitsize, negate_p: false, precision: rnprec);
8938	if (rr_and_mask.get_precision ())
8939	rr_mask = wi::lshift (x: wide_int::from (x: rr_and_mask, precision: rnprec, sgn: UNSIGNED),
8940	y: xrr_bitpos);
8941	else
8942	rr_mask = wi::shifted_mask (start: xrr_bitpos, width: rr_bitsize, negate_p: false, precision: rnprec);
8943	r_mask = lr_mask | rr_mask;
8944
8945	/* Load the right-hand operand of the combined compare. */
8946	toshift[1][0] = MIN (xlr_bitpos, xrr_bitpos);
8947	shifted[1][0] = 0;
8948
8949	if (!r_split_load)
8950	{
8951	bitpos[1][0] = rnbitpos;
8952	bitsiz[1][0] = rnbitsize;
8953	ld_arg[1][0] = make_bit_field_load (loc: ll_loc[3], inner: lr_inner, orig_inner: lr_arg,
8954	type: rntype, bitsize: rnbitsize, bitpos: rnbitpos,
8955	unsignedp: lr_unsignedp || rr_unsignedp,
8956	reversep: lr_reversep, point: lr_load);
8957	}
8958
8959	/* ... and the second part of the right-hand operand if needed. */
8960	if (parts > 1)
8961	{
8962	if (r_split_load)
8963	{
8964	gimple *point[2];
8965	point[0] = lr_load;
8966	point[1] = rr_load;
8967	build_split_load (ln_arg: ld_arg[1], bitpos: bitpos[1], bitsiz: bitsiz[1], toshift: toshift[1],
8968	shifted: shifted[1], loc: rl_loc[3], inner: lr_inner, orig_inner: lr_arg,
8969	mode: rnmode, mode2: rnmode2, bit_pos: rnbitpos, reversep: lr_reversep, point);
8970	}
8971	else
8972	reuse_split_load (ln_arg: ld_arg[1], bitpos: bitpos[1], bitsiz: bitsiz[1], toshift: toshift[1],
8973	shifted: shifted[1], mask: xmask[1],
8974	boundary: lnbitpos + GET_MODE_BITSIZE (mode: lnmode)
8975	- ll_bitpos + lr_bitpos, reversep: lr_reversep);
8976	}
8977	}
8978
8979	/* Now issue the loads for the left-hand combined operand/s. */
8980	wide_int l_mask = ll_mask | rl_mask;
8981	toshift[0][0] = MIN (xll_bitpos, xrl_bitpos);
8982	shifted[0][0] = 0;
8983
8984	if (!l_split_load)
8985	{
8986	bitpos[0][0] = lnbitpos;
8987	bitsiz[0][0] = lnbitsize;
8988	ld_arg[0][0] = make_bit_field_load (loc: ll_loc[3], inner: ll_inner, orig_inner: ll_arg,
8989	type: lntype, bitsize: lnbitsize, bitpos: lnbitpos,
8990	unsignedp: ll_unsignedp || rl_unsignedp,
8991	reversep: ll_reversep, point: ll_load);
8992	}
8993
8994	if (parts > 1)
8995	{
8996	if (l_split_load)
8997	{
8998	gimple *point[2];
8999	point[0] = ll_load;
9000	point[1] = rl_load;
9001	build_split_load (ln_arg: ld_arg[0], bitpos: bitpos[0], bitsiz: bitsiz[0], toshift: toshift[0],
9002	shifted: shifted[0], loc: rl_loc[3], inner: ll_inner, orig_inner: ll_arg,
9003	mode: lnmode, mode2: lnmode2, bit_pos: lnbitpos, reversep: ll_reversep, point);
9004	}
9005	else
9006	reuse_split_load (ln_arg: ld_arg[0], bitpos: bitpos[0], bitsiz: bitsiz[0], toshift: toshift[0],
9007	shifted: shifted[0], mask: xmask[0],
9008	boundary: rnbitpos + GET_MODE_BITSIZE (mode: rnmode)
9009	- lr_bitpos + ll_bitpos, reversep: ll_reversep);
9010	}
9011
9012	/* Compute the compares. */
9013	for (int i = 0; i < parts; i++)
9014	{
9015	tree op[2] = { ld_arg[0][i], ld_arg[1][i] };
9016	wide_int mask[2] = { l_mask, r_mask };
9017	location_t *locs[2] = { i ? rl_loc : ll_loc, i ? rr_loc : lr_loc };
9018
9019	/* Figure out the masks, and unshare the original operands. */
9020	for (int j = 0; j < 2; j++)
9021	{
9022	unsigned prec = TYPE_PRECISION (TREE_TYPE (op[j]));
9023	op[j] = unshare_expr (op[j]);
9024
9025	/* Mask out the bits belonging to the other part. */
9026	if (xmask[j][i].get_precision ())
9027	mask[j] &= xmask[j][i];
9028
9029	if (shifted[j][i])
9030	{
9031	wide_int shift = wide_int::from (x: shifted[j][i], precision: prec, sgn: UNSIGNED);
9032	mask[j] = wi::lrshift (x: mask[j], y: shift);
9033	}
9034	mask[j] = wide_int::from (x: mask[j], precision: prec, sgn: UNSIGNED);
9035	}
9036
9037	/* Line up the operands for a compare. */
9038	HOST_WIDE_INT shift = (toshift[0][i] - toshift[1][i]);
9039
9040	if (shift)
9041	{
9042	int j;
9043	if (shift > 0)
9044	j = 0;
9045	else
9046	{
9047	j = 1;
9048	shift = -shift;
9049	}
9050
9051	tree shiftsz = bitsize_int (shift);
9052	op[j] = fold_build2_loc (locs[j][1], RSHIFT_EXPR, TREE_TYPE (op[j]),
9053	op[j], shiftsz);
9054	mask[j] = wi::lrshift (x: mask[j], y: shift);
9055	}
9056
9057	/* Convert to the smaller type before masking out unwanted
9058	bits. */
9059	tree type = TREE_TYPE (op[0]);
9060	if (type != TREE_TYPE (op[1]))
9061	{
9062	int j = (TYPE_PRECISION (type)
9063	< TYPE_PRECISION (TREE_TYPE (op[1])));
9064	if (!j)
9065	type = TREE_TYPE (op[1]);
9066	op[j] = fold_convert_loc (locs[j][0], type, op[j]);
9067	mask[j] = wide_int::from (x: mask[j], TYPE_PRECISION (type), sgn: UNSIGNED);
9068	}
9069
9070	/* Apply masks. */
9071	for (int j = 0; j < 2; j++)
9072	if (mask[j] != wi::mask (width: 0, negate_p: true, precision: mask[j].get_precision ()))
9073	op[j] = fold_build2_loc (locs[j][2], BIT_AND_EXPR, type,
9074	op[j], wide_int_to_tree (type, cst: mask[j]));
9075
9076	cmp[i] = fold_build2_loc (i ? rloc : lloc, wanted_code, truth_type,
9077	op[0], op[1]);
9078	}
9079
9080	/* Reorder the compares if needed. */
9081	if (first1)
9082	std::swap (a&: cmp[0], b&: cmp[1]);
9083
9084	/* Prepare to return the resulting compares. Combine two parts if
9085	needed. */
9086	if (parts == 1)
9087	result = cmp[0];
9088	else if (!separatep || !maybe_separate)
9089	{
9090	/* Only fold if any of the cmp is known, otherwise we may lose the
9091	sequence point, and that may prevent further optimizations. */
9092	if (TREE_CODE (cmp[0]) == INTEGER_CST
9093	|| TREE_CODE (cmp[1]) == INTEGER_CST)
9094	result = fold_build2_loc (rloc, orig_code, truth_type, cmp[0], cmp[1]);
9095	else
9096	result = build2_loc (loc: rloc, code: orig_code, type: truth_type, arg0: cmp[0], arg1: cmp[1]);
9097	}
9098	else
9099	{
9100	result = cmp[0];
9101	*separatep = cmp[1];
9102	}
9103
9104	return result;
9105	}
9106
9107	/* Try to simplify the AND of two comparisons, specified by
9108	(OP1A CODE1 OP1B) and (OP2B CODE2 OP2B), respectively.
9109	If this can be simplified to a single expression (without requiring
9110	introducing more SSA variables to hold intermediate values),
9111	return the resulting tree. Otherwise return NULL_TREE.
9112	If the result expression is non-null, it has boolean type. */
9113
9114	tree
9115	maybe_fold_and_comparisons (tree type,
9116	enum tree_code code1, tree op1a, tree op1b,
9117	enum tree_code code2, tree op2a, tree op2b,
9118	basic_block outer_cond_bb)
9119	{
9120	if (tree t = and_comparisons_1 (type, code1, op1a, op1b, code2, op2a, op2b,
9121	outer_cond_bb))
9122	return t;
9123
9124	if (tree t = and_comparisons_1 (type, code1: code2, op1a: op2a, op1b: op2b, code2: code1, op2a: op1a, op2b: op1b,
9125	outer_cond_bb))
9126	return t;
9127
9128	if (tree t = maybe_fold_comparisons_from_match_pd (type, code: BIT_AND_EXPR, code1,
9129	op1a, op1b, code2, op2a,
9130	op2b, outer_cond_bb))
9131	return t;
9132
9133	return NULL_TREE;
9134	}
9135
9136	/* Helper function for or_comparisons_1: try to simplify the OR of the
9137	ssa variable VAR with the comparison specified by (OP2A CODE2 OP2B).
9138	If INVERT is true, invert the value of VAR before doing the OR.
9139	Return NULL_EXPR if we can't simplify this to a single expression. */
9140
9141	static tree
9142	or_var_with_comparison (tree type, tree var, bool invert,
9143	enum tree_code code2, tree op2a, tree op2b,
9144	basic_block outer_cond_bb)
9145	{
9146	tree t;
9147	gimple *stmt = SSA_NAME_DEF_STMT (var);
9148
9149	/* We can only deal with variables whose definitions are assignments. */
9150	if (!is_gimple_assign (gs: stmt))
9151	return NULL_TREE;
9152
9153	/* If we have an inverted comparison, apply DeMorgan's law and rewrite
9154	!var OR (op2a code2 op2b) => !(var AND !(op2a code2 op2b))
9155	Then we only have to consider the simpler non-inverted cases. */
9156	if (invert)
9157	t = and_var_with_comparison_1 (type, stmt,
9158	code2: invert_tree_comparison (code2, false),
9159	op2a, op2b, outer_cond_bb);
9160	else
9161	t = or_var_with_comparison_1 (type, stmt, code2, op2a, op2b,
9162	outer_cond_bb);
9163	return canonicalize_bool (expr: t, invert);
9164	}
9165
9166	/* Try to simplify the OR of the ssa variable defined by the assignment
9167	STMT with the comparison specified by (OP2A CODE2 OP2B).
9168	Return NULL_EXPR if we can't simplify this to a single expression. */
9169
9170	static tree
9171	or_var_with_comparison_1 (tree type, gimple *stmt,
9172	enum tree_code code2, tree op2a, tree op2b,
9173	basic_block outer_cond_bb)
9174	{
9175	tree var = gimple_assign_lhs (gs: stmt);
9176	tree true_test_var = NULL_TREE;
9177	tree false_test_var = NULL_TREE;
9178	enum tree_code innercode = gimple_assign_rhs_code (gs: stmt);
9179
9180	/* Check for identities like (var OR (var != 0)) => true . */
9181	if (TREE_CODE (op2a) == SSA_NAME
9182	&& TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE)
9183	{
9184	if ((code2 == NE_EXPR && integer_zerop (op2b))
9185	|| (code2 == EQ_EXPR && integer_nonzerop (op2b)))
9186	{
9187	true_test_var = op2a;
9188	if (var == true_test_var)
9189	return var;
9190	}
9191	else if ((code2 == EQ_EXPR && integer_zerop (op2b))
9192	|| (code2 == NE_EXPR && integer_nonzerop (op2b)))
9193	{
9194	false_test_var = op2a;
9195	if (var == false_test_var)
9196	return boolean_true_node;
9197	}
9198	}
9199
9200	/* If the definition is a comparison, recurse on it. */
9201	if (TREE_CODE_CLASS (innercode) == tcc_comparison)
9202	{
9203	tree t = or_comparisons_1 (type, code1: innercode,
9204	op1a: gimple_assign_rhs1 (gs: stmt),
9205	op1b: gimple_assign_rhs2 (gs: stmt),
9206	code2, op2a, op2b, outer_cond_bb);
9207	if (t)
9208	return t;
9209	}
9210
9211	/* If the definition is an AND or OR expression, we may be able to
9212	simplify by reassociating. */
9213	if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
9214	&& (innercode == BIT_AND_EXPR || innercode == BIT_IOR_EXPR))
9215	{
9216	tree inner1 = gimple_assign_rhs1 (gs: stmt);
9217	tree inner2 = gimple_assign_rhs2 (gs: stmt);
9218	gimple *s;
9219	tree t;
9220	tree partial = NULL_TREE;
9221	bool is_or = (innercode == BIT_IOR_EXPR);
9222
9223	/* Check for boolean identities that don't require recursive examination
9224	of inner1/inner2:
9225	inner1 OR (inner1 OR inner2) => inner1 OR inner2 => var
9226	inner1 OR (inner1 AND inner2) => inner1
9227	!inner1 OR (inner1 OR inner2) => true
9228	!inner1 OR (inner1 AND inner2) => !inner1 OR inner2
9229	*/
9230	if (inner1 == true_test_var)
9231	return (is_or ? var : inner1);
9232	else if (inner2 == true_test_var)
9233	return (is_or ? var : inner2);
9234	else if (inner1 == false_test_var)
9235	return (is_or
9236	? boolean_true_node
9237	: or_var_with_comparison (type, var: inner2, invert: false, code2, op2a,
9238	op2b, outer_cond_bb));
9239	else if (inner2 == false_test_var)
9240	return (is_or
9241	? boolean_true_node
9242	: or_var_with_comparison (type, var: inner1, invert: false, code2, op2a,
9243	op2b, outer_cond_bb));
9244
9245	/* Next, redistribute/reassociate the OR across the inner tests.
9246	Compute the first partial result, (inner1 OR (op2a code op2b)) */
9247	if (TREE_CODE (inner1) == SSA_NAME
9248	&& is_gimple_assign (gs: s = SSA_NAME_DEF_STMT (inner1))
9249	&& TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
9250	&& (t = maybe_fold_or_comparisons (type, gimple_assign_rhs_code (gs: s),
9251	gimple_assign_rhs1 (gs: s),
9252	gimple_assign_rhs2 (gs: s),
9253	code2, op2a, op2b,
9254	outer_cond_bb)))
9255	{
9256	/* Handle the OR case, where we are reassociating:
9257	(inner1 OR inner2) OR (op2a code2 op2b)
9258	=> (t OR inner2)
9259	If the partial result t is a constant, we win. Otherwise
9260	continue on to try reassociating with the other inner test. */
9261	if (is_or)
9262	{
9263	if (integer_onep (t))
9264	return boolean_true_node;
9265	else if (integer_zerop (t))
9266	return inner2;
9267	}
9268
9269	/* Handle the AND case, where we are redistributing:
9270	(inner1 AND inner2) OR (op2a code2 op2b)
9271	=> (t AND (inner2 OR (op2a code op2b))) */
9272	else if (integer_zerop (t))
9273	return boolean_false_node;
9274
9275	/* Save partial result for later. */
9276	partial = t;
9277	}
9278
9279	/* Compute the second partial result, (inner2 OR (op2a code op2b)) */
9280	if (TREE_CODE (inner2) == SSA_NAME
9281	&& is_gimple_assign (gs: s = SSA_NAME_DEF_STMT (inner2))
9282	&& TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
9283	&& (t = maybe_fold_or_comparisons (type, gimple_assign_rhs_code (gs: s),
9284	gimple_assign_rhs1 (gs: s),
9285	gimple_assign_rhs2 (gs: s),
9286	code2, op2a, op2b,
9287	outer_cond_bb)))
9288	{
9289	/* Handle the OR case, where we are reassociating:
9290	(inner1 OR inner2) OR (op2a code2 op2b)
9291	=> (inner1 OR t)
9292	=> (t OR partial) */
9293	if (is_or)
9294	{
9295	if (integer_zerop (t))
9296	return inner1;
9297	else if (integer_onep (t))
9298	return boolean_true_node;
9299	/* If both are the same, we can apply the identity
9300	(x OR x) == x. */
9301	else if (partial && same_bool_result_p (op1: t, op2: partial))
9302	return t;
9303	}
9304
9305	/* Handle the AND case, where we are redistributing:
9306	(inner1 AND inner2) OR (op2a code2 op2b)
9307	=> (t AND (inner1 OR (op2a code2 op2b)))
9308	=> (t AND partial) */
9309	else
9310	{
9311	if (integer_zerop (t))
9312	return boolean_false_node;
9313	else if (partial)
9314	{
9315	/* We already got a simplification for the other
9316	operand to the redistributed AND expression. The
9317	interesting case is when at least one is true.
9318	Or, if both are the same, we can apply the identity
9319	(x AND x) == x. */
9320	if (integer_onep (partial))
9321	return t;
9322	else if (integer_onep (t))
9323	return partial;
9324	else if (same_bool_result_p (op1: t, op2: partial))
9325	return t;
9326	}
9327	}
9328	}
9329	}
9330	return NULL_TREE;
9331	}
9332
9333	/* Try to simplify the OR of two comparisons defined by
9334	(OP1A CODE1 OP1B) and (OP2A CODE2 OP2B), respectively.
9335	If this can be done without constructing an intermediate value,
9336	return the resulting tree; otherwise NULL_TREE is returned.
9337	This function is deliberately asymmetric as it recurses on SSA_DEFs
9338	in the first comparison but not the second. */
9339
9340	static tree
9341	or_comparisons_1 (tree type, enum tree_code code1, tree op1a, tree op1b,
9342	enum tree_code code2, tree op2a, tree op2b,
9343	basic_block outer_cond_bb)
9344	{
9345	tree truth_type = truth_type_for (TREE_TYPE (op1a));
9346
9347	/* First check for ((x CODE1 y) OR (x CODE2 y)). */
9348	if (operand_equal_p (op1a, op2a, flags: 0)
9349	&& operand_equal_p (op1b, op2b, flags: 0))
9350	{
9351	/* Result will be either NULL_TREE, or a combined comparison. */
9352	tree t = combine_comparisons (UNKNOWN_LOCATION,
9353	TRUTH_ORIF_EXPR, code1, code2,
9354	truth_type, op1a, op1b);
9355	if (t)
9356	return t;
9357	}
9358
9359	/* Likewise the swapped case of the above. */
9360	if (operand_equal_p (op1a, op2b, flags: 0)
9361	&& operand_equal_p (op1b, op2a, flags: 0))
9362	{
9363	/* Result will be either NULL_TREE, or a combined comparison. */
9364	tree t = combine_comparisons (UNKNOWN_LOCATION,
9365	TRUTH_ORIF_EXPR, code1,
9366	swap_tree_comparison (code2),
9367	truth_type, op1a, op1b);
9368	if (t)
9369	return t;
9370	}
9371
9372	/* Perhaps the first comparison is (NAME != 0) or (NAME == 1) where
9373	NAME's definition is a truth value. See if there are any simplifications
9374	that can be done against the NAME's definition. */
9375	if (TREE_CODE (op1a) == SSA_NAME
9376	&& (code1 == NE_EXPR || code1 == EQ_EXPR)
9377	&& (integer_zerop (op1b) || integer_onep (op1b)))
9378	{
9379	bool invert = ((code1 == EQ_EXPR && integer_zerop (op1b))
9380	|| (code1 == NE_EXPR && integer_onep (op1b)));
9381	gimple *stmt = SSA_NAME_DEF_STMT (op1a);
9382	switch (gimple_code (g: stmt))
9383	{
9384	case GIMPLE_ASSIGN:
9385	/* Try to simplify by copy-propagating the definition. */
9386	return or_var_with_comparison (type, var: op1a, invert, code2, op2a,
9387	op2b, outer_cond_bb);
9388
9389	case GIMPLE_PHI:
9390	/* If every argument to the PHI produces the same result when
9391	ORed with the second comparison, we win.
9392	Do not do this unless the type is bool since we need a bool
9393	result here anyway. */
9394	if (TREE_CODE (TREE_TYPE (op1a)) == BOOLEAN_TYPE)
9395	{
9396	tree result = NULL_TREE;
9397	unsigned i;
9398	for (i = 0; i < gimple_phi_num_args (gs: stmt); i++)
9399	{
9400	tree arg = gimple_phi_arg_def (gs: stmt, index: i);
9401
9402	/* If this PHI has itself as an argument, ignore it.
9403	If all the other args produce the same result,
9404	we're still OK. */
9405	if (arg == gimple_phi_result (gs: stmt))
9406	continue;
9407	else if (TREE_CODE (arg) == INTEGER_CST)
9408	{
9409	if (invert ? integer_zerop (arg) : integer_nonzerop (arg))
9410	{
9411	if (!result)
9412	result = boolean_true_node;
9413	else if (!integer_onep (result))
9414	return NULL_TREE;
9415	}
9416	else if (!result)
9417	result = fold_build2 (code2, boolean_type_node,
9418	op2a, op2b);
9419	else if (!same_bool_comparison_p (expr: result,
9420	code: code2, op1: op2a, op2: op2b))
9421	return NULL_TREE;
9422	}
9423	else if (TREE_CODE (arg) == SSA_NAME
9424	&& !SSA_NAME_IS_DEFAULT_DEF (arg))
9425	{
9426	tree temp;
9427	gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
9428	/* In simple cases we can look through PHI nodes,
9429	but we have to be careful with loops.
9430	See PR49073. */
9431	if (! dom_info_available_p (CDI_DOMINATORS)
9432	|| gimple_bb (g: def_stmt) == gimple_bb (g: stmt)
9433	|| dominated_by_p (CDI_DOMINATORS,
9434	gimple_bb (g: def_stmt),
9435	gimple_bb (g: stmt)))
9436	return NULL_TREE;
9437	temp = or_var_with_comparison (type, var: arg, invert, code2,
9438	op2a, op2b, outer_cond_bb);
9439	if (!temp)
9440	return NULL_TREE;
9441	else if (!result)
9442	result = temp;
9443	else if (!same_bool_result_p (op1: result, op2: temp))
9444	return NULL_TREE;
9445	}
9446	else
9447	return NULL_TREE;
9448	}
9449	return result;
9450	}
9451
9452	default:
9453	break;
9454	}
9455	}
9456	return NULL_TREE;
9457	}
9458
9459	/* Try to simplify the OR of two comparisons, specified by
9460	(OP1A CODE1 OP1B) and (OP2B CODE2 OP2B), respectively.
9461	If this can be simplified to a single expression (without requiring
9462	introducing more SSA variables to hold intermediate values),
9463	return the resulting tree. Otherwise return NULL_TREE.
9464	If the result expression is non-null, it has boolean type. */
9465
9466	tree
9467	maybe_fold_or_comparisons (tree type,
9468	enum tree_code code1, tree op1a, tree op1b,
9469	enum tree_code code2, tree op2a, tree op2b,
9470	basic_block outer_cond_bb)
9471	{
9472	if (tree t = or_comparisons_1 (type, code1, op1a, op1b, code2, op2a, op2b,
9473	outer_cond_bb))
9474	return t;
9475
9476	if (tree t = or_comparisons_1 (type, code1: code2, op1a: op2a, op1b: op2b, code2: code1, op2a: op1a, op2b: op1b,
9477	outer_cond_bb))
9478	return t;
9479
9480	if (tree t = maybe_fold_comparisons_from_match_pd (type, code: BIT_IOR_EXPR, code1,
9481	op1a, op1b, code2, op2a,
9482	op2b, outer_cond_bb))
9483	return t;
9484
9485	return NULL_TREE;
9486	}
9487
9488	/* Fold STMT to a constant using VALUEIZE to valueize SSA names.
9489
9490	Either NULL_TREE, a simplified but non-constant or a constant
9491	is returned.
9492
9493	??? This should go into a gimple-fold-inline.h file to be eventually
9494	privatized with the single valueize function used in the various TUs
9495	to avoid the indirect function call overhead. */
9496
9497	tree
9498	gimple_fold_stmt_to_constant_1 (gimple *stmt, tree (*valueize) (tree),
9499	tree (*gvalueize) (tree))
9500	{
9501	gimple_match_op res_op;
9502	/* ??? The SSA propagators do not correctly deal with following SSA use-def
9503	edges if there are intermediate VARYING defs. For this reason
9504	do not follow SSA edges here even though SCCVN can technically
9505	just deal fine with that. */
9506	if (gimple_simplify (stmt, &res_op, NULL, gvalueize, valueize))
9507	{
9508	tree res = NULL_TREE;
9509	if (gimple_simplified_result_is_gimple_val (op: &res_op))
9510	res = res_op.ops[0];
9511	else if (mprts_hook)
9512	res = mprts_hook (&res_op);
9513	if (res)
9514	{
9515	if (dump_file && dump_flags & TDF_DETAILS)
9516	{
9517	fprintf (stream: dump_file, format: "Match-and-simplified ");
9518	print_gimple_expr (dump_file, stmt, 0, TDF_SLIM);
9519	fprintf (stream: dump_file, format: " to ");
9520	print_generic_expr (dump_file, res);
9521	fprintf (stream: dump_file, format: "\n");
9522	}
9523	return res;
9524	}
9525	}
9526
9527	location_t loc = gimple_location (g: stmt);
9528	switch (gimple_code (g: stmt))
9529	{
9530	case GIMPLE_ASSIGN:
9531	{
9532	enum tree_code subcode = gimple_assign_rhs_code (gs: stmt);
9533
9534	switch (get_gimple_rhs_class (code: subcode))
9535	{
9536	case GIMPLE_SINGLE_RHS:
9537	{
9538	tree rhs = gimple_assign_rhs1 (gs: stmt);
9539	enum tree_code_class kind = TREE_CODE_CLASS (subcode);
9540
9541	if (TREE_CODE (rhs) == SSA_NAME)
9542	{
9543	/* If the RHS is an SSA_NAME, return its known constant value,
9544	if any. */
9545	return (*valueize) (rhs);
9546	}
9547	/* Handle propagating invariant addresses into address
9548	operations. */
9549	else if (TREE_CODE (rhs) == ADDR_EXPR
9550	&& !is_gimple_min_invariant (rhs))
9551	{
9552	poly_int64 offset = 0;
9553	tree base;
9554	base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (rhs, 0),
9555	&offset,
9556	valueize);
9557	if (base
9558	&& (CONSTANT_CLASS_P (base)
9559	|| decl_address_invariant_p (base)))
9560	return build_invariant_address (TREE_TYPE (rhs),
9561	base, offset);
9562	}
9563	else if (TREE_CODE (rhs) == CONSTRUCTOR
9564	&& TREE_CODE (TREE_TYPE (rhs)) == VECTOR_TYPE
9565	&& known_eq (CONSTRUCTOR_NELTS (rhs),
9566	TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs))))
9567	{
9568	unsigned i, nelts;
9569	tree val;
9570
9571	nelts = CONSTRUCTOR_NELTS (rhs);
9572	tree_vector_builder vec (TREE_TYPE (rhs), nelts, 1);
9573	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), i, val)
9574	{
9575	val = (*valueize) (val);
9576	if (TREE_CODE (val) == INTEGER_CST
9577	|| TREE_CODE (val) == REAL_CST
9578	|| TREE_CODE (val) == FIXED_CST)
9579	vec.quick_push (obj: val);
9580	else
9581	return NULL_TREE;
9582	}
9583
9584	return vec.build ();
9585	}
9586	if (subcode == OBJ_TYPE_REF)
9587	{
9588	tree val = (*valueize) (OBJ_TYPE_REF_EXPR (rhs));
9589	/* If callee is constant, we can fold away the wrapper. */
9590	if (is_gimple_min_invariant (val))
9591	return val;
9592	}
9593
9594	if (kind == tcc_reference)
9595	{
9596	if ((TREE_CODE (rhs) == VIEW_CONVERT_EXPR
9597	|| TREE_CODE (rhs) == REALPART_EXPR
9598	|| TREE_CODE (rhs) == IMAGPART_EXPR)
9599	&& TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME)
9600	{
9601	tree val = (*valueize) (TREE_OPERAND (rhs, 0));
9602	return fold_unary_loc (EXPR_LOCATION (rhs),
9603	TREE_CODE (rhs),
9604	TREE_TYPE (rhs), val);
9605	}
9606	else if (TREE_CODE (rhs) == BIT_FIELD_REF
9607	&& TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME)
9608	{
9609	tree val = (*valueize) (TREE_OPERAND (rhs, 0));
9610	return fold_ternary_loc (EXPR_LOCATION (rhs),
9611	TREE_CODE (rhs),
9612	TREE_TYPE (rhs), val,
9613	TREE_OPERAND (rhs, 1),
9614	TREE_OPERAND (rhs, 2));
9615	}
9616	else if (TREE_CODE (rhs) == MEM_REF
9617	&& TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME)
9618	{
9619	tree val = (*valueize) (TREE_OPERAND (rhs, 0));
9620	if (TREE_CODE (val) == ADDR_EXPR
9621	&& is_gimple_min_invariant (val))
9622	{
9623	tree tem = fold_build2 (MEM_REF, TREE_TYPE (rhs),
9624	unshare_expr (val),
9625	TREE_OPERAND (rhs, 1));
9626	if (tem)
9627	rhs = tem;
9628	}
9629	}
9630	return fold_const_aggregate_ref_1 (rhs, valueize);
9631	}
9632	else if (kind == tcc_declaration)
9633	return get_symbol_constant_value (sym: rhs);
9634	return rhs;
9635	}
9636
9637	case GIMPLE_UNARY_RHS:
9638	return NULL_TREE;
9639
9640	case GIMPLE_BINARY_RHS:
9641	/* Translate &x + CST into an invariant form suitable for
9642	further propagation. */
9643	if (subcode == POINTER_PLUS_EXPR)
9644	{
9645	tree op0 = (*valueize) (gimple_assign_rhs1 (gs: stmt));
9646	tree op1 = (*valueize) (gimple_assign_rhs2 (gs: stmt));
9647	if (TREE_CODE (op0) == ADDR_EXPR
9648	&& TREE_CODE (op1) == INTEGER_CST)
9649	{
9650	tree off = fold_convert (ptr_type_node, op1);
9651	return build1_loc
9652	(loc, code: ADDR_EXPR, TREE_TYPE (op0),
9653	fold_build2 (MEM_REF,
9654	TREE_TYPE (TREE_TYPE (op0)),
9655	unshare_expr (op0), off));
9656	}
9657	}
9658	/* Canonicalize bool != 0 and bool == 0 appearing after
9659	valueization. While gimple_simplify handles this
9660	it can get confused by the ~X == 1 -> X == 0 transform
9661	which we cant reduce to a SSA name or a constant
9662	(and we have no way to tell gimple_simplify to not
9663	consider those transforms in the first place). */
9664	else if (subcode == EQ_EXPR
9665	|| subcode == NE_EXPR)
9666	{
9667	tree lhs = gimple_assign_lhs (gs: stmt);
9668	tree op0 = gimple_assign_rhs1 (gs: stmt);
9669	if (useless_type_conversion_p (TREE_TYPE (lhs),
9670	TREE_TYPE (op0)))
9671	{
9672	tree op1 = (*valueize) (gimple_assign_rhs2 (gs: stmt));
9673	op0 = (*valueize) (op0);
9674	if (TREE_CODE (op0) == INTEGER_CST)
9675	std::swap (a&: op0, b&: op1);
9676	if (TREE_CODE (op1) == INTEGER_CST
9677	&& ((subcode == NE_EXPR && integer_zerop (op1))
9678	|| (subcode == EQ_EXPR && integer_onep (op1))))
9679	return op0;
9680	}
9681	}
9682	return NULL_TREE;
9683
9684	case GIMPLE_TERNARY_RHS:
9685	{
9686	/* Handle ternary operators that can appear in GIMPLE form. */
9687	tree op0 = (*valueize) (gimple_assign_rhs1 (gs: stmt));
9688	tree op1 = (*valueize) (gimple_assign_rhs2 (gs: stmt));
9689	tree op2 = (*valueize) (gimple_assign_rhs3 (gs: stmt));
9690	return fold_ternary_loc (loc, subcode,
9691	TREE_TYPE (gimple_assign_lhs (stmt)),
9692	op0, op1, op2);
9693	}
9694
9695	default:
9696	gcc_unreachable ();
9697	}
9698	}
9699
9700	case GIMPLE_CALL:
9701	{
9702	tree fn;
9703	gcall *call_stmt = as_a <gcall *> (p: stmt);
9704
9705	if (gimple_call_internal_p (gs: stmt))
9706	{
9707	enum tree_code subcode = ERROR_MARK;
9708	switch (gimple_call_internal_fn (gs: stmt))
9709	{
9710	case IFN_UBSAN_CHECK_ADD:
9711	subcode = PLUS_EXPR;
9712	break;
9713	case IFN_UBSAN_CHECK_SUB:
9714	subcode = MINUS_EXPR;
9715	break;
9716	case IFN_UBSAN_CHECK_MUL:
9717	subcode = MULT_EXPR;
9718	break;
9719	case IFN_BUILTIN_EXPECT:
9720	{
9721	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
9722	tree op0 = (*valueize) (arg0);
9723	if (TREE_CODE (op0) == INTEGER_CST)
9724	return op0;
9725	return NULL_TREE;
9726	}
9727	default:
9728	return NULL_TREE;
9729	}
9730	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
9731	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
9732	tree op0 = (*valueize) (arg0);
9733	tree op1 = (*valueize) (arg1);
9734
9735	if (TREE_CODE (op0) != INTEGER_CST
9736	|| TREE_CODE (op1) != INTEGER_CST)
9737	{
9738	switch (subcode)
9739	{
9740	case MULT_EXPR:
9741	/* x * 0 = 0 * x = 0 without overflow. */
9742	if (integer_zerop (op0) || integer_zerop (op1))
9743	return build_zero_cst (TREE_TYPE (arg0));
9744	break;
9745	case MINUS_EXPR:
9746	/* y - y = 0 without overflow. */
9747	if (operand_equal_p (op0, op1, flags: 0))
9748	return build_zero_cst (TREE_TYPE (arg0));
9749	break;
9750	default:
9751	break;
9752	}
9753	}
9754	tree res
9755	= fold_binary_loc (loc, subcode, TREE_TYPE (arg0), op0, op1);
9756	if (res
9757	&& TREE_CODE (res) == INTEGER_CST
9758	&& !TREE_OVERFLOW (res))
9759	return res;
9760	return NULL_TREE;
9761	}
9762
9763	fn = (*valueize) (gimple_call_fn (gs: stmt));
9764	if (TREE_CODE (fn) == ADDR_EXPR
9765	&& TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL
9766	&& fndecl_built_in_p (TREE_OPERAND (fn, 0))
9767	&& gimple_builtin_call_types_compatible_p (stmt,
9768	TREE_OPERAND (fn, 0)))
9769	{
9770	tree *args = XALLOCAVEC (tree, gimple_call_num_args (stmt));
9771	tree retval;
9772	unsigned i;
9773	for (i = 0; i < gimple_call_num_args (gs: stmt); ++i)
9774	args[i] = (*valueize) (gimple_call_arg (gs: stmt, index: i));
9775	retval = fold_builtin_call_array (loc,
9776	gimple_call_return_type (gs: call_stmt),
9777	fn, gimple_call_num_args (gs: stmt), args);
9778	if (retval)
9779	{
9780	/* fold_call_expr wraps the result inside a NOP_EXPR. */
9781	STRIP_NOPS (retval);
9782	retval = fold_convert (gimple_call_return_type (call_stmt),
9783	retval);
9784	}
9785	return retval;
9786	}
9787	return NULL_TREE;
9788	}
9789
9790	default:
9791	return NULL_TREE;
9792	}
9793	}
9794
9795	/* Fold STMT to a constant using VALUEIZE to valueize SSA names.
9796	Returns NULL_TREE if folding to a constant is not possible, otherwise
9797	returns a constant according to is_gimple_min_invariant. */
9798
9799	tree
9800	gimple_fold_stmt_to_constant (gimple *stmt, tree (*valueize) (tree))
9801	{
9802	tree res = gimple_fold_stmt_to_constant_1 (stmt, valueize);
9803	if (res && is_gimple_min_invariant (res))
9804	return res;
9805	return NULL_TREE;
9806	}
9807
9808
9809	/* The following set of functions are supposed to fold references using
9810	their constant initializers. */
9811
9812	/* See if we can find constructor defining value of BASE.
9813	When we know the consructor with constant offset (such as
9814	base is array[40] and we do know constructor of array), then
9815	BIT_OFFSET is adjusted accordingly.
9816
9817	As a special case, return error_mark_node when constructor
9818	is not explicitly available, but it is known to be zero
9819	such as 'static const int a;'. */
9820	static tree
9821	get_base_constructor (tree base, poly_int64 *bit_offset,
9822	tree (*valueize)(tree))
9823	{
9824	poly_int64 bit_offset2, size, max_size;
9825	bool reverse;
9826
9827	if (TREE_CODE (base) == MEM_REF)
9828	{
9829	poly_offset_int boff = *bit_offset + mem_ref_offset (base) * BITS_PER_UNIT;
9830	if (!boff.to_shwi (r: bit_offset))
9831	return NULL_TREE;
9832
9833	if (valueize
9834	&& TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME)
9835	base = valueize (TREE_OPERAND (base, 0));
9836	if (!base || TREE_CODE (base) != ADDR_EXPR)
9837	return NULL_TREE;
9838	base = TREE_OPERAND (base, 0);
9839	}
9840	else if (valueize
9841	&& TREE_CODE (base) == SSA_NAME)
9842	base = valueize (base);
9843
9844	/* Get a CONSTRUCTOR. If BASE is a VAR_DECL, get its
9845	DECL_INITIAL. If BASE is a nested reference into another
9846	ARRAY_REF or COMPONENT_REF, make a recursive call to resolve
9847	the inner reference. */
9848	switch (TREE_CODE (base))
9849	{
9850	case VAR_DECL:
9851	case CONST_DECL:
9852	{
9853	tree init = ctor_for_folding (base);
9854
9855	/* Our semantic is exact opposite of ctor_for_folding;
9856	NULL means unknown, while error_mark_node is 0. */
9857	if (init == error_mark_node)
9858	return NULL_TREE;
9859	if (!init)
9860	return error_mark_node;
9861	return init;
9862	}
9863
9864	case VIEW_CONVERT_EXPR:
9865	return get_base_constructor (TREE_OPERAND (base, 0),
9866	bit_offset, valueize);
9867
9868	case ARRAY_REF:
9869	case COMPONENT_REF:
9870	base = get_ref_base_and_extent (base, &bit_offset2, &size, &max_size,
9871	&reverse);
9872	if (!known_size_p (a: max_size) || maybe_ne (a: size, b: max_size))
9873	return NULL_TREE;
9874	*bit_offset += bit_offset2;
9875	return get_base_constructor (base, bit_offset, valueize);
9876
9877	case CONSTRUCTOR:
9878	return base;
9879
9880	default:
9881	if (CONSTANT_CLASS_P (base))
9882	return base;
9883
9884	return NULL_TREE;
9885	}
9886	}
9887
9888	/* CTOR is a CONSTRUCTOR of an array or vector type. Fold a reference of SIZE
9889	bits to the memory at bit OFFSET. If non-null, TYPE is the expected type of
9890	the reference; otherwise the type of the referenced element is used instead.
9891	When SIZE is zero, attempt to fold a reference to the entire element OFFSET
9892	refers to. Increment *SUBOFF by the bit offset of the accessed element. */
9893
9894	static tree
9895	fold_array_ctor_reference (tree type, tree ctor,
9896	unsigned HOST_WIDE_INT offset,
9897	unsigned HOST_WIDE_INT size,
9898	tree from_decl,
9899	unsigned HOST_WIDE_INT *suboff)
9900	{
9901	offset_int low_bound;
9902	offset_int elt_size;
9903	offset_int access_index;
9904	tree domain_type = NULL_TREE;
9905	HOST_WIDE_INT inner_offset;
9906
9907	/* Compute low bound and elt size. */
9908	if (TREE_CODE (TREE_TYPE (ctor)) == ARRAY_TYPE)
9909	domain_type = TYPE_DOMAIN (TREE_TYPE (ctor));
9910	if (domain_type && TYPE_MIN_VALUE (domain_type))
9911	{
9912	/* Static constructors for variably sized objects make no sense. */
9913	if (TREE_CODE (TYPE_MIN_VALUE (domain_type)) != INTEGER_CST)
9914	return NULL_TREE;
9915	low_bound = wi::to_offset (TYPE_MIN_VALUE (domain_type));
9916	}
9917	else
9918	low_bound = 0;
9919	/* Static constructors for variably sized objects make no sense. */
9920	if (TREE_CODE (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ctor)))) != INTEGER_CST)
9921	return NULL_TREE;
9922	elt_size = wi::to_offset (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ctor))));
9923
9924	/* When TYPE is non-null, verify that it specifies a constant-sized
9925	access of a multiple of the array element size. Avoid division
9926	by zero below when ELT_SIZE is zero, such as with the result of
9927	an initializer for a zero-length array or an empty struct. */
9928	if (elt_size == 0
9929	|| (type
9930	&& (!TYPE_SIZE_UNIT (type)
9931	|| TREE_CODE (TYPE_SIZE_UNIT (type)) != INTEGER_CST)))
9932	return NULL_TREE;
9933
9934	/* Compute the array index we look for. */
9935	access_index = wi::udiv_trunc (x: offset_int (offset / BITS_PER_UNIT),
9936	y: elt_size);
9937	access_index += low_bound;
9938
9939	/* And offset within the access. */
9940	inner_offset = offset % (elt_size.to_uhwi () * BITS_PER_UNIT);
9941
9942	unsigned HOST_WIDE_INT elt_sz = elt_size.to_uhwi ();
9943	if (size > elt_sz * BITS_PER_UNIT)
9944	{
9945	/* native_encode_expr constraints. */
9946	if (size > MAX_BITSIZE_MODE_ANY_MODE
9947	|| size % BITS_PER_UNIT != 0
9948	|| inner_offset % BITS_PER_UNIT != 0
9949	|| elt_sz > MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT)
9950	return NULL_TREE;
9951
9952	unsigned ctor_idx;
9953	tree val = get_array_ctor_element_at_index (ctor, access_index,
9954	&ctor_idx);
9955	if (!val && ctor_idx >= CONSTRUCTOR_NELTS (ctor))
9956	return build_zero_cst (type);
9957
9958	/* native-encode adjacent ctor elements. */
9959	unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
9960	unsigned bufoff = 0;
9961	offset_int index = 0;
9962	offset_int max_index = access_index;
9963	constructor_elt *elt = CONSTRUCTOR_ELT (ctor, ctor_idx);
9964	if (!val)
9965	val = build_zero_cst (TREE_TYPE (TREE_TYPE (ctor)));
9966	else if (!CONSTANT_CLASS_P (val))
9967	return NULL_TREE;
9968	if (!elt->index)
9969	;
9970	else if (TREE_CODE (elt->index) == RANGE_EXPR)
9971	{
9972	index = wi::to_offset (TREE_OPERAND (elt->index, 0));
9973	max_index = wi::to_offset (TREE_OPERAND (elt->index, 1));
9974	}
9975	else
9976	index = max_index = wi::to_offset (t: elt->index);
9977	index = wi::umax (x: index, y: access_index);
9978	do
9979	{
9980	if (bufoff + elt_sz > sizeof (buf))
9981	elt_sz = sizeof (buf) - bufoff;
9982	int len;
9983	if (TREE_CODE (val) == RAW_DATA_CST)
9984	{
9985	gcc_assert (inner_offset == 0);
9986	if (!elt->index || TREE_CODE (elt->index) != INTEGER_CST)
9987	return NULL_TREE;
9988	inner_offset = (access_index
9989	- wi::to_offset (t: elt->index)).to_uhwi ();
9990	len = MIN (sizeof (buf) - bufoff,
9991	(unsigned) (RAW_DATA_LENGTH (val) - inner_offset));
9992	memcpy (dest: buf + bufoff, RAW_DATA_POINTER (val) + inner_offset,
9993	n: len);
9994	access_index += len - 1;
9995	}
9996	else
9997	{
9998	len = native_encode_expr (val, buf + bufoff, elt_sz,
9999	off: inner_offset / BITS_PER_UNIT);
10000	if (len != (int) elt_sz - inner_offset / BITS_PER_UNIT)
10001	return NULL_TREE;
10002	}
10003	inner_offset = 0;
10004	bufoff += len;
10005
10006	access_index += 1;
10007	if (wi::cmpu (x: access_index, y: index) == 0)
10008	val = elt->value;
10009	else if (wi::cmpu (x: access_index, y: max_index) > 0)
10010	{
10011	ctor_idx++;
10012	if (ctor_idx >= CONSTRUCTOR_NELTS (ctor))
10013	{
10014	val = build_zero_cst (TREE_TYPE (TREE_TYPE (ctor)));
10015	++max_index;
10016	}
10017	else
10018	{
10019	elt = CONSTRUCTOR_ELT (ctor, ctor_idx);
10020	index = 0;
10021	max_index = access_index;
10022	if (!elt->index)
10023	;
10024	else if (TREE_CODE (elt->index) == RANGE_EXPR)
10025	{
10026	index = wi::to_offset (TREE_OPERAND (elt->index, 0));
10027	max_index = wi::to_offset (TREE_OPERAND (elt->index, 1));
10028	}
10029	else
10030	index = max_index = wi::to_offset (t: elt->index);
10031	index = wi::umax (x: index, y: access_index);
10032	if (wi::cmpu (x: access_index, y: index) == 0)
10033	val = elt->value;
10034	else
10035	val = build_zero_cst (TREE_TYPE (TREE_TYPE (ctor)));
10036	}
10037	}
10038	}
10039	while (bufoff < size / BITS_PER_UNIT);
10040	*suboff += size;
10041	return native_interpret_expr (type, buf, size / BITS_PER_UNIT);
10042	}
10043
10044	unsigned ctor_idx;
10045	if (tree val = get_array_ctor_element_at_index (ctor, access_index,
10046	&ctor_idx))
10047	{
10048	if (TREE_CODE (val) == RAW_DATA_CST)
10049	{
10050	if (size != BITS_PER_UNIT || elt_sz != 1 || inner_offset != 0)
10051	return NULL_TREE;
10052	constructor_elt *elt = CONSTRUCTOR_ELT (ctor, ctor_idx);
10053	if (elt->index == NULL_TREE || TREE_CODE (elt->index) != INTEGER_CST)
10054	return NULL_TREE;
10055	unsigned o = (access_index - wi::to_offset (t: elt->index)).to_uhwi ();
10056	val = build_int_cst (TREE_TYPE (val), RAW_DATA_UCHAR_ELT (val, o));
10057	}
10058	if (!size && TREE_CODE (val) != CONSTRUCTOR)
10059	{
10060	/* For the final reference to the entire accessed element
10061	(SIZE is zero), reset INNER_OFFSET, disegard TYPE (which
10062	may be null) in favor of the type of the element, and set
10063	SIZE to the size of the accessed element. */
10064	inner_offset = 0;
10065	type = TREE_TYPE (val);
10066	size = elt_sz * BITS_PER_UNIT;
10067	}
10068	else if (size && access_index < CONSTRUCTOR_NELTS (ctor) - 1
10069	&& TREE_CODE (val) == CONSTRUCTOR
10070	&& (elt_sz * BITS_PER_UNIT - inner_offset) < size)
10071	/* If this isn't the last element in the CTOR and a CTOR itself
10072	and it does not cover the whole object we are requesting give up
10073	since we're not set up for combining from multiple CTORs. */
10074	return NULL_TREE;
10075
10076	*suboff += access_index.to_uhwi () * elt_sz * BITS_PER_UNIT;
10077	return fold_ctor_reference (type, val, inner_offset, size, from_decl,
10078	suboff);
10079	}
10080
10081	/* Memory not explicitly mentioned in constructor is 0 (or
10082	the reference is out of range). */
10083	return type ? build_zero_cst (type) : NULL_TREE;
10084	}
10085
10086	/* CTOR is a CONSTRUCTOR of a record or union type. Fold a reference of SIZE
10087	bits to the memory at bit OFFSET. If non-null, TYPE is the expected type of
10088	the reference; otherwise the type of the referenced member is used instead.
10089	When SIZE is zero, attempt to fold a reference to the entire member OFFSET
10090	refers to. Increment *SUBOFF by the bit offset of the accessed member. */
10091
10092	static tree
10093	fold_nonarray_ctor_reference (tree type, tree ctor,
10094	unsigned HOST_WIDE_INT offset,
10095	unsigned HOST_WIDE_INT size,
10096	tree from_decl,
10097	unsigned HOST_WIDE_INT *suboff)
10098	{
10099	unsigned HOST_WIDE_INT cnt;
10100	tree cfield, cval;
10101
10102	FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), cnt, cfield, cval)
10103	{
10104	tree byte_offset = DECL_FIELD_OFFSET (cfield);
10105	tree field_offset = DECL_FIELD_BIT_OFFSET (cfield);
10106	tree field_size = DECL_SIZE (cfield);
10107
10108	if (!field_size)
10109	{
10110	/* Determine the size of the flexible array member from
10111	the size of the initializer provided for it. */
10112	field_size = TYPE_SIZE (TREE_TYPE (cval));
10113	}
10114
10115	/* Variable sized objects in static constructors makes no sense,
10116	but field_size can be NULL for flexible array members. */
10117	gcc_assert (TREE_CODE (field_offset) == INTEGER_CST
10118	&& TREE_CODE (byte_offset) == INTEGER_CST
10119	&& (field_size != NULL_TREE
10120	? TREE_CODE (field_size) == INTEGER_CST
10121	: TREE_CODE (TREE_TYPE (cfield)) == ARRAY_TYPE));
10122
10123	/* Compute bit offset of the field. */
10124	offset_int bitoffset
10125	= (wi::to_offset (t: field_offset)
10126	+ (wi::to_offset (t: byte_offset) << LOG2_BITS_PER_UNIT));
10127	/* Compute bit offset where the field ends. */
10128	offset_int bitoffset_end;
10129	if (field_size != NULL_TREE)
10130	bitoffset_end = bitoffset + wi::to_offset (t: field_size);
10131	else
10132	bitoffset_end = 0;
10133
10134	/* Compute the bit offset of the end of the desired access.
10135	As a special case, if the size of the desired access is
10136	zero, assume the access is to the entire field (and let
10137	the caller make any necessary adjustments by storing
10138	the actual bounds of the field in FIELDBOUNDS). */
10139	offset_int access_end = offset_int (offset);
10140	if (size)
10141	access_end += size;
10142	else
10143	access_end = bitoffset_end;
10144
10145	/* Is there any overlap between the desired access at
10146	[OFFSET, OFFSET+SIZE) and the offset of the field within
10147	the object at [BITOFFSET, BITOFFSET_END)? */
10148	if (wi::cmps (x: access_end, y: bitoffset) > 0
10149	&& (field_size == NULL_TREE
10150	|| wi::lts_p (x: offset, y: bitoffset_end)))
10151	{
10152	*suboff += bitoffset.to_uhwi ();
10153
10154	if (!size && TREE_CODE (cval) != CONSTRUCTOR)
10155	{
10156	/* For the final reference to the entire accessed member
10157	(SIZE is zero), reset OFFSET, disegard TYPE (which may
10158	be null) in favor of the type of the member, and set
10159	SIZE to the size of the accessed member. */
10160	offset = bitoffset.to_uhwi ();
10161	type = TREE_TYPE (cval);
10162	size = (bitoffset_end - bitoffset).to_uhwi ();
10163	}
10164
10165	/* We do have overlap. Now see if the field is large enough
10166	to cover the access. Give up for accesses that extend
10167	beyond the end of the object or that span multiple fields. */
10168	if (wi::cmps (x: access_end, y: bitoffset_end) > 0)
10169	return NULL_TREE;
10170	if (offset < bitoffset)
10171	return NULL_TREE;
10172
10173	offset_int inner_offset = offset_int (offset) - bitoffset;
10174
10175	/* Integral bit-fields are left-justified on big-endian targets, so
10176	we must arrange for native_encode_int to start at their MSB. */
10177	if (DECL_BIT_FIELD (cfield) && INTEGRAL_TYPE_P (TREE_TYPE (cfield)))
10178	{
10179	if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
10180	return NULL_TREE;
10181	if (BYTES_BIG_ENDIAN)
10182	{
10183	tree ctype = TREE_TYPE (cfield);
10184	unsigned int encoding_size;
10185	if (TYPE_MODE (ctype) != BLKmode)
10186	encoding_size
10187	= GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (ctype));
10188	else
10189	encoding_size = TREE_INT_CST_LOW (TYPE_SIZE (ctype));
10190	inner_offset += encoding_size - wi::to_offset (t: field_size);
10191	}
10192	}
10193
10194	return fold_ctor_reference (type, cval,
10195	inner_offset.to_uhwi (), size,
10196	from_decl, suboff);
10197	}
10198	}
10199
10200	if (!type)
10201	return NULL_TREE;
10202
10203	return build_zero_cst (type);
10204	}
10205
10206	/* CTOR is a value initializing memory. Fold a reference of TYPE and
10207	bit size POLY_SIZE to the memory at bit POLY_OFFSET. When POLY_SIZE
10208	is zero, attempt to fold a reference to the entire subobject
10209	which OFFSET refers to. This is used when folding accesses to
10210	string members of aggregates. When non-null, set *SUBOFF to
10211	the bit offset of the accessed subobject. */
10212
10213	tree
10214	fold_ctor_reference (tree type, tree ctor, const poly_uint64 &poly_offset,
10215	const poly_uint64 &poly_size, tree from_decl,
10216	unsigned HOST_WIDE_INT *suboff /* = NULL */)
10217	{
10218	tree ret;
10219
10220	/* We found the field with exact match. */
10221	if (type
10222	&& useless_type_conversion_p (type, TREE_TYPE (ctor))
10223	&& known_eq (poly_offset, 0U))
10224	return canonicalize_constructor_val (cval: unshare_expr (ctor), from_decl);
10225
10226	/* The remaining optimizations need a constant size and offset. */
10227	unsigned HOST_WIDE_INT size, offset;
10228	if (!poly_size.is_constant (const_value: &size) || !poly_offset.is_constant (const_value: &offset))
10229	return NULL_TREE;
10230
10231	/* We are at the end of walk, see if we can view convert the
10232	result. */
10233	if (!AGGREGATE_TYPE_P (TREE_TYPE (ctor)) && !offset
10234	/* VIEW_CONVERT_EXPR is defined only for matching sizes. */
10235	&& known_eq (wi::to_poly_widest (TYPE_SIZE (type)), size)
10236	&& known_eq (wi::to_poly_widest (TYPE_SIZE (TREE_TYPE (ctor))), size))
10237	{
10238	ret = canonicalize_constructor_val (cval: unshare_expr (ctor), from_decl);
10239	if (ret)
10240	{
10241	ret = fold_unary (VIEW_CONVERT_EXPR, type, ret);
10242	if (ret)
10243	STRIP_USELESS_TYPE_CONVERSION (ret);
10244	}
10245	return ret;
10246	}
10247
10248	/* For constants and byte-aligned/sized reads, try to go through
10249	native_encode/interpret. */
10250	if (CONSTANT_CLASS_P (ctor)
10251	&& BITS_PER_UNIT == 8
10252	&& offset % BITS_PER_UNIT == 0
10253	&& offset / BITS_PER_UNIT <= INT_MAX
10254	&& size % BITS_PER_UNIT == 0
10255	&& size <= MAX_BITSIZE_MODE_ANY_MODE
10256	&& can_native_interpret_type_p (type))
10257	{
10258	unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
10259	int len = native_encode_expr (ctor, buf, size / BITS_PER_UNIT,
10260	off: offset / BITS_PER_UNIT);
10261	if (len > 0)
10262	return native_interpret_expr (type, buf, len);
10263	}
10264
10265	/* For constructors, try first a recursive local processing, but in any case
10266	this requires the native storage order. */
10267	if (TREE_CODE (ctor) == CONSTRUCTOR
10268	&& !(AGGREGATE_TYPE_P (TREE_TYPE (ctor))
10269	&& TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (ctor))))
10270	{
10271	unsigned HOST_WIDE_INT dummy = 0;
10272	if (!suboff)
10273	suboff = &dummy;
10274
10275	tree ret;
10276	if (TREE_CODE (TREE_TYPE (ctor)) == ARRAY_TYPE
10277	|| TREE_CODE (TREE_TYPE (ctor)) == VECTOR_TYPE)
10278	ret = fold_array_ctor_reference (type, ctor, offset, size,
10279	from_decl, suboff);
10280	else
10281	ret = fold_nonarray_ctor_reference (type, ctor, offset, size,
10282	from_decl, suboff);
10283
10284	/* Otherwise fall back to native_encode_initializer. This may be done
10285	only from the outermost fold_ctor_reference call (because it itself
10286	recurses into CONSTRUCTORs and doesn't update suboff). */
10287	if (ret == NULL_TREE
10288	&& suboff == &dummy
10289	&& BITS_PER_UNIT == 8
10290	&& offset % BITS_PER_UNIT == 0
10291	&& offset / BITS_PER_UNIT <= INT_MAX
10292	&& size % BITS_PER_UNIT == 0
10293	&& size <= MAX_BITSIZE_MODE_ANY_MODE
10294	&& can_native_interpret_type_p (type))
10295	{
10296	unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
10297	int len = native_encode_initializer (ctor, buf, size / BITS_PER_UNIT,
10298	off: offset / BITS_PER_UNIT);
10299	if (len > 0)
10300	return native_interpret_expr (type, buf, len);
10301	}
10302
10303	return ret;
10304	}
10305
10306	return NULL_TREE;
10307	}
10308
10309	/* Return the tree representing the element referenced by T if T is an
10310	ARRAY_REF or COMPONENT_REF into constant aggregates valuezing SSA
10311	names using VALUEIZE. Return NULL_TREE otherwise. */
10312
10313	tree
10314	fold_const_aggregate_ref_1 (tree t, tree (*valueize) (tree))
10315	{
10316	tree ctor, idx, base;
10317	poly_int64 offset, size, max_size;
10318	tree tem;
10319	bool reverse;
10320
10321	if (TREE_THIS_VOLATILE (t))
10322	return NULL_TREE;
10323
10324	if (DECL_P (t))
10325	return get_symbol_constant_value (sym: t);
10326
10327	tem = fold_read_from_constant_string (t);
10328	if (tem)
10329	return tem;
10330
10331	switch (TREE_CODE (t))
10332	{
10333	case ARRAY_REF:
10334	case ARRAY_RANGE_REF:
10335	/* Constant indexes are handled well by get_base_constructor.
10336	Only special case variable offsets.
10337	FIXME: This code can't handle nested references with variable indexes
10338	(they will be handled only by iteration of ccp). Perhaps we can bring
10339	get_ref_base_and_extent here and make it use a valueize callback. */
10340	if (TREE_CODE (TREE_OPERAND (t, 1)) == SSA_NAME
10341	&& valueize
10342	&& (idx = (*valueize) (TREE_OPERAND (t, 1)))
10343	&& poly_int_tree_p (t: idx))
10344	{
10345	tree low_bound, unit_size;
10346
10347	/* If the resulting bit-offset is constant, track it. */
10348	if ((low_bound = array_ref_low_bound (t),
10349	poly_int_tree_p (t: low_bound))
10350	&& (unit_size = array_ref_element_size (t),
10351	tree_fits_uhwi_p (unit_size)))
10352	{
10353	poly_offset_int woffset
10354	= wi::sext (a: wi::to_poly_offset (t: idx)
10355	- wi::to_poly_offset (t: low_bound),
10356	TYPE_PRECISION (sizetype));
10357	woffset *= tree_to_uhwi (unit_size);
10358	woffset *= BITS_PER_UNIT;
10359	if (woffset.to_shwi (r: &offset))
10360	{
10361	base = TREE_OPERAND (t, 0);
10362	ctor = get_base_constructor (base, bit_offset: &offset, valueize);
10363	/* Empty constructor. Always fold to 0. */
10364	if (ctor == error_mark_node)
10365	return build_zero_cst (TREE_TYPE (t));
10366	/* Out of bound array access. Value is undefined,
10367	but don't fold. */
10368	if (maybe_lt (a: offset, b: 0))
10369	return NULL_TREE;
10370	/* We cannot determine ctor. */
10371	if (!ctor)
10372	return NULL_TREE;
10373	return fold_ctor_reference (TREE_TYPE (t), ctor, poly_offset: offset,
10374	poly_size: tree_to_uhwi (unit_size)
10375	* BITS_PER_UNIT,
10376	from_decl: base);
10377	}
10378	}
10379	}
10380	/* Fallthru. */
10381
10382	case COMPONENT_REF:
10383	case BIT_FIELD_REF:
10384	case TARGET_MEM_REF:
10385	case MEM_REF:
10386	base = get_ref_base_and_extent (t, &offset, &size, &max_size, &reverse);
10387	ctor = get_base_constructor (base, bit_offset: &offset, valueize);
10388
10389	/* We cannot determine ctor. */
10390	if (!ctor)
10391	return NULL_TREE;
10392	/* Empty constructor. Always fold to 0. */
10393	if (ctor == error_mark_node)
10394	return build_zero_cst (TREE_TYPE (t));
10395	/* We do not know precise access. */
10396	if (!known_size_p (a: max_size) || maybe_ne (a: max_size, b: size))
10397	return NULL_TREE;
10398	/* Out of bound array access. Value is undefined, but don't fold. */
10399	if (maybe_lt (a: offset, b: 0))
10400	return NULL_TREE;
10401	/* Access with reverse storage order. */
10402	if (reverse)
10403	return NULL_TREE;
10404
10405	tem = fold_ctor_reference (TREE_TYPE (t), ctor, poly_offset: offset, poly_size: size, from_decl: base);
10406	if (tem)
10407	return tem;
10408
10409	/* For bit field reads try to read the representative and
10410	adjust. */
10411	if (TREE_CODE (t) == COMPONENT_REF
10412	&& DECL_BIT_FIELD (TREE_OPERAND (t, 1))
10413	&& DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1)))
10414	{
10415	HOST_WIDE_INT csize, coffset;
10416	tree field = TREE_OPERAND (t, 1);
10417	tree repr = DECL_BIT_FIELD_REPRESENTATIVE (field);
10418	if (INTEGRAL_TYPE_P (TREE_TYPE (repr))
10419	&& size.is_constant (const_value: &csize)
10420	&& offset.is_constant (const_value: &coffset)
10421	&& (coffset % BITS_PER_UNIT != 0
10422	|| csize % BITS_PER_UNIT != 0)
10423	&& BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN)
10424	{
10425	poly_int64 bitoffset;
10426	poly_uint64 field_offset, repr_offset;
10427	if (poly_int_tree_p (DECL_FIELD_OFFSET (field), value: &field_offset)
10428	&& poly_int_tree_p (DECL_FIELD_OFFSET (repr), value: &repr_offset))
10429	bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
10430	else
10431	bitoffset = 0;
10432	bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
10433	- tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
10434	HOST_WIDE_INT bitoff;
10435	int diff = (TYPE_PRECISION (TREE_TYPE (repr))
10436	- TYPE_PRECISION (TREE_TYPE (field)));
10437	if (bitoffset.is_constant (const_value: &bitoff)
10438	&& bitoff >= 0
10439	&& bitoff <= diff)
10440	{
10441	offset -= bitoff;
10442	size = tree_to_uhwi (DECL_SIZE (repr));
10443
10444	tem = fold_ctor_reference (TREE_TYPE (repr), ctor, poly_offset: offset,
10445	poly_size: size, from_decl: base);
10446	if (tem && TREE_CODE (tem) == INTEGER_CST)
10447	{
10448	if (!BYTES_BIG_ENDIAN)
10449	tem = wide_int_to_tree (TREE_TYPE (field),
10450	cst: wi::lrshift (x: wi::to_wide (t: tem),
10451	y: bitoff));
10452	else
10453	tem = wide_int_to_tree (TREE_TYPE (field),
10454	cst: wi::lrshift (x: wi::to_wide (t: tem),
10455	y: diff - bitoff));
10456	return tem;
10457	}
10458	}
10459	}
10460	}
10461	break;
10462
10463	case REALPART_EXPR:
10464	case IMAGPART_EXPR:
10465	{
10466	tree c = fold_const_aggregate_ref_1 (TREE_OPERAND (t, 0), valueize);
10467	if (c && TREE_CODE (c) == COMPLEX_CST)
10468	return fold_build1_loc (EXPR_LOCATION (t),
10469	TREE_CODE (t), TREE_TYPE (t), c);
10470	break;
10471	}
10472
10473	default:
10474	break;
10475	}
10476
10477	return NULL_TREE;
10478	}
10479
10480	tree
10481	fold_const_aggregate_ref (tree t)
10482	{
10483	return fold_const_aggregate_ref_1 (t, NULL);
10484	}
10485
10486	/* Lookup virtual method with index TOKEN in a virtual table V
10487	at OFFSET.
10488	Set CAN_REFER if non-NULL to false if method
10489	is not referable or if the virtual table is ill-formed (such as rewriten
10490	by non-C++ produced symbol). Otherwise just return NULL in that calse. */
10491
10492	tree
10493	gimple_get_virt_method_for_vtable (HOST_WIDE_INT token,
10494	tree v,
10495	unsigned HOST_WIDE_INT offset,
10496	bool *can_refer)
10497	{
10498	tree vtable = v, init, fn;
10499	unsigned HOST_WIDE_INT size;
10500	unsigned HOST_WIDE_INT elt_size, access_index;
10501	tree domain_type;
10502
10503	if (can_refer)
10504	*can_refer = true;
10505
10506	/* First of all double check we have virtual table. */
10507	if (!VAR_P (v) || !DECL_VIRTUAL_P (v))
10508	{
10509	/* Pass down that we lost track of the target. */
10510	if (can_refer)
10511	*can_refer = false;
10512	return NULL_TREE;
10513	}
10514
10515	init = ctor_for_folding (v);
10516
10517	/* The virtual tables should always be born with constructors
10518	and we always should assume that they are avaialble for
10519	folding. At the moment we do not stream them in all cases,
10520	but it should never happen that ctor seem unreachable. */
10521	gcc_assert (init);
10522	if (init == error_mark_node)
10523	{
10524	/* Pass down that we lost track of the target. */
10525	if (can_refer)
10526	*can_refer = false;
10527	return NULL_TREE;
10528	}
10529	gcc_checking_assert (TREE_CODE (TREE_TYPE (v)) == ARRAY_TYPE);
10530	size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (v))));
10531	offset *= BITS_PER_UNIT;
10532	offset += token * size;
10533
10534	/* Lookup the value in the constructor that is assumed to be array.
10535	This is equivalent to
10536	fn = fold_ctor_reference (TREE_TYPE (TREE_TYPE (v)), init,
10537	offset, size, NULL);
10538	but in a constant time. We expect that frontend produced a simple
10539	array without indexed initializers. */
10540
10541	gcc_checking_assert (TREE_CODE (TREE_TYPE (init)) == ARRAY_TYPE);
10542	domain_type = TYPE_DOMAIN (TREE_TYPE (init));
10543	gcc_checking_assert (integer_zerop (TYPE_MIN_VALUE (domain_type)));
10544	elt_size = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (init))));
10545
10546	access_index = offset / BITS_PER_UNIT / elt_size;
10547	gcc_checking_assert (offset % (elt_size * BITS_PER_UNIT) == 0);
10548
10549	/* This code makes an assumption that there are no
10550	indexed fileds produced by C++ FE, so we can directly index the array. */
10551	if (access_index < CONSTRUCTOR_NELTS (init))
10552	{
10553	fn = CONSTRUCTOR_ELT (init, access_index)->value;
10554	gcc_checking_assert (!CONSTRUCTOR_ELT (init, access_index)->index);
10555	STRIP_NOPS (fn);
10556	}
10557	else
10558	fn = NULL;
10559
10560	/* For type inconsistent program we may end up looking up virtual method
10561	in virtual table that does not contain TOKEN entries. We may overrun
10562	the virtual table and pick up a constant or RTTI info pointer.
10563	In any case the call is undefined. */
10564	if (!fn
10565	|| (TREE_CODE (fn) != ADDR_EXPR && TREE_CODE (fn) != FDESC_EXPR)
10566	|| TREE_CODE (TREE_OPERAND (fn, 0)) != FUNCTION_DECL)
10567	fn = builtin_decl_unreachable ();
10568	else
10569	{
10570	fn = TREE_OPERAND (fn, 0);
10571
10572	/* When cgraph node is missing and function is not public, we cannot
10573	devirtualize. This can happen in WHOPR when the actual method
10574	ends up in other partition, because we found devirtualization
10575	possibility too late. */
10576	if (!can_refer_decl_in_current_unit_p (decl: fn, from_decl: vtable))
10577	{
10578	if (can_refer)
10579	{
10580	*can_refer = false;
10581	return fn;
10582	}
10583	return NULL_TREE;
10584	}
10585	}
10586
10587	/* Make sure we create a cgraph node for functions we'll reference.
10588	They can be non-existent if the reference comes from an entry
10589	of an external vtable for example. */
10590	cgraph_node::get_create (fn);
10591
10592	return fn;
10593	}
10594
10595	/* Return a declaration of a function which an OBJ_TYPE_REF references. TOKEN
10596	is integer form of OBJ_TYPE_REF_TOKEN of the reference expression.
10597	KNOWN_BINFO carries the binfo describing the true type of
10598	OBJ_TYPE_REF_OBJECT(REF).
10599	Set CAN_REFER if non-NULL to false if method
10600	is not referable or if the virtual table is ill-formed (such as rewriten
10601	by non-C++ produced symbol). Otherwise just return NULL in that calse. */
10602
10603	tree
10604	gimple_get_virt_method_for_binfo (HOST_WIDE_INT token, tree known_binfo,
10605	bool *can_refer)
10606	{
10607	unsigned HOST_WIDE_INT offset;
10608	tree v;
10609
10610	v = BINFO_VTABLE (known_binfo);
10611	/* If there is no virtual methods table, leave the OBJ_TYPE_REF alone. */
10612	if (!v)
10613	return NULL_TREE;
10614
10615	if (!vtable_pointer_value_to_vtable (v, &v, &offset))
10616	{
10617	if (can_refer)
10618	*can_refer = false;
10619	return NULL_TREE;
10620	}
10621	return gimple_get_virt_method_for_vtable (token, v, offset, can_refer);
10622	}
10623
10624	/* Given a pointer value T, return a simplified version of an
10625	indirection through T, or NULL_TREE if no simplification is
10626	possible. Note that the resulting type may be different from
10627	the type pointed to in the sense that it is still compatible
10628	from the langhooks point of view. */
10629
10630	tree
10631	gimple_fold_indirect_ref (tree t)
10632	{
10633	tree ptype = TREE_TYPE (t), type = TREE_TYPE (ptype);
10634	tree sub = t;
10635	tree subtype;
10636
10637	STRIP_NOPS (sub);
10638	subtype = TREE_TYPE (sub);
10639	if (!POINTER_TYPE_P (subtype)
10640	|| TYPE_REF_CAN_ALIAS_ALL (ptype))
10641	return NULL_TREE;
10642
10643	if (TREE_CODE (sub) == ADDR_EXPR)
10644	{
10645	tree op = TREE_OPERAND (sub, 0);
10646	tree optype = TREE_TYPE (op);
10647	/* *&p => p */
10648	if (useless_type_conversion_p (type, optype))
10649	return op;
10650
10651	/* *(foo *)&fooarray => fooarray[0] */
10652	if (TREE_CODE (optype) == ARRAY_TYPE
10653	&& TREE_CODE (TYPE_SIZE (TREE_TYPE (optype))) == INTEGER_CST
10654	&& useless_type_conversion_p (type, TREE_TYPE (optype)))
10655	{
10656	tree type_domain = TYPE_DOMAIN (optype);
10657	tree min_val = size_zero_node;
10658	if (type_domain && TYPE_MIN_VALUE (type_domain))
10659	min_val = TYPE_MIN_VALUE (type_domain);
10660	if (TREE_CODE (min_val) == INTEGER_CST)
10661	return build4 (ARRAY_REF, type, op, min_val, NULL_TREE, NULL_TREE);
10662	}
10663	/* *(foo *)&complexfoo => __real__ complexfoo */
10664	else if (TREE_CODE (optype) == COMPLEX_TYPE
10665	&& useless_type_conversion_p (type, TREE_TYPE (optype)))
10666	return fold_build1 (REALPART_EXPR, type, op);
10667	/* *(foo *)&vectorfoo => BIT_FIELD_REF<vectorfoo,...> */
10668	else if (TREE_CODE (optype) == VECTOR_TYPE
10669	&& useless_type_conversion_p (type, TREE_TYPE (optype)))
10670	{
10671	tree part_width = TYPE_SIZE (type);
10672	tree index = bitsize_int (0);
10673	return fold_build3 (BIT_FIELD_REF, type, op, part_width, index);
10674	}
10675	}
10676
10677	/* *(p + CST) -> ... */
10678	if (TREE_CODE (sub) == POINTER_PLUS_EXPR
10679	&& TREE_CODE (TREE_OPERAND (sub, 1)) == INTEGER_CST)
10680	{
10681	tree addr = TREE_OPERAND (sub, 0);
10682	tree off = TREE_OPERAND (sub, 1);
10683	tree addrtype;
10684
10685	STRIP_NOPS (addr);
10686	addrtype = TREE_TYPE (addr);
10687
10688	/* ((foo*)&vectorfoo)[1] -> BIT_FIELD_REF<vectorfoo,...> */
10689	if (TREE_CODE (addr) == ADDR_EXPR
10690	&& TREE_CODE (TREE_TYPE (addrtype)) == VECTOR_TYPE
10691	&& useless_type_conversion_p (type, TREE_TYPE (TREE_TYPE (addrtype)))
10692	&& tree_fits_uhwi_p (off))
10693	{
10694	unsigned HOST_WIDE_INT offset = tree_to_uhwi (off);
10695	tree part_width = TYPE_SIZE (type);
10696	unsigned HOST_WIDE_INT part_widthi
10697	= tree_to_shwi (part_width) / BITS_PER_UNIT;
10698	unsigned HOST_WIDE_INT indexi = offset * BITS_PER_UNIT;
10699	tree index = bitsize_int (indexi);
10700	if (known_lt (offset / part_widthi,
10701	TYPE_VECTOR_SUBPARTS (TREE_TYPE (addrtype))))
10702	return fold_build3 (BIT_FIELD_REF, type, TREE_OPERAND (addr, 0),
10703	part_width, index);
10704	}
10705
10706	/* ((foo*)&complexfoo)[1] -> __imag__ complexfoo */
10707	if (TREE_CODE (addr) == ADDR_EXPR
10708	&& TREE_CODE (TREE_TYPE (addrtype)) == COMPLEX_TYPE
10709	&& useless_type_conversion_p (type, TREE_TYPE (TREE_TYPE (addrtype))))
10710	{
10711	tree size = TYPE_SIZE_UNIT (type);
10712	if (tree_int_cst_equal (size, off))
10713	return fold_build1 (IMAGPART_EXPR, type, TREE_OPERAND (addr, 0));
10714	}
10715
10716	/* *(p + CST) -> MEM_REF <p, CST>. */
10717	if (TREE_CODE (addr) != ADDR_EXPR
10718	|| DECL_P (TREE_OPERAND (addr, 0)))
10719	return fold_build2 (MEM_REF, type,
10720	addr,
10721	wide_int_to_tree (ptype, wi::to_wide (off)));
10722	}
10723
10724	/* *(foo *)fooarrptr => (*fooarrptr)[0] */
10725	if (TREE_CODE (TREE_TYPE (subtype)) == ARRAY_TYPE
10726	&& TREE_CODE (TYPE_SIZE (TREE_TYPE (TREE_TYPE (subtype)))) == INTEGER_CST
10727	&& useless_type_conversion_p (type, TREE_TYPE (TREE_TYPE (subtype))))
10728	{
10729	tree type_domain;
10730	tree min_val = size_zero_node;
10731	tree osub = sub;
10732	sub = gimple_fold_indirect_ref (t: sub);
10733	if (! sub)
10734	sub = build1 (INDIRECT_REF, TREE_TYPE (subtype), osub);
10735	type_domain = TYPE_DOMAIN (TREE_TYPE (sub));
10736	if (type_domain && TYPE_MIN_VALUE (type_domain))
10737	min_val = TYPE_MIN_VALUE (type_domain);
10738	if (TREE_CODE (min_val) == INTEGER_CST)
10739	return build4 (ARRAY_REF, type, sub, min_val, NULL_TREE, NULL_TREE);
10740	}
10741
10742	return NULL_TREE;
10743	}
10744
10745	/* Return true if CODE is an operation that when operating on signed
10746	integer types involves undefined behavior on overflow and the
10747	operation can be expressed with unsigned arithmetic. */
10748
10749	bool
10750	arith_code_with_undefined_signed_overflow (tree_code code)
10751	{
10752	switch (code)
10753	{
10754	case ABS_EXPR:
10755	case PLUS_EXPR:
10756	case MINUS_EXPR:
10757	case MULT_EXPR:
10758	case NEGATE_EXPR:
10759	case POINTER_PLUS_EXPR:
10760	return true;
10761	default:
10762	return false;
10763	}
10764	}
10765
10766	/* Return true if STMT has an operation that operates on a signed
10767	integer types involves undefined behavior on overflow and the
10768	operation can be expressed with unsigned arithmetic.
10769	Also returns true if STMT is a VCE that needs to be rewritten
10770	if moved to be executed unconditionally. */
10771
10772	bool
10773	gimple_needing_rewrite_undefined (gimple *stmt)
10774	{
10775	if (!is_gimple_assign (gs: stmt))
10776	return false;
10777	tree lhs = gimple_assign_lhs (gs: stmt);
10778	if (!lhs)
10779	return false;
10780	tree lhs_type = TREE_TYPE (lhs);
10781	if (!INTEGRAL_TYPE_P (lhs_type)
10782	&& !POINTER_TYPE_P (lhs_type))
10783	return false;
10784	tree rhs = gimple_assign_rhs1 (gs: stmt);
10785	/* Boolean loads need special handling as they are treated as a full MODE load
10786	and don't mask off the bits for the precision. */
10787	if (gimple_assign_load_p (stmt)
10788	/* Booleans are the integral type which has this non-masking issue. */
10789	&& TREE_CODE (lhs_type) == BOOLEAN_TYPE
10790	/* Only non mode precision booleans are need the masking. */
10791	&& !type_has_mode_precision_p (t: lhs_type)
10792	/* BFR should be the correct thing and just grab the precision. */
10793	&& TREE_CODE (rhs) != BIT_FIELD_REF
10794	/* Bit-fields loads don't need a rewrite as the masking
10795	happens for them. */
10796	&& (TREE_CODE (rhs) != COMPONENT_REF
10797	|| !DECL_BIT_FIELD (TREE_OPERAND (rhs, 1))))
10798	return true;
10799	/* VCE from integral types to a integral types but with
10800	a smaller precision need to be changed into casts
10801	to be well defined. */
10802	if (gimple_assign_rhs_code (gs: stmt) == VIEW_CONVERT_EXPR
10803	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (rhs, 0)))
10804	&& is_gimple_val (TREE_OPERAND (rhs, 0))
10805	&& TYPE_PRECISION (lhs_type)
10806	< TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (rhs, 0))))
10807	return true;
10808	if (!TYPE_OVERFLOW_UNDEFINED (lhs_type))
10809	return false;
10810	if (!arith_code_with_undefined_signed_overflow
10811	(code: gimple_assign_rhs_code (gs: stmt)))
10812	return false;
10813	return true;
10814	}
10815
10816	/* Rewrite STMT, an assignment with a signed integer or pointer arithmetic
10817	operation that can be transformed to unsigned arithmetic by converting
10818	its operand, carrying out the operation in the corresponding unsigned
10819	type and converting the result back to the original type.
10820
10821	If IN_PLACE is true, *GSI points to STMT, adjust the stmt in place and
10822	return NULL.
10823	Otherwise returns a sequence of statements that replace STMT and also
10824	contain a modified form of STMT itself. */
10825
10826	static gimple_seq
10827	rewrite_to_defined_unconditional (gimple_stmt_iterator *gsi, gimple *stmt,
10828	bool in_place)
10829	{
10830	gcc_assert (gimple_needing_rewrite_undefined (stmt));
10831	if (dump_file && (dump_flags & TDF_DETAILS))
10832	{
10833	fprintf (stream: dump_file, format: "rewriting stmt for being uncondtional defined");
10834	print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
10835	}
10836	gimple_seq stmts = NULL;
10837	tree lhs = gimple_assign_lhs (gs: stmt);
10838
10839	/* Boolean loads need to be rewritten to be a load from the same mode
10840	and then a cast to the other type so the other bits are masked off
10841	correctly since the load was done conditionally. It is similar to the VCE
10842	case below. */
10843	if (gimple_assign_load_p (stmt)
10844	&& TREE_CODE (TREE_TYPE (lhs)) == BOOLEAN_TYPE)
10845	{
10846	tree rhs = gimple_assign_rhs1 (gs: stmt);
10847
10848	/* Double check that gimple_needing_rewrite_undefined was called. */
10849	/* Bit-fields loads will do the masking so don't need the rewriting. */
10850	gcc_assert (TREE_CODE (rhs) != COMPONENT_REF
10851	|| !DECL_BIT_FIELD (TREE_OPERAND (rhs, 1)));
10852	/* BFR is like a bit field load and will do the correct thing. */
10853	gcc_assert (TREE_CODE (lhs) != BIT_FIELD_REF);
10854	/* Complex boolean types are not valid so REAL/IMAG part will
10855	never show up. */
10856	gcc_assert (TREE_CODE (rhs) != REALPART_EXPR
10857	&& TREE_CODE (lhs) != IMAGPART_EXPR);
10858
10859	auto bits = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (rhs)));
10860	tree new_type = build_nonstandard_integer_type (bits, true);
10861	location_t loc = gimple_location (g: stmt);
10862	tree mem_ref = fold_build1_loc (loc, VIEW_CONVERT_EXPR, new_type, rhs);
10863	/* Replace the original load with a new load and a new lhs. */
10864	tree new_lhs = make_ssa_name (var: new_type);
10865	gimple_assign_set_rhs1 (gs: stmt, rhs: mem_ref);
10866	gimple_assign_set_lhs (gs: stmt, lhs: new_lhs);
10867
10868	if (in_place)
10869	update_stmt (s: stmt);
10870	else
10871	{
10872	gimple_set_modified (s: stmt, modifiedp: true);
10873	gimple_seq_add_stmt (&stmts, stmt);
10874	}
10875
10876	/* Build the conversion statement. */
10877	gimple *cvt = gimple_build_assign (lhs, NOP_EXPR, new_lhs);
10878	if (in_place)
10879	{
10880	gsi_insert_after (gsi, cvt, GSI_SAME_STMT);
10881	update_stmt (s: stmt);
10882	}
10883	else
10884	gimple_seq_add_stmt (&stmts, cvt);
10885	return stmts;
10886	}
10887
10888	/* VCE from integral types to another integral types but with
10889	smaller precisions need to be changed into casts
10890	to be well defined. */
10891	if (gimple_assign_rhs_code (gs: stmt) == VIEW_CONVERT_EXPR)
10892	{
10893	tree rhs = gimple_assign_rhs1 (gs: stmt);
10894	tree new_rhs = TREE_OPERAND (rhs, 0);
10895	gcc_assert (TYPE_PRECISION (TREE_TYPE (rhs))
10896	< TYPE_PRECISION (TREE_TYPE (new_rhs)));
10897	gcc_assert (is_gimple_val (new_rhs));
10898	gimple_assign_set_rhs_code (s: stmt, code: NOP_EXPR);
10899	gimple_assign_set_rhs1 (gs: stmt, rhs: new_rhs);
10900	if (in_place)
10901	update_stmt (s: stmt);
10902	else
10903	{
10904	gimple_set_modified (s: stmt, modifiedp: true);
10905	gimple_seq_add_stmt (&stmts, stmt);
10906	}
10907	return stmts;
10908	}
10909	tree type = unsigned_type_for (TREE_TYPE (lhs));
10910	if (gimple_assign_rhs_code (gs: stmt) == ABS_EXPR)
10911	gimple_assign_set_rhs_code (s: stmt, code: ABSU_EXPR);
10912	else
10913	for (unsigned i = 1; i < gimple_num_ops (gs: stmt); ++i)
10914	{
10915	tree op = gimple_op (gs: stmt, i);
10916	op = gimple_convert (seq: &stmts, type, op);
10917	gimple_set_op (gs: stmt, i, op);
10918	}
10919	gimple_assign_set_lhs (gs: stmt, lhs: make_ssa_name (var: type, stmt));
10920	if (gimple_assign_rhs_code (gs: stmt) == POINTER_PLUS_EXPR)
10921	gimple_assign_set_rhs_code (s: stmt, code: PLUS_EXPR);
10922	gimple_set_modified (s: stmt, modifiedp: true);
10923	if (in_place)
10924	{
10925	if (stmts)
10926	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
10927	stmts = NULL;
10928	}
10929	else
10930	gimple_seq_add_stmt (&stmts, stmt);
10931	gimple *cvt = gimple_build_assign (lhs, NOP_EXPR, gimple_assign_lhs (gs: stmt));
10932	if (in_place)
10933	{
10934	gsi_insert_after (gsi, cvt, GSI_SAME_STMT);
10935	update_stmt (s: stmt);
10936	}
10937	else
10938	gimple_seq_add_stmt (&stmts, cvt);
10939
10940	return stmts;
10941	}
10942
10943	void
10944	rewrite_to_defined_unconditional (gimple_stmt_iterator *gsi)
10945	{
10946	rewrite_to_defined_unconditional (gsi, stmt: gsi_stmt (i: *gsi), in_place: true);
10947	}
10948
10949	gimple_seq
10950	rewrite_to_defined_unconditional (gimple *stmt)
10951	{
10952	return rewrite_to_defined_unconditional (gsi: nullptr, stmt, in_place: false);
10953	}
10954
10955	/* The valueization hook we use for the gimple_build API simplification.
10956	This makes us match fold_buildN behavior by only combining with
10957	statements in the sequence(s) we are currently building. */
10958
10959	static tree
10960	gimple_build_valueize (tree op)
10961	{
10962	if (gimple_bb (SSA_NAME_DEF_STMT (op)) == NULL)
10963	return op;
10964	return NULL_TREE;
10965	}
10966
10967	/* Helper for gimple_build to perform the final insertion of stmts on SEQ. */
10968
10969	static inline void
10970	gimple_build_insert_seq (gimple_stmt_iterator *gsi,
10971	bool before, gsi_iterator_update update,
10972	gimple_seq seq)
10973	{
10974	if (before)
10975	{
10976	if (gsi->bb)
10977	gsi_insert_seq_before (gsi, seq, update);
10978	else
10979	gsi_insert_seq_before_without_update (gsi, seq, update);
10980	}
10981	else
10982	{
10983	if (gsi->bb)
10984	gsi_insert_seq_after (gsi, seq, update);
10985	else
10986	gsi_insert_seq_after_without_update (gsi, seq, update);
10987	}
10988	}
10989
10990	/* Build the expression CODE OP0 of type TYPE with location LOC,
10991	simplifying it first if possible. Returns the built
10992	expression value and inserts statements possibly defining it
10993	before GSI if BEFORE is true or after GSI if false and advance
10994	the iterator accordingly.
10995	If gsi refers to a basic block simplifying is allowed to look
10996	at all SSA defs while when it does not it is restricted to
10997	SSA defs that are not associated with a basic block yet,
10998	indicating they belong to the currently building sequence. */
10999
11000	tree
11001	gimple_build (gimple_stmt_iterator *gsi,
11002	bool before, gsi_iterator_update update,
11003	location_t loc, enum tree_code code, tree type, tree op0)
11004	{
11005	gimple_seq seq = NULL;
11006	tree res
11007	= gimple_simplify (code, type, op0, &seq,
11008	gsi->bb ? follow_all_ssa_edges : gimple_build_valueize);
11009	if (!res)
11010	{
11011	res = make_ssa_name (var: type);
11012	gimple *stmt;
11013	if (code == REALPART_EXPR
11014	|| code == IMAGPART_EXPR
11015	|| code == VIEW_CONVERT_EXPR)
11016	stmt = gimple_build_assign (res, code, build1 (code, type, op0));
11017	else
11018	stmt = gimple_build_assign (res, code, op0);
11019	gimple_set_location (g: stmt, location: loc);
11020	gimple_seq_add_stmt_without_update (&seq, stmt);
11021	}
11022	gimple_build_insert_seq (gsi, before, update, seq);
11023	return res;
11024	}
11025
11026	/* Build the expression OP0 CODE OP1 of type TYPE with location LOC,
11027	simplifying it first if possible. Returns the built
11028	expression value inserting any new statements at GSI honoring BEFORE
11029	and UPDATE. */
11030
11031	tree
11032	gimple_build (gimple_stmt_iterator *gsi,
11033	bool before, gsi_iterator_update update,
11034	location_t loc, enum tree_code code, tree type,
11035	tree op0, tree op1)
11036	{
11037	gimple_seq seq = NULL;
11038	tree res
11039	= gimple_simplify (code, type, op0, op1, &seq,
11040	gsi->bb ? follow_all_ssa_edges : gimple_build_valueize);
11041	if (!res)
11042	{
11043	res = make_ssa_name (var: type);
11044	gimple *stmt = gimple_build_assign (res, code, op0, op1);
11045	gimple_set_location (g: stmt, location: loc);
11046	gimple_seq_add_stmt_without_update (&seq, stmt);
11047	}
11048	gimple_build_insert_seq (gsi, before, update, seq);
11049	return res;
11050	}
11051
11052	/* Build the expression (CODE OP0 OP1 OP2) of type TYPE with location LOC,
11053	simplifying it first if possible. Returns the built
11054	expression value inserting any new statements at GSI honoring BEFORE
11055	and UPDATE. */
11056
11057	tree
11058	gimple_build (gimple_stmt_iterator *gsi,
11059	bool before, gsi_iterator_update update,
11060	location_t loc, enum tree_code code, tree type,
11061	tree op0, tree op1, tree op2)
11062	{
11063
11064	gimple_seq seq = NULL;
11065	tree res
11066	= gimple_simplify (code, type, op0, op1, op2, &seq,
11067	gsi->bb ? follow_all_ssa_edges : gimple_build_valueize);
11068	if (!res)
11069	{
11070	res = make_ssa_name (var: type);
11071	gimple *stmt;
11072	if (code == BIT_FIELD_REF)
11073	stmt = gimple_build_assign (res, code,
11074	build3 (code, type, op0, op1, op2));
11075	else
11076	stmt = gimple_build_assign (res, code, op0, op1, op2);
11077	gimple_set_location (g: stmt, location: loc);
11078	gimple_seq_add_stmt_without_update (&seq, stmt);
11079	}
11080	gimple_build_insert_seq (gsi, before, update, seq);
11081	return res;
11082	}
11083
11084	/* Build the call FN () with a result of type TYPE (or no result if TYPE is
11085	void) with a location LOC. Returns the built expression value (or NULL_TREE
11086	if TYPE is void) inserting any new statements at GSI honoring BEFORE
11087	and UPDATE. */
11088
11089	tree
11090	gimple_build (gimple_stmt_iterator *gsi,
11091	bool before, gsi_iterator_update update,
11092	location_t loc, combined_fn fn, tree type)
11093	{
11094	tree res = NULL_TREE;
11095	gimple_seq seq = NULL;
11096	gcall *stmt;
11097	if (internal_fn_p (code: fn))
11098	stmt = gimple_build_call_internal (as_internal_fn (code: fn), 0);
11099	else
11100	{
11101	tree decl = builtin_decl_implicit (fncode: as_builtin_fn (code: fn));
11102	stmt = gimple_build_call (decl, 0);
11103	}
11104	if (!VOID_TYPE_P (type))
11105	{
11106	res = make_ssa_name (var: type);
11107	gimple_call_set_lhs (gs: stmt, lhs: res);
11108	}
11109	gimple_set_location (g: stmt, location: loc);
11110	gimple_seq_add_stmt_without_update (&seq, stmt);
11111	gimple_build_insert_seq (gsi, before, update, seq);
11112	return res;
11113	}
11114
11115	/* Build the call FN (ARG0) with a result of type TYPE
11116	(or no result if TYPE is void) with location LOC,
11117	simplifying it first if possible. Returns the built
11118	expression value (or NULL_TREE if TYPE is void) inserting any new
11119	statements at GSI honoring BEFORE and UPDATE. */
11120
11121	tree
11122	gimple_build (gimple_stmt_iterator *gsi,
11123	bool before, gsi_iterator_update update,
11124	location_t loc, combined_fn fn,
11125	tree type, tree arg0)
11126	{
11127	gimple_seq seq = NULL;
11128	tree res = gimple_simplify (fn, type, arg0, &seq, gimple_build_valueize);
11129	if (!res)
11130	{
11131	gcall *stmt;
11132	if (internal_fn_p (code: fn))
11133	stmt = gimple_build_call_internal (as_internal_fn (code: fn), 1, arg0);
11134	else
11135	{
11136	tree decl = builtin_decl_implicit (fncode: as_builtin_fn (code: fn));
11137	stmt = gimple_build_call (decl, 1, arg0);
11138	}
11139	if (!VOID_TYPE_P (type))
11140	{
11141	res = make_ssa_name (var: type);
11142	gimple_call_set_lhs (gs: stmt, lhs: res);
11143	}
11144	gimple_set_location (g: stmt, location: loc);
11145	gimple_seq_add_stmt_without_update (&seq, stmt);
11146	}
11147	gimple_build_insert_seq (gsi, before, update, seq);
11148	return res;
11149	}
11150
11151	/* Build the call FN (ARG0, ARG1) with a result of type TYPE
11152	(or no result if TYPE is void) with location LOC,
11153	simplifying it first if possible. Returns the built
11154	expression value (or NULL_TREE if TYPE is void) inserting any new
11155	statements at GSI honoring BEFORE and UPDATE. */
11156
11157	tree
11158	gimple_build (gimple_stmt_iterator *gsi,
11159	bool before, gsi_iterator_update update,
11160	location_t loc, combined_fn fn,
11161	tree type, tree arg0, tree arg1)
11162	{
11163	gimple_seq seq = NULL;
11164	tree res = gimple_simplify (fn, type, arg0, arg1, &seq,
11165	gimple_build_valueize);
11166	if (!res)
11167	{
11168	gcall *stmt;
11169	if (internal_fn_p (code: fn))
11170	stmt = gimple_build_call_internal (as_internal_fn (code: fn), 2, arg0, arg1);
11171	else
11172	{
11173	tree decl = builtin_decl_implicit (fncode: as_builtin_fn (code: fn));
11174	stmt = gimple_build_call (decl, 2, arg0, arg1);
11175	}
11176	if (!VOID_TYPE_P (type))
11177	{
11178	res = make_ssa_name (var: type);
11179	gimple_call_set_lhs (gs: stmt, lhs: res);
11180	}
11181	gimple_set_location (g: stmt, location: loc);
11182	gimple_seq_add_stmt_without_update (&seq, stmt);
11183	}
11184	gimple_build_insert_seq (gsi, before, update, seq);
11185	return res;
11186	}
11187
11188	/* Build the call FN (ARG0, ARG1, ARG2) with a result of type TYPE
11189	(or no result if TYPE is void) with location LOC,
11190	simplifying it first if possible. Returns the built
11191	expression value (or NULL_TREE if TYPE is void) inserting any new
11192	statements at GSI honoring BEFORE and UPDATE. */
11193
11194	tree
11195	gimple_build (gimple_stmt_iterator *gsi,
11196	bool before, gsi_iterator_update update,
11197	location_t loc, combined_fn fn,
11198	tree type, tree arg0, tree arg1, tree arg2)
11199	{
11200	gimple_seq seq = NULL;
11201	tree res = gimple_simplify (fn, type, arg0, arg1, arg2,
11202	&seq, gimple_build_valueize);
11203	if (!res)
11204	{
11205	gcall *stmt;
11206	if (internal_fn_p (code: fn))
11207	stmt = gimple_build_call_internal (as_internal_fn (code: fn),
11208	3, arg0, arg1, arg2);
11209	else
11210	{
11211	tree decl = builtin_decl_implicit (fncode: as_builtin_fn (code: fn));
11212	stmt = gimple_build_call (decl, 3, arg0, arg1, arg2);
11213	}
11214	if (!VOID_TYPE_P (type))
11215	{
11216	res = make_ssa_name (var: type);
11217	gimple_call_set_lhs (gs: stmt, lhs: res);
11218	}
11219	gimple_set_location (g: stmt, location: loc);
11220	gimple_seq_add_stmt_without_update (&seq, stmt);
11221	}
11222	gimple_build_insert_seq (gsi, before, update, seq);
11223	return res;
11224	}
11225
11226	/* Build CODE (OP0) with a result of type TYPE (or no result if TYPE is
11227	void) with location LOC, simplifying it first if possible. Returns the
11228	built expression value (or NULL_TREE if TYPE is void) inserting any new
11229	statements at GSI honoring BEFORE and UPDATE. */
11230
11231	tree
11232	gimple_build (gimple_stmt_iterator *gsi,
11233	bool before, gsi_iterator_update update,
11234	location_t loc, code_helper code, tree type, tree op0)
11235	{
11236	if (code.is_tree_code ())
11237	return gimple_build (gsi, before, update, loc, code: tree_code (code), type, op0);
11238	return gimple_build (gsi, before, update, loc, fn: combined_fn (code), type, arg0: op0);
11239	}
11240
11241	/* Build CODE (OP0, OP1) with a result of type TYPE (or no result if TYPE is
11242	void) with location LOC, simplifying it first if possible. Returns the
11243	built expression value (or NULL_TREE if TYPE is void) inserting any new
11244	statements at GSI honoring BEFORE and UPDATE. */
11245
11246	tree
11247	gimple_build (gimple_stmt_iterator *gsi,
11248	bool before, gsi_iterator_update update,
11249	location_t loc, code_helper code, tree type, tree op0, tree op1)
11250	{
11251	if (code.is_tree_code ())
11252	return gimple_build (gsi, before, update,
11253	loc, code: tree_code (code), type, op0, op1);
11254	return gimple_build (gsi, before, update,
11255	loc, fn: combined_fn (code), type, arg0: op0, arg1: op1);
11256	}
11257
11258	/* Build CODE (OP0, OP1, OP2) with a result of type TYPE (or no result if TYPE
11259	is void) with location LOC, simplifying it first if possible. Returns the
11260	built expression value (or NULL_TREE if TYPE is void) inserting any new
11261	statements at GSI honoring BEFORE and UPDATE. */
11262
11263	tree
11264	gimple_build (gimple_stmt_iterator *gsi,
11265	bool before, gsi_iterator_update update,
11266	location_t loc, code_helper code,
11267	tree type, tree op0, tree op1, tree op2)
11268	{
11269	if (code.is_tree_code ())
11270	return gimple_build (gsi, before, update,
11271	loc, code: tree_code (code), type, op0, op1, op2);
11272	return gimple_build (gsi, before, update,
11273	loc, fn: combined_fn (code), type, arg0: op0, arg1: op1, arg2: op2);
11274	}
11275
11276	/* Build the conversion (TYPE) OP with a result of type TYPE
11277	with location LOC if such conversion is neccesary in GIMPLE,
11278	simplifying it first.
11279	Returns the built expression inserting any new statements
11280	at GSI honoring BEFORE and UPDATE. */
11281
11282	tree
11283	gimple_convert (gimple_stmt_iterator *gsi,
11284	bool before, gsi_iterator_update update,
11285	location_t loc, tree type, tree op)
11286	{
11287	if (useless_type_conversion_p (type, TREE_TYPE (op)))
11288	return op;
11289	return gimple_build (gsi, before, update, loc, code: NOP_EXPR, type, op0: op);
11290	}
11291
11292	/* Build the conversion (ptrofftype) OP with a result of a type
11293	compatible with ptrofftype with location LOC if such conversion
11294	is neccesary in GIMPLE, simplifying it first.
11295	Returns the built expression value inserting any new statements
11296	at GSI honoring BEFORE and UPDATE. */
11297
11298	tree
11299	gimple_convert_to_ptrofftype (gimple_stmt_iterator *gsi,
11300	bool before, gsi_iterator_update update,
11301	location_t loc, tree op)
11302	{
11303	if (ptrofftype_p (TREE_TYPE (op)))
11304	return op;
11305	return gimple_convert (gsi, before, update, loc, sizetype, op);
11306	}
11307
11308	/* Build a vector of type TYPE in which each element has the value OP.
11309	Return a gimple value for the result, inserting any new statements
11310	at GSI honoring BEFORE and UPDATE. */
11311
11312	tree
11313	gimple_build_vector_from_val (gimple_stmt_iterator *gsi,
11314	bool before, gsi_iterator_update update,
11315	location_t loc, tree type, tree op)
11316	{
11317	if (!TYPE_VECTOR_SUBPARTS (node: type).is_constant ()
11318	&& !CONSTANT_CLASS_P (op))
11319	return gimple_build (gsi, before, update,
11320	loc, code: VEC_DUPLICATE_EXPR, type, op0: op);
11321
11322	tree res, vec = build_vector_from_val (type, op);
11323	if (is_gimple_val (vec))
11324	return vec;
11325	if (gimple_in_ssa_p (cfun))
11326	res = make_ssa_name (var: type);
11327	else
11328	res = create_tmp_reg (type);
11329	gimple_seq seq = NULL;
11330	gimple *stmt = gimple_build_assign (res, vec);
11331	gimple_set_location (g: stmt, location: loc);
11332	gimple_seq_add_stmt_without_update (&seq, stmt);
11333	gimple_build_insert_seq (gsi, before, update, seq);
11334	return res;
11335	}
11336
11337	/* Build a vector from BUILDER, handling the case in which some elements
11338	are non-constant. Return a gimple value for the result, inserting
11339	any new instructions to GSI honoring BEFORE and UPDATE.
11340
11341	BUILDER must not have a stepped encoding on entry. This is because
11342	the function is not geared up to handle the arithmetic that would
11343	be needed in the variable case, and any code building a vector that
11344	is known to be constant should use BUILDER->build () directly. */
11345
11346	tree
11347	gimple_build_vector (gimple_stmt_iterator *gsi,
11348	bool before, gsi_iterator_update update,
11349	location_t loc, tree_vector_builder *builder)
11350	{
11351	gcc_assert (builder->nelts_per_pattern () <= 2);
11352	unsigned int encoded_nelts = builder->encoded_nelts ();
11353	for (unsigned int i = 0; i < encoded_nelts; ++i)
11354	if (!CONSTANT_CLASS_P ((*builder)[i]))
11355	{
11356	gimple_seq seq = NULL;
11357	tree type = builder->type ();
11358	unsigned int nelts = TYPE_VECTOR_SUBPARTS (node: type).to_constant ();
11359	vec<constructor_elt, va_gc> *v;
11360	vec_alloc (v, nelems: nelts);
11361	for (i = 0; i < nelts; ++i)
11362	CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, builder->elt (i));
11363
11364	tree res;
11365	if (gimple_in_ssa_p (cfun))
11366	res = make_ssa_name (var: type);
11367	else
11368	res = create_tmp_reg (type);
11369	gimple *stmt = gimple_build_assign (res, build_constructor (type, v));
11370	gimple_set_location (g: stmt, location: loc);
11371	gimple_seq_add_stmt_without_update (&seq, stmt);
11372	gimple_build_insert_seq (gsi, before, update, seq);
11373	return res;
11374	}
11375	return builder->build ();
11376	}
11377
11378	/* Emit gimple statements into &stmts that take a value given in OLD_SIZE
11379	and generate a value guaranteed to be rounded upwards to ALIGN.
11380
11381	Return the tree node representing this size, it is of TREE_TYPE TYPE. */
11382
11383	tree
11384	gimple_build_round_up (gimple_stmt_iterator *gsi,
11385	bool before, gsi_iterator_update update,
11386	location_t loc, tree type,
11387	tree old_size, unsigned HOST_WIDE_INT align)
11388	{
11389	unsigned HOST_WIDE_INT tg_mask = align - 1;
11390	/* tree new_size = (old_size + tg_mask) & ~tg_mask; */
11391	gcc_assert (INTEGRAL_TYPE_P (type));
11392	tree tree_mask = build_int_cst (type, tg_mask);
11393	tree oversize = gimple_build (gsi, before, update,
11394	loc, code: PLUS_EXPR, type, op0: old_size, op1: tree_mask);
11395
11396	tree mask = build_int_cst (type, -align);
11397	return gimple_build (gsi, before, update,
11398	loc, code: BIT_AND_EXPR, type, op0: oversize, op1: mask);
11399	}
11400
11401	/* Return true if the result of assignment STMT is known to be non-negative.
11402	If the return value is based on the assumption that signed overflow is
11403	undefined, set *STRICT_OVERFLOW_P to true; otherwise, don't change
11404	*STRICT_OVERFLOW_P. DEPTH is the current nesting depth of the query. */
11405
11406	static bool
11407	gimple_assign_nonnegative_warnv_p (gimple *stmt, bool *strict_overflow_p,
11408	int depth)
11409	{
11410	enum tree_code code = gimple_assign_rhs_code (gs: stmt);
11411	tree type = TREE_TYPE (gimple_assign_lhs (stmt));
11412	switch (get_gimple_rhs_class (code))
11413	{
11414	case GIMPLE_UNARY_RHS:
11415	return tree_unary_nonnegative_warnv_p (gimple_assign_rhs_code (gs: stmt),
11416	type,
11417	gimple_assign_rhs1 (gs: stmt),
11418	strict_overflow_p, depth);
11419	case GIMPLE_BINARY_RHS:
11420	return tree_binary_nonnegative_warnv_p (gimple_assign_rhs_code (gs: stmt),
11421	type,
11422	gimple_assign_rhs1 (gs: stmt),
11423	gimple_assign_rhs2 (gs: stmt),
11424	strict_overflow_p, depth);
11425	case GIMPLE_TERNARY_RHS:
11426	return false;
11427	case GIMPLE_SINGLE_RHS:
11428	return tree_single_nonnegative_warnv_p (gimple_assign_rhs1 (gs: stmt),
11429	strict_overflow_p, depth);
11430	case GIMPLE_INVALID_RHS:
11431	break;
11432	}
11433	gcc_unreachable ();
11434	}
11435
11436	/* Return true if return value of call STMT is known to be non-negative.
11437	If the return value is based on the assumption that signed overflow is
11438	undefined, set *STRICT_OVERFLOW_P to true; otherwise, don't change
11439	*STRICT_OVERFLOW_P. DEPTH is the current nesting depth of the query. */
11440
11441	static bool
11442	gimple_call_nonnegative_warnv_p (gimple *stmt, bool *strict_overflow_p,
11443	int depth)
11444	{
11445	tree arg0
11446	= gimple_call_num_args (gs: stmt) > 0 ? gimple_call_arg (gs: stmt, index: 0) : NULL_TREE;
11447	tree arg1
11448	= gimple_call_num_args (gs: stmt) > 1 ? gimple_call_arg (gs: stmt, index: 1) : NULL_TREE;
11449	tree lhs = gimple_call_lhs (gs: stmt);
11450	return (lhs
11451	&& tree_call_nonnegative_warnv_p (TREE_TYPE (lhs),
11452	gimple_call_combined_fn (stmt),
11453	arg0, arg1,
11454	strict_overflow_p, depth));
11455	}
11456
11457	/* Return true if return value of call STMT is known to be non-negative.
11458	If the return value is based on the assumption that signed overflow is
11459	undefined, set *STRICT_OVERFLOW_P to true; otherwise, don't change
11460	*STRICT_OVERFLOW_P. DEPTH is the current nesting depth of the query. */
11461
11462	static bool
11463	gimple_phi_nonnegative_warnv_p (gimple *stmt, bool *strict_overflow_p,
11464	int depth)
11465	{
11466	for (unsigned i = 0; i < gimple_phi_num_args (gs: stmt); ++i)
11467	{
11468	tree arg = gimple_phi_arg_def (gs: stmt, index: i);
11469	if (!tree_single_nonnegative_warnv_p (arg, strict_overflow_p, depth + 1))
11470	return false;
11471	}
11472	return true;
11473	}
11474
11475	/* Return true if STMT is known to compute a non-negative value.
11476	If the return value is based on the assumption that signed overflow is
11477	undefined, set *STRICT_OVERFLOW_P to true; otherwise, don't change
11478	*STRICT_OVERFLOW_P. DEPTH is the current nesting depth of the query. */
11479
11480	bool
11481	gimple_stmt_nonnegative_warnv_p (gimple *stmt, bool *strict_overflow_p,
11482	int depth)
11483	{
11484	tree type = gimple_range_type (s: stmt);
11485	if (type && frange::supports_p (type))
11486	{
11487	frange r;
11488	bool sign;
11489	if (get_global_range_query ()->range_of_stmt (r, stmt)
11490	&& r.signbit_p (signbit&: sign))
11491	return !sign;
11492	}
11493	switch (gimple_code (g: stmt))
11494	{
11495	case GIMPLE_ASSIGN:
11496	return gimple_assign_nonnegative_warnv_p (stmt, strict_overflow_p,
11497	depth);
11498	case GIMPLE_CALL:
11499	return gimple_call_nonnegative_warnv_p (stmt, strict_overflow_p,
11500	depth);
11501	case GIMPLE_PHI:
11502	return gimple_phi_nonnegative_warnv_p (stmt, strict_overflow_p,
11503	depth);
11504	default:
11505	return false;
11506	}
11507	}
11508
11509	/* Return true if the floating-point value computed by assignment STMT
11510	is known to have an integer value. We also allow +Inf, -Inf and NaN
11511	to be considered integer values. Return false for signaling NaN.
11512
11513	DEPTH is the current nesting depth of the query. */
11514
11515	static bool
11516	gimple_assign_integer_valued_real_p (gimple *stmt, int depth)
11517	{
11518	enum tree_code code = gimple_assign_rhs_code (gs: stmt);
11519	switch (get_gimple_rhs_class (code))
11520	{
11521	case GIMPLE_UNARY_RHS:
11522	return integer_valued_real_unary_p (gimple_assign_rhs_code (gs: stmt),
11523	gimple_assign_rhs1 (gs: stmt), depth);
11524	case GIMPLE_BINARY_RHS:
11525	return integer_valued_real_binary_p (gimple_assign_rhs_code (gs: stmt),
11526	gimple_assign_rhs1 (gs: stmt),
11527	gimple_assign_rhs2 (gs: stmt), depth);
11528	case GIMPLE_TERNARY_RHS:
11529	return false;
11530	case GIMPLE_SINGLE_RHS:
11531	return integer_valued_real_single_p (gimple_assign_rhs1 (gs: stmt), depth);
11532	case GIMPLE_INVALID_RHS:
11533	break;
11534	}
11535	gcc_unreachable ();
11536	}
11537
11538	/* Return true if the floating-point value computed by call STMT is known
11539	to have an integer value. We also allow +Inf, -Inf and NaN to be
11540	considered integer values. Return false for signaling NaN.
11541
11542	DEPTH is the current nesting depth of the query. */
11543
11544	static bool
11545	gimple_call_integer_valued_real_p (gimple *stmt, int depth)
11546	{
11547	tree arg0 = (gimple_call_num_args (gs: stmt) > 0
11548	? gimple_call_arg (gs: stmt, index: 0)
11549	: NULL_TREE);
11550	tree arg1 = (gimple_call_num_args (gs: stmt) > 1
11551	? gimple_call_arg (gs: stmt, index: 1)
11552	: NULL_TREE);
11553	return integer_valued_real_call_p (gimple_call_combined_fn (stmt),
11554	arg0, arg1, depth);
11555	}
11556
11557	/* Return true if the floating-point result of phi STMT is known to have
11558	an integer value. We also allow +Inf, -Inf and NaN to be considered
11559	integer values. Return false for signaling NaN.
11560
11561	DEPTH is the current nesting depth of the query. */
11562
11563	static bool
11564	gimple_phi_integer_valued_real_p (gimple *stmt, int depth)
11565	{
11566	for (unsigned i = 0; i < gimple_phi_num_args (gs: stmt); ++i)
11567	{
11568	tree arg = gimple_phi_arg_def (gs: stmt, index: i);
11569	if (!integer_valued_real_single_p (arg, depth + 1))
11570	return false;
11571	}
11572	return true;
11573	}
11574
11575	/* Return true if the floating-point value computed by STMT is known
11576	to have an integer value. We also allow +Inf, -Inf and NaN to be
11577	considered integer values. Return false for signaling NaN.
11578
11579	DEPTH is the current nesting depth of the query. */
11580
11581	bool
11582	gimple_stmt_integer_valued_real_p (gimple *stmt, int depth)
11583	{
11584	switch (gimple_code (g: stmt))
11585	{
11586	case GIMPLE_ASSIGN:
11587	return gimple_assign_integer_valued_real_p (stmt, depth);
11588	case GIMPLE_CALL:
11589	return gimple_call_integer_valued_real_p (stmt, depth);
11590	case GIMPLE_PHI:
11591	return gimple_phi_integer_valued_real_p (stmt, depth);
11592	default:
11593	return false;
11594	}
11595	}
11596