1	/* Expand builtin functions.
2	Copyright (C) 1988-2024 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it under
7	the terms of the GNU General Public License as published by the Free
8	Software Foundation; either version 3, or (at your option) any later
9	version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14	for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	/* Legacy warning! Please add no further builtin simplifications here
21	(apart from pure constant folding) - builtin simplifications should go
22	to match.pd or gimple-fold.cc instead. */
23
24	#include "config.h"
25	#include "system.h"
26	#include "coretypes.h"
27	#include "backend.h"
28	#include "target.h"
29	#include "rtl.h"
30	#include "tree.h"
31	#include "memmodel.h"
32	#include "gimple.h"
33	#include "predict.h"
34	#include "tm_p.h"
35	#include "stringpool.h"
36	#include "tree-vrp.h"
37	#include "tree-ssanames.h"
38	#include "expmed.h"
39	#include "optabs.h"
40	#include "emit-rtl.h"
41	#include "recog.h"
42	#include "diagnostic-core.h"
43	#include "alias.h"
44	#include "fold-const.h"
45	#include "fold-const-call.h"
46	#include "gimple-ssa-warn-access.h"
47	#include "stor-layout.h"
48	#include "calls.h"
49	#include "varasm.h"
50	#include "tree-object-size.h"
51	#include "tree-ssa-strlen.h"
52	#include "realmpfr.h"
53	#include "cfgrtl.h"
54	#include "except.h"
55	#include "dojump.h"
56	#include "explow.h"
57	#include "stmt.h"
58	#include "expr.h"
59	#include "libfuncs.h"
60	#include "output.h"
61	#include "typeclass.h"
62	#include "langhooks.h"
63	#include "value-prof.h"
64	#include "builtins.h"
65	#include "stringpool.h"
66	#include "attribs.h"
67	#include "asan.h"
68	#include "internal-fn.h"
69	#include "case-cfn-macros.h"
70	#include "gimple-iterator.h"
71	#include "gimple-fold.h"
72	#include "intl.h"
73	#include "file-prefix-map.h" /* remap_macro_filename() */
74	#include "ipa-strub.h" /* strub_watermark_parm() */
75	#include "gomp-constants.h"
76	#include "omp-general.h"
77	#include "tree-dfa.h"
78	#include "gimple-ssa.h"
79	#include "tree-ssa-live.h"
80	#include "tree-outof-ssa.h"
81	#include "attr-fnspec.h"
82	#include "demangle.h"
83	#include "gimple-range.h"
84	#include "pointer-query.h"
85
86	struct target_builtins default_target_builtins;
87	#if SWITCHABLE_TARGET
88	struct target_builtins *this_target_builtins = &default_target_builtins;
89	#endif
90
91	/* Define the names of the builtin function types and codes. */
92	const char *const built_in_class_names[BUILT_IN_LAST]
93	= {"NOT_BUILT_IN", "BUILT_IN_FRONTEND", "BUILT_IN_MD", "BUILT_IN_NORMAL"};
94
95	#define DEF_BUILTIN(X, N, C, T, LT, B, F, NA, AT, IM, COND) #X,
96	const char * built_in_names[(int) END_BUILTINS] =
97	{
98	#include "builtins.def"
99	};
100
101	/* Setup an array of builtin_info_type, make sure each element decl is
102	initialized to NULL_TREE. */
103	builtin_info_type builtin_info[(int)END_BUILTINS];
104
105	/* Non-zero if __builtin_constant_p should be folded right away. */
106	bool force_folding_builtin_constant_p;
107
108	static int target_char_cast (tree, char *);
109	static int apply_args_size (void);
110	static int apply_result_size (void);
111	static rtx result_vector (int, rtx);
112	static void expand_builtin_prefetch (tree);
113	static rtx expand_builtin_apply_args (void);
114	static rtx expand_builtin_apply_args_1 (void);
115	static rtx expand_builtin_apply (rtx, rtx, rtx);
116	static void expand_builtin_return (rtx);
117	static rtx expand_builtin_classify_type (tree);
118	static rtx expand_builtin_mathfn_3 (tree, rtx, rtx);
119	static rtx expand_builtin_mathfn_ternary (tree, rtx, rtx);
120	static rtx expand_builtin_interclass_mathfn (tree, rtx);
121	static rtx expand_builtin_sincos (tree);
122	static rtx expand_builtin_fegetround (tree, rtx, machine_mode);
123	static rtx expand_builtin_feclear_feraise_except (tree, rtx, machine_mode,
124	optab);
125	static rtx expand_builtin_cexpi (tree, rtx);
126	static rtx expand_builtin_issignaling (tree, rtx);
127	static rtx expand_builtin_int_roundingfn (tree, rtx);
128	static rtx expand_builtin_int_roundingfn_2 (tree, rtx);
129	static rtx expand_builtin_next_arg (void);
130	static rtx expand_builtin_va_start (tree);
131	static rtx expand_builtin_va_end (tree);
132	static rtx expand_builtin_va_copy (tree);
133	static rtx inline_expand_builtin_bytecmp (tree, rtx);
134	static rtx expand_builtin_strcmp (tree, rtx);
135	static rtx expand_builtin_strncmp (tree, rtx, machine_mode);
136	static rtx expand_builtin_memcpy (tree, rtx);
137	static rtx expand_builtin_memory_copy_args (tree dest, tree src, tree len,
138	rtx target, tree exp,
139	memop_ret retmode,
140	bool might_overlap);
141	static rtx expand_builtin_memmove (tree, rtx);
142	static rtx expand_builtin_mempcpy (tree, rtx);
143	static rtx expand_builtin_mempcpy_args (tree, tree, tree, rtx, tree, memop_ret);
144	static rtx expand_builtin_strcpy (tree, rtx);
145	static rtx expand_builtin_strcpy_args (tree, tree, tree, rtx);
146	static rtx expand_builtin_stpcpy (tree, rtx, machine_mode);
147	static rtx expand_builtin_strncpy (tree, rtx);
148	static rtx expand_builtin_memset_args (tree, tree, tree, rtx, machine_mode, tree);
149	static rtx expand_builtin_bzero (tree);
150	static rtx expand_builtin_strlen (tree, rtx, machine_mode);
151	static rtx expand_builtin_strnlen (tree, rtx, machine_mode);
152	static rtx expand_builtin_alloca (tree);
153	static rtx expand_builtin_unop (machine_mode, tree, rtx, rtx, optab);
154	static rtx expand_builtin_frame_address (tree, tree);
155	static rtx expand_builtin_stack_address ();
156	static tree stabilize_va_list_loc (location_t, tree, int);
157	static rtx expand_builtin_expect (tree, rtx);
158	static rtx expand_builtin_expect_with_probability (tree, rtx);
159	static tree fold_builtin_constant_p (tree);
160	static tree fold_builtin_classify_type (tree);
161	static tree fold_builtin_strlen (location_t, tree, tree, tree);
162	static tree fold_builtin_inf (location_t, tree, int);
163	static tree rewrite_call_expr (location_t, tree, int, tree, int, ...);
164	static bool validate_arg (const_tree, enum tree_code code);
165	static rtx expand_builtin_fabs (tree, rtx, rtx);
166	static rtx expand_builtin_signbit (tree, rtx);
167	static tree fold_builtin_memcmp (location_t, tree, tree, tree);
168	static tree fold_builtin_isascii (location_t, tree);
169	static tree fold_builtin_toascii (location_t, tree);
170	static tree fold_builtin_isdigit (location_t, tree);
171	static tree fold_builtin_fabs (location_t, tree, tree);
172	static tree fold_builtin_abs (location_t, tree, tree);
173	static tree fold_builtin_unordered_cmp (location_t, tree, tree, tree, enum tree_code,
174	enum tree_code);
175	static tree fold_builtin_iseqsig (location_t, tree, tree);
176	static tree fold_builtin_varargs (location_t, tree, tree*, int);
177
178	static tree fold_builtin_strpbrk (location_t, tree, tree, tree, tree);
179	static tree fold_builtin_strspn (location_t, tree, tree, tree);
180	static tree fold_builtin_strcspn (location_t, tree, tree, tree);
181
182	static rtx expand_builtin_object_size (tree);
183	static rtx expand_builtin_memory_chk (tree, rtx, machine_mode,
184	enum built_in_function);
185	static void maybe_emit_chk_warning (tree, enum built_in_function);
186	static void maybe_emit_sprintf_chk_warning (tree, enum built_in_function);
187	static tree fold_builtin_object_size (tree, tree, enum built_in_function);
188
189	unsigned HOST_WIDE_INT target_newline;
190	unsigned HOST_WIDE_INT target_percent;
191	static unsigned HOST_WIDE_INT target_c;
192	static unsigned HOST_WIDE_INT target_s;
193	char target_percent_c[3];
194	char target_percent_s[3];
195	char target_percent_s_newline[4];
196	static tree do_mpfr_remquo (tree, tree, tree);
197	static tree do_mpfr_lgamma_r (tree, tree, tree);
198	static void expand_builtin_sync_synchronize (void);
199
200	/* Return true if NAME starts with __builtin_ or __sync_. */
201
202	static bool
203	is_builtin_name (const char *name)
204	{
205	return (startswith (str: name, prefix: "__builtin_")
206	|| startswith (str: name, prefix: "__sync_")
207	|| startswith (str: name, prefix: "__atomic_"));
208	}
209
210	/* Return true if NODE should be considered for inline expansion regardless
211	of the optimization level. This means whenever a function is invoked with
212	its "internal" name, which normally contains the prefix "__builtin". */
213
214	bool
215	called_as_built_in (tree node)
216	{
217	/* Note that we must use DECL_NAME, not DECL_ASSEMBLER_NAME_SET_P since
218	we want the name used to call the function, not the name it
219	will have. */
220	const char *name = IDENTIFIER_POINTER (DECL_NAME (node));
221	return is_builtin_name (name);
222	}
223
224	/* Compute values M and N such that M divides (address of EXP - N) and such
225	that N < M. If these numbers can be determined, store M in alignp and N in
226	*BITPOSP and return true. Otherwise return false and store BITS_PER_UNIT to
227	*alignp and any bit-offset to *bitposp.
228
229	Note that the address (and thus the alignment) computed here is based
230	on the address to which a symbol resolves, whereas DECL_ALIGN is based
231	on the address at which an object is actually located. These two
232	addresses are not always the same. For example, on ARM targets,
233	the address &foo of a Thumb function foo() has the lowest bit set,
234	whereas foo() itself starts on an even address.
235
236	If ADDR_P is true we are taking the address of the memory reference EXP
237	and thus cannot rely on the access taking place. */
238
239	bool
240	get_object_alignment_2 (tree exp, unsigned int *alignp,
241	unsigned HOST_WIDE_INT *bitposp, bool addr_p)
242	{
243	poly_int64 bitsize, bitpos;
244	tree offset;
245	machine_mode mode;
246	int unsignedp, reversep, volatilep;
247	unsigned int align = BITS_PER_UNIT;
248	bool known_alignment = false;
249
250	/* Get the innermost object and the constant (bitpos) and possibly
251	variable (offset) offset of the access. */
252	exp = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode,
253	&unsignedp, &reversep, &volatilep);
254
255	/* Extract alignment information from the innermost object and
256	possibly adjust bitpos and offset. */
257	if (TREE_CODE (exp) == FUNCTION_DECL)
258	{
259	/* Function addresses can encode extra information besides their
260	alignment. However, if TARGET_PTRMEMFUNC_VBIT_LOCATION
261	allows the low bit to be used as a virtual bit, we know
262	that the address itself must be at least 2-byte aligned. */
263	if (TARGET_PTRMEMFUNC_VBIT_LOCATION == ptrmemfunc_vbit_in_pfn)
264	align = 2 * BITS_PER_UNIT;
265	}
266	else if (TREE_CODE (exp) == LABEL_DECL)
267	;
268	else if (TREE_CODE (exp) == CONST_DECL)
269	{
270	/* The alignment of a CONST_DECL is determined by its initializer. */
271	exp = DECL_INITIAL (exp);
272	align = TYPE_ALIGN (TREE_TYPE (exp));
273	if (CONSTANT_CLASS_P (exp))
274	align = targetm.constant_alignment (exp, align);
275
276	known_alignment = true;
277	}
278	else if (DECL_P (exp))
279	{
280	align = DECL_ALIGN (exp);
281	known_alignment = true;
282	}
283	else if (TREE_CODE (exp) == INDIRECT_REF
284	|| TREE_CODE (exp) == MEM_REF
285	|| TREE_CODE (exp) == TARGET_MEM_REF)
286	{
287	tree addr = TREE_OPERAND (exp, 0);
288	unsigned ptr_align;
289	unsigned HOST_WIDE_INT ptr_bitpos;
290	unsigned HOST_WIDE_INT ptr_bitmask = ~0;
291
292	/* If the address is explicitely aligned, handle that. */
293	if (TREE_CODE (addr) == BIT_AND_EXPR
294	&& TREE_CODE (TREE_OPERAND (addr, 1)) == INTEGER_CST)
295	{
296	ptr_bitmask = TREE_INT_CST_LOW (TREE_OPERAND (addr, 1));
297	ptr_bitmask *= BITS_PER_UNIT;
298	align = least_bit_hwi (x: ptr_bitmask);
299	addr = TREE_OPERAND (addr, 0);
300	}
301
302	known_alignment
303	= get_pointer_alignment_1 (addr, &ptr_align, &ptr_bitpos);
304	align = MAX (ptr_align, align);
305
306	/* Re-apply explicit alignment to the bitpos. */
307	ptr_bitpos &= ptr_bitmask;
308
309	/* The alignment of the pointer operand in a TARGET_MEM_REF
310	has to take the variable offset parts into account. */
311	if (TREE_CODE (exp) == TARGET_MEM_REF)
312	{
313	if (TMR_INDEX (exp))
314	{
315	unsigned HOST_WIDE_INT step = 1;
316	if (TMR_STEP (exp))
317	step = TREE_INT_CST_LOW (TMR_STEP (exp));
318	align = MIN (align, least_bit_hwi (step) * BITS_PER_UNIT);
319	}
320	if (TMR_INDEX2 (exp))
321	align = BITS_PER_UNIT;
322	known_alignment = false;
323	}
324
325	/* When EXP is an actual memory reference then we can use
326	TYPE_ALIGN of a pointer indirection to derive alignment.
327	Do so only if get_pointer_alignment_1 did not reveal absolute
328	alignment knowledge and if using that alignment would
329	improve the situation. */
330	unsigned int talign;
331	if (!addr_p && !known_alignment
332	&& (talign = min_align_of_type (TREE_TYPE (exp)) * BITS_PER_UNIT)
333	&& talign > align)
334	align = talign;
335	else
336	{
337	/* Else adjust bitpos accordingly. */
338	bitpos += ptr_bitpos;
339	if (TREE_CODE (exp) == MEM_REF
340	|| TREE_CODE (exp) == TARGET_MEM_REF)
341	bitpos += mem_ref_offset (exp).force_shwi () * BITS_PER_UNIT;
342	}
343	}
344	else if (TREE_CODE (exp) == STRING_CST)
345	{
346	/* STRING_CST are the only constant objects we allow to be not
347	wrapped inside a CONST_DECL. */
348	align = TYPE_ALIGN (TREE_TYPE (exp));
349	if (CONSTANT_CLASS_P (exp))
350	align = targetm.constant_alignment (exp, align);
351
352	known_alignment = true;
353	}
354
355	/* If there is a non-constant offset part extract the maximum
356	alignment that can prevail. */
357	if (offset)
358	{
359	unsigned int trailing_zeros = tree_ctz (offset);
360	if (trailing_zeros < HOST_BITS_PER_INT)
361	{
362	unsigned int inner = (1U << trailing_zeros) * BITS_PER_UNIT;
363	if (inner)
364	align = MIN (align, inner);
365	}
366	}
367
368	/* Account for the alignment of runtime coefficients, so that the constant
369	bitpos is guaranteed to be accurate. */
370	unsigned int alt_align = ::known_alignment (a: bitpos - bitpos.coeffs[0]);
371	if (alt_align != 0 && alt_align < align)
372	{
373	align = alt_align;
374	known_alignment = false;
375	}
376
377	*alignp = align;
378	*bitposp = bitpos.coeffs[0] & (align - 1);
379	return known_alignment;
380	}
381
382	/* For a memory reference expression EXP compute values M and N such that M
383	divides (&EXP - N) and such that N < M. If these numbers can be determined,
384	store M in alignp and N in *BITPOSP and return true. Otherwise return false
385	and store BITS_PER_UNIT to *alignp and any bit-offset to *bitposp. */
386
387	bool
388	get_object_alignment_1 (tree exp, unsigned int *alignp,
389	unsigned HOST_WIDE_INT *bitposp)
390	{
391	/* Strip a WITH_SIZE_EXPR, get_inner_reference doesn't know how to deal
392	with it. */
393	if (TREE_CODE (exp) == WITH_SIZE_EXPR)
394	exp = TREE_OPERAND (exp, 0);
395	return get_object_alignment_2 (exp, alignp, bitposp, addr_p: false);
396	}
397
398	/* Return the alignment in bits of EXP, an object. */
399
400	unsigned int
401	get_object_alignment (tree exp)
402	{
403	unsigned HOST_WIDE_INT bitpos = 0;
404	unsigned int align;
405
406	get_object_alignment_1 (exp, alignp: &align, bitposp: &bitpos);
407
408	/* align and bitpos now specify known low bits of the pointer.
409	ptr & (align - 1) == bitpos. */
410
411	if (bitpos != 0)
412	align = least_bit_hwi (x: bitpos);
413	return align;
414	}
415
416	/* For a pointer valued expression EXP compute values M and N such that M
417	divides (EXP - N) and such that N < M. If these numbers can be determined,
418	store M in alignp and N in *BITPOSP and return true. Return false if
419	the results are just a conservative approximation.
420
421	If EXP is not a pointer, false is returned too. */
422
423	bool
424	get_pointer_alignment_1 (tree exp, unsigned int *alignp,
425	unsigned HOST_WIDE_INT *bitposp)
426	{
427	STRIP_NOPS (exp);
428
429	if (TREE_CODE (exp) == ADDR_EXPR)
430	return get_object_alignment_2 (TREE_OPERAND (exp, 0),
431	alignp, bitposp, addr_p: true);
432	else if (TREE_CODE (exp) == POINTER_PLUS_EXPR)
433	{
434	unsigned int align;
435	unsigned HOST_WIDE_INT bitpos;
436	bool res = get_pointer_alignment_1 (TREE_OPERAND (exp, 0),
437	alignp: &align, bitposp: &bitpos);
438	if (TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
439	bitpos += TREE_INT_CST_LOW (TREE_OPERAND (exp, 1)) * BITS_PER_UNIT;
440	else
441	{
442	unsigned int trailing_zeros = tree_ctz (TREE_OPERAND (exp, 1));
443	if (trailing_zeros < HOST_BITS_PER_INT)
444	{
445	unsigned int inner = (1U << trailing_zeros) * BITS_PER_UNIT;
446	if (inner)
447	align = MIN (align, inner);
448	}
449	}
450	*alignp = align;
451	*bitposp = bitpos & (align - 1);
452	return res;
453	}
454	else if (TREE_CODE (exp) == SSA_NAME
455	&& POINTER_TYPE_P (TREE_TYPE (exp)))
456	{
457	unsigned int ptr_align, ptr_misalign;
458	struct ptr_info_def *pi = SSA_NAME_PTR_INFO (exp);
459
460	if (pi && get_ptr_info_alignment (pi, &ptr_align, &ptr_misalign))
461	{
462	*bitposp = ptr_misalign * BITS_PER_UNIT;
463	*alignp = ptr_align * BITS_PER_UNIT;
464	/* Make sure to return a sensible alignment when the multiplication
465	by BITS_PER_UNIT overflowed. */
466	if (*alignp == 0)
467	*alignp = 1u << (HOST_BITS_PER_INT - 1);
468	/* We cannot really tell whether this result is an approximation. */
469	return false;
470	}
471	else
472	{
473	*bitposp = 0;
474	*alignp = BITS_PER_UNIT;
475	return false;
476	}
477	}
478	else if (TREE_CODE (exp) == INTEGER_CST)
479	{
480	*alignp = BIGGEST_ALIGNMENT;
481	*bitposp = ((TREE_INT_CST_LOW (exp) * BITS_PER_UNIT)
482	& (BIGGEST_ALIGNMENT - 1));
483	return true;
484	}
485
486	*bitposp = 0;
487	*alignp = BITS_PER_UNIT;
488	return false;
489	}
490
491	/* Return the alignment in bits of EXP, a pointer valued expression.
492	The alignment returned is, by default, the alignment of the thing that
493	EXP points to. If it is not a POINTER_TYPE, 0 is returned.
494
495	Otherwise, look at the expression to see if we can do better, i.e., if the
496	expression is actually pointing at an object whose alignment is tighter. */
497
498	unsigned int
499	get_pointer_alignment (tree exp)
500	{
501	unsigned HOST_WIDE_INT bitpos = 0;
502	unsigned int align;
503
504	get_pointer_alignment_1 (exp, alignp: &align, bitposp: &bitpos);
505
506	/* align and bitpos now specify known low bits of the pointer.
507	ptr & (align - 1) == bitpos. */
508
509	if (bitpos != 0)
510	align = least_bit_hwi (x: bitpos);
511
512	return align;
513	}
514
515	/* Return the number of leading non-zero elements in the sequence
516	[ PTR, PTR + MAXELTS ) where each element's size is ELTSIZE bytes.
517	ELTSIZE must be a power of 2 less than 8. Used by c_strlen. */
518
519	unsigned
520	string_length (const void *ptr, unsigned eltsize, unsigned maxelts)
521	{
522	gcc_checking_assert (eltsize == 1 || eltsize == 2 || eltsize == 4);
523
524	unsigned n;
525
526	if (eltsize == 1)
527	{
528	/* Optimize the common case of plain char. */
529	for (n = 0; n < maxelts; n++)
530	{
531	const char *elt = (const char*) ptr + n;
532	if (!*elt)
533	break;
534	}
535	}
536	else
537	{
538	for (n = 0; n < maxelts; n++)
539	{
540	const char *elt = (const char*) ptr + n * eltsize;
541	if (!memcmp (s1: elt, s2: "\0\0\0\0", n: eltsize))
542	break;
543	}
544	}
545	return n;
546	}
547
548	/* Compute the length of a null-terminated character string or wide
549	character string handling character sizes of 1, 2, and 4 bytes.
550	TREE_STRING_LENGTH is not the right way because it evaluates to
551	the size of the character array in bytes (as opposed to characters)
552	and because it can contain a zero byte in the middle.
553
554	ONLY_VALUE should be nonzero if the result is not going to be emitted
555	into the instruction stream and zero if it is going to be expanded.
556	E.g. with i++ ? "foo" : "bar", if ONLY_VALUE is nonzero, constant 3
557	is returned, otherwise NULL, since
558	len = c_strlen (ARG, 1); if (len) expand_expr (len, ...); would not
559	evaluate the side-effects.
560
561	If ONLY_VALUE is two then we do not emit warnings about out-of-bound
562	accesses. Note that this implies the result is not going to be emitted
563	into the instruction stream.
564
565	Additional information about the string accessed may be recorded
566	in DATA. For example, if ARG references an unterminated string,
567	then the declaration will be stored in the DECL field. If the
568	length of the unterminated string can be determined, it'll be
569	stored in the LEN field. Note this length could well be different
570	than what a C strlen call would return.
571
572	ELTSIZE is 1 for normal single byte character strings, and 2 or
573	4 for wide characer strings. ELTSIZE is by default 1.
574
575	The value returned is of type `ssizetype'. */
576
577	tree
578	c_strlen (tree arg, int only_value, c_strlen_data *data, unsigned eltsize)
579	{
580	/* If we were not passed a DATA pointer, then get one to a local
581	structure. That avoids having to check DATA for NULL before
582	each time we want to use it. */
583	c_strlen_data local_strlen_data = { };
584	if (!data)
585	data = &local_strlen_data;
586
587	gcc_checking_assert (eltsize == 1 || eltsize == 2 || eltsize == 4);
588
589	tree src = STRIP_NOPS (arg);
590	if (TREE_CODE (src) == COND_EXPR
591	&& (only_value || !TREE_SIDE_EFFECTS (TREE_OPERAND (src, 0))))
592	{
593	tree len1, len2;
594
595	len1 = c_strlen (TREE_OPERAND (src, 1), only_value, data, eltsize);
596	len2 = c_strlen (TREE_OPERAND (src, 2), only_value, data, eltsize);
597	if (tree_int_cst_equal (len1, len2))
598	return len1;
599	}
600
601	if (TREE_CODE (src) == COMPOUND_EXPR
602	&& (only_value || !TREE_SIDE_EFFECTS (TREE_OPERAND (src, 0))))
603	return c_strlen (TREE_OPERAND (src, 1), only_value, data, eltsize);
604
605	location_t loc = EXPR_LOC_OR_LOC (src, input_location);
606
607	/* Offset from the beginning of the string in bytes. */
608	tree byteoff;
609	tree memsize;
610	tree decl;
611	src = string_constant (src, &byteoff, &memsize, &decl);
612	if (src == 0)
613	return NULL_TREE;
614
615	/* Determine the size of the string element. */
616	if (eltsize != tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (src)))))
617	return NULL_TREE;
618
619	/* Set MAXELTS to ARRAY_SIZE (SRC) - 1, the maximum possible
620	length of SRC. Prefer TYPE_SIZE() to TREE_STRING_LENGTH() if possible
621	in case the latter is less than the size of the array, such as when
622	SRC refers to a short string literal used to initialize a large array.
623	In that case, the elements of the array after the terminating NUL are
624	all NUL. */
625	HOST_WIDE_INT strelts = TREE_STRING_LENGTH (src);
626	strelts = strelts / eltsize;
627
628	if (!tree_fits_uhwi_p (memsize))
629	return NULL_TREE;
630
631	HOST_WIDE_INT maxelts = tree_to_uhwi (memsize) / eltsize;
632
633	/* PTR can point to the byte representation of any string type, including
634	char* and wchar_t*. */
635	const char *ptr = TREE_STRING_POINTER (src);
636
637	if (byteoff && TREE_CODE (byteoff) != INTEGER_CST)
638	{
639	/* The code below works only for single byte character types. */
640	if (eltsize != 1)
641	return NULL_TREE;
642
643	/* If the string has an internal NUL character followed by any
644	non-NUL characters (e.g., "foo\0bar"), we can't compute
645	the offset to the following NUL if we don't know where to
646	start searching for it. */
647	unsigned len = string_length (ptr, eltsize, maxelts: strelts);
648
649	/* Return when an embedded null character is found or none at all.
650	In the latter case, set the DECL/LEN field in the DATA structure
651	so that callers may examine them. */
652	if (len + 1 < strelts)
653	return NULL_TREE;
654	else if (len >= maxelts)
655	{
656	data->decl = decl;
657	data->off = byteoff;
658	data->minlen = ssize_int (len);
659	return NULL_TREE;
660	}
661
662	/* For empty strings the result should be zero. */
663	if (len == 0)
664	return ssize_int (0);
665
666	/* We don't know the starting offset, but we do know that the string
667	has no internal zero bytes. If the offset falls within the bounds
668	of the string subtract the offset from the length of the string,
669	and return that. Otherwise the length is zero. Take care to
670	use SAVE_EXPR in case the OFFSET has side-effects. */
671	tree offsave = TREE_SIDE_EFFECTS (byteoff) ? save_expr (byteoff)
672	: byteoff;
673	offsave = fold_convert_loc (loc, sizetype, offsave);
674	tree condexp = fold_build2_loc (loc, LE_EXPR, boolean_type_node, offsave,
675	size_int (len));
676	tree lenexp = fold_build2_loc (loc, MINUS_EXPR, sizetype, size_int (len),
677	offsave);
678	lenexp = fold_convert_loc (loc, ssizetype, lenexp);
679	return fold_build3_loc (loc, COND_EXPR, ssizetype, condexp, lenexp,
680	build_zero_cst (ssizetype));
681	}
682
683	/* Offset from the beginning of the string in elements. */
684	HOST_WIDE_INT eltoff;
685
686	/* We have a known offset into the string. Start searching there for
687	a null character if we can represent it as a single HOST_WIDE_INT. */
688	if (byteoff == 0)
689	eltoff = 0;
690	else if (! tree_fits_uhwi_p (byteoff) || tree_to_uhwi (byteoff) % eltsize)
691	eltoff = -1;
692	else
693	eltoff = tree_to_uhwi (byteoff) / eltsize;
694
695	/* If the offset is known to be out of bounds, warn, and call strlen at
696	runtime. */
697	if (eltoff < 0 || eltoff >= maxelts)
698	{
699	/* Suppress multiple warnings for propagated constant strings. */
700	if (only_value != 2
701	&& !warning_suppressed_p (arg, OPT_Warray_bounds_)
702	&& warning_at (loc, OPT_Warray_bounds_,
703	"offset %qwi outside bounds of constant string",
704	eltoff))
705	{
706	if (decl)
707	inform (DECL_SOURCE_LOCATION (decl), "%qE declared here", decl);
708	suppress_warning (arg, OPT_Warray_bounds_);
709	}
710	return NULL_TREE;
711	}
712
713	/* If eltoff is larger than strelts but less than maxelts the
714	string length is zero, since the excess memory will be zero. */
715	if (eltoff > strelts)
716	return ssize_int (0);
717
718	/* Use strlen to search for the first zero byte. Since any strings
719	constructed with build_string will have nulls appended, we win even
720	if we get handed something like (char[4])"abcd".
721
722	Since ELTOFF is our starting index into the string, no further
723	calculation is needed. */
724	unsigned len = string_length (ptr: ptr + eltoff * eltsize, eltsize,
725	maxelts: strelts - eltoff);
726
727	/* Don't know what to return if there was no zero termination.
728	Ideally this would turn into a gcc_checking_assert over time.
729	Set DECL/LEN so callers can examine them. */
730	if (len >= maxelts - eltoff)
731	{
732	data->decl = decl;
733	data->off = byteoff;
734	data->minlen = ssize_int (len);
735	return NULL_TREE;
736	}
737
738	return ssize_int (len);
739	}
740
741	/* Return a constant integer corresponding to target reading
742	GET_MODE_BITSIZE (MODE) bits from string constant STR. If
743	NULL_TERMINATED_P, reading stops after '\0' character, all further ones
744	are assumed to be zero, otherwise it reads as many characters
745	as needed. */
746
747	rtx
748	c_readstr (const char *str, fixed_size_mode mode,
749	bool null_terminated_p/*=true*/)
750	{
751	auto_vec<target_unit, MAX_BITSIZE_MODE_ANY_INT / BITS_PER_UNIT> bytes;
752
753	bytes.reserve (nelems: GET_MODE_SIZE (mode));
754
755	target_unit ch = 1;
756	for (unsigned int i = 0; i < GET_MODE_SIZE (mode); ++i)
757	{
758	if (ch || !null_terminated_p)
759	ch = (unsigned char) str[i];
760	bytes.quick_push (obj: ch);
761	}
762
763	return native_decode_rtx (mode, bytes, 0);
764	}
765
766	/* Cast a target constant CST to target CHAR and if that value fits into
767	host char type, return zero and put that value into variable pointed to by
768	P. */
769
770	static int
771	target_char_cast (tree cst, char *p)
772	{
773	unsigned HOST_WIDE_INT val, hostval;
774
775	if (TREE_CODE (cst) != INTEGER_CST
776	|| CHAR_TYPE_SIZE > HOST_BITS_PER_WIDE_INT)
777	return 1;
778
779	/* Do not care if it fits or not right here. */
780	val = TREE_INT_CST_LOW (cst);
781
782	if (CHAR_TYPE_SIZE < HOST_BITS_PER_WIDE_INT)
783	val &= (HOST_WIDE_INT_1U << CHAR_TYPE_SIZE) - 1;
784
785	hostval = val;
786	if (HOST_BITS_PER_CHAR < HOST_BITS_PER_WIDE_INT)
787	hostval &= (HOST_WIDE_INT_1U << HOST_BITS_PER_CHAR) - 1;
788
789	if (val != hostval)
790	return 1;
791
792	*p = hostval;
793	return 0;
794	}
795
796	/* Similar to save_expr, but assumes that arbitrary code is not executed
797	in between the multiple evaluations. In particular, we assume that a
798	non-addressable local variable will not be modified. */
799
800	static tree
801	builtin_save_expr (tree exp)
802	{
803	if (TREE_CODE (exp) == SSA_NAME
804	|| (TREE_ADDRESSABLE (exp) == 0
805	&& (TREE_CODE (exp) == PARM_DECL
806	|| (VAR_P (exp) && !TREE_STATIC (exp)))))
807	return exp;
808
809	return save_expr (exp);
810	}
811
812	/* Given TEM, a pointer to a stack frame, follow the dynamic chain COUNT
813	times to get the address of either a higher stack frame, or a return
814	address located within it (depending on FNDECL_CODE). */
815
816	static rtx
817	expand_builtin_return_addr (enum built_in_function fndecl_code, int count)
818	{
819	int i;
820	rtx tem = INITIAL_FRAME_ADDRESS_RTX;
821	if (tem == NULL_RTX)
822	{
823	/* For a zero count with __builtin_return_address, we don't care what
824	frame address we return, because target-specific definitions will
825	override us. Therefore frame pointer elimination is OK, and using
826	the soft frame pointer is OK.
827
828	For a nonzero count, or a zero count with __builtin_frame_address,
829	we require a stable offset from the current frame pointer to the
830	previous one, so we must use the hard frame pointer, and
831	we must disable frame pointer elimination. */
832	if (count == 0 && fndecl_code == BUILT_IN_RETURN_ADDRESS)
833	tem = frame_pointer_rtx;
834	else
835	{
836	tem = hard_frame_pointer_rtx;
837
838	/* Tell reload not to eliminate the frame pointer. */
839	crtl->accesses_prior_frames = 1;
840	}
841	}
842
843	if (count > 0)
844	SETUP_FRAME_ADDRESSES ();
845
846	/* On the SPARC, the return address is not in the frame, it is in a
847	register. There is no way to access it off of the current frame
848	pointer, but it can be accessed off the previous frame pointer by
849	reading the value from the register window save area. */
850	if (RETURN_ADDR_IN_PREVIOUS_FRAME && fndecl_code == BUILT_IN_RETURN_ADDRESS)
851	count--;
852
853	/* Scan back COUNT frames to the specified frame. */
854	for (i = 0; i < count; i++)
855	{
856	/* Assume the dynamic chain pointer is in the word that the
857	frame address points to, unless otherwise specified. */
858	tem = DYNAMIC_CHAIN_ADDRESS (tem);
859	tem = memory_address (Pmode, tem);
860	tem = gen_frame_mem (Pmode, tem);
861	tem = copy_to_reg (tem);
862	}
863
864	/* For __builtin_frame_address, return what we've got. But, on
865	the SPARC for example, we may have to add a bias. */
866	if (fndecl_code == BUILT_IN_FRAME_ADDRESS)
867	return FRAME_ADDR_RTX (tem);
868
869	/* For __builtin_return_address, get the return address from that frame. */
870	#ifdef RETURN_ADDR_RTX
871	tem = RETURN_ADDR_RTX (count, tem);
872	#else
873	tem = memory_address (Pmode,
874	plus_constant (Pmode, tem, GET_MODE_SIZE (Pmode)));
875	tem = gen_frame_mem (Pmode, tem);
876	#endif
877	return tem;
878	}
879
880	/* Alias set used for setjmp buffer. */
881	static alias_set_type setjmp_alias_set = -1;
882
883	/* Construct the leading half of a __builtin_setjmp call. Control will
884	return to RECEIVER_LABEL. This is also called directly by the SJLJ
885	exception handling code. */
886
887	void
888	expand_builtin_setjmp_setup (rtx buf_addr, rtx receiver_label)
889	{
890	machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
891	rtx stack_save;
892	rtx mem;
893
894	if (setjmp_alias_set == -1)
895	setjmp_alias_set = new_alias_set ();
896
897	buf_addr = convert_memory_address (Pmode, buf_addr);
898
899	buf_addr = force_reg (Pmode, force_operand (buf_addr, NULL_RTX));
900
901	/* We store the frame pointer and the address of receiver_label in
902	the buffer and use the rest of it for the stack save area, which
903	is machine-dependent. */
904
905	mem = gen_rtx_MEM (Pmode, buf_addr);
906	set_mem_alias_set (mem, setjmp_alias_set);
907	emit_move_insn (mem, hard_frame_pointer_rtx);
908
909	mem = gen_rtx_MEM (Pmode, plus_constant (Pmode, buf_addr,
910	GET_MODE_SIZE (Pmode))),
911	set_mem_alias_set (mem, setjmp_alias_set);
912
913	emit_move_insn (validize_mem (mem),
914	force_reg (Pmode, gen_rtx_LABEL_REF (Pmode, receiver_label)));
915
916	stack_save = gen_rtx_MEM (sa_mode,
917	plus_constant (Pmode, buf_addr,
918	2 * GET_MODE_SIZE (Pmode)));
919	set_mem_alias_set (stack_save, setjmp_alias_set);
920	emit_stack_save (SAVE_NONLOCAL, &stack_save);
921
922	/* If there is further processing to do, do it. */
923	if (targetm.have_builtin_setjmp_setup ())
924	emit_insn (targetm.gen_builtin_setjmp_setup (buf_addr));
925
926	/* We have a nonlocal label. */
927	cfun->has_nonlocal_label = 1;
928	}
929
930	/* Construct the trailing part of a __builtin_setjmp call. This is
931	also called directly by the SJLJ exception handling code.
932	If RECEIVER_LABEL is NULL, instead contruct a nonlocal goto handler. */
933
934	void
935	expand_builtin_setjmp_receiver (rtx receiver_label)
936	{
937	rtx chain;
938
939	/* Mark the FP as used when we get here, so we have to make sure it's
940	marked as used by this function. */
941	emit_use (hard_frame_pointer_rtx);
942
943	/* Mark the static chain as clobbered here so life information
944	doesn't get messed up for it. */
945	chain = rtx_for_static_chain (current_function_decl, true);
946	if (chain && REG_P (chain))
947	emit_clobber (chain);
948
949	if (!HARD_FRAME_POINTER_IS_ARG_POINTER && fixed_regs[ARG_POINTER_REGNUM])
950	{
951	/* If the argument pointer can be eliminated in favor of the
952	frame pointer, we don't need to restore it. We assume here
953	that if such an elimination is present, it can always be used.
954	This is the case on all known machines; if we don't make this
955	assumption, we do unnecessary saving on many machines. */
956	size_t i;
957	static const struct elims {const int from, to;} elim_regs[] = ELIMINABLE_REGS;
958
959	for (i = 0; i < ARRAY_SIZE (elim_regs); i++)
960	if (elim_regs[i].from == ARG_POINTER_REGNUM
961	&& elim_regs[i].to == HARD_FRAME_POINTER_REGNUM)
962	break;
963
964	if (i == ARRAY_SIZE (elim_regs))
965	{
966	/* Now restore our arg pointer from the address at which it
967	was saved in our stack frame. */
968	emit_move_insn (crtl->args.internal_arg_pointer,
969	copy_to_reg (get_arg_pointer_save_area ()));
970	}
971	}
972
973	if (receiver_label != NULL && targetm.have_builtin_setjmp_receiver ())
974	emit_insn (targetm.gen_builtin_setjmp_receiver (receiver_label));
975	else if (targetm.have_nonlocal_goto_receiver ())
976	emit_insn (targetm.gen_nonlocal_goto_receiver ());
977	else
978	{ /* Nothing */ }
979
980	/* We must not allow the code we just generated to be reordered by
981	scheduling. Specifically, the update of the frame pointer must
982	happen immediately, not later. */
983	emit_insn (gen_blockage ());
984	}
985
986	/* __builtin_longjmp is passed a pointer to an array of five words (not
987	all will be used on all machines). It operates similarly to the C
988	library function of the same name, but is more efficient. Much of
989	the code below is copied from the handling of non-local gotos. */
990
991	static void
992	expand_builtin_longjmp (rtx buf_addr, rtx value)
993	{
994	rtx fp, lab, stack;
995	rtx_insn *insn, *last;
996	machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
997
998	/* DRAP is needed for stack realign if longjmp is expanded to current
999	function */
1000	if (SUPPORTS_STACK_ALIGNMENT)
1001	crtl->need_drap = true;
1002
1003	if (setjmp_alias_set == -1)
1004	setjmp_alias_set = new_alias_set ();
1005
1006	buf_addr = convert_memory_address (Pmode, buf_addr);
1007
1008	buf_addr = force_reg (Pmode, buf_addr);
1009
1010	/* We require that the user must pass a second argument of 1, because
1011	that is what builtin_setjmp will return. */
1012	gcc_assert (value == const1_rtx);
1013
1014	last = get_last_insn ();
1015	if (targetm.have_builtin_longjmp ())
1016	emit_insn (targetm.gen_builtin_longjmp (buf_addr));
1017	else
1018	{
1019	fp = gen_rtx_MEM (Pmode, buf_addr);
1020	lab = gen_rtx_MEM (Pmode, plus_constant (Pmode, buf_addr,
1021	GET_MODE_SIZE (Pmode)));
1022
1023	stack = gen_rtx_MEM (sa_mode, plus_constant (Pmode, buf_addr,
1024	2 * GET_MODE_SIZE (Pmode)));
1025	set_mem_alias_set (fp, setjmp_alias_set);
1026	set_mem_alias_set (lab, setjmp_alias_set);
1027	set_mem_alias_set (stack, setjmp_alias_set);
1028
1029	/* Pick up FP, label, and SP from the block and jump. This code is
1030	from expand_goto in stmt.cc; see there for detailed comments. */
1031	if (targetm.have_nonlocal_goto ())
1032	/* We have to pass a value to the nonlocal_goto pattern that will
1033	get copied into the static_chain pointer, but it does not matter
1034	what that value is, because builtin_setjmp does not use it. */
1035	emit_insn (targetm.gen_nonlocal_goto (value, lab, stack, fp));
1036	else
1037	{
1038	emit_clobber (gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode)));
1039	emit_clobber (gen_rtx_MEM (BLKmode, hard_frame_pointer_rtx));
1040
1041	lab = copy_to_reg (lab);
1042
1043	/* Restore the frame pointer and stack pointer. We must use a
1044	temporary since the setjmp buffer may be a local. */
1045	fp = copy_to_reg (fp);
1046	emit_stack_restore (SAVE_NONLOCAL, stack);
1047
1048	/* Ensure the frame pointer move is not optimized. */
1049	emit_insn (gen_blockage ());
1050	emit_clobber (hard_frame_pointer_rtx);
1051	emit_clobber (frame_pointer_rtx);
1052	emit_move_insn (hard_frame_pointer_rtx, fp);
1053
1054	emit_use (hard_frame_pointer_rtx);
1055	emit_use (stack_pointer_rtx);
1056	emit_indirect_jump (lab);
1057	}
1058	}
1059
1060	/* Search backwards and mark the jump insn as a non-local goto.
1061	Note that this precludes the use of __builtin_longjmp to a
1062	__builtin_setjmp target in the same function. However, we've
1063	already cautioned the user that these functions are for
1064	internal exception handling use only. */
1065	for (insn = get_last_insn (); insn; insn = PREV_INSN (insn))
1066	{
1067	gcc_assert (insn != last);
1068
1069	if (JUMP_P (insn))
1070	{
1071	add_reg_note (insn, REG_NON_LOCAL_GOTO, const0_rtx);
1072	break;
1073	}
1074	else if (CALL_P (insn))
1075	break;
1076	}
1077	}
1078
1079	static inline bool
1080	more_const_call_expr_args_p (const const_call_expr_arg_iterator *iter)
1081	{
1082	return (iter->i < iter->n);
1083	}
1084
1085	/* This function validates the types of a function call argument list
1086	against a specified list of tree_codes. If the last specifier is a 0,
1087	that represents an ellipsis, otherwise the last specifier must be a
1088	VOID_TYPE. */
1089
1090	static bool
1091	validate_arglist (const_tree callexpr, ...)
1092	{
1093	enum tree_code code;
1094	bool res = 0;
1095	va_list ap;
1096	const_call_expr_arg_iterator iter;
1097	const_tree arg;
1098
1099	va_start (ap, callexpr);
1100	init_const_call_expr_arg_iterator (exp: callexpr, iter: &iter);
1101
1102	/* Get a bitmap of pointer argument numbers declared attribute nonnull. */
1103	tree fn = CALL_EXPR_FN (callexpr);
1104	bitmap argmap = get_nonnull_args (TREE_TYPE (TREE_TYPE (fn)));
1105
1106	for (unsigned argno = 1; ; ++argno)
1107	{
1108	code = (enum tree_code) va_arg (ap, int);
1109
1110	switch (code)
1111	{
1112	case 0:
1113	/* This signifies an ellipses, any further arguments are all ok. */
1114	res = true;
1115	goto end;
1116	case VOID_TYPE:
1117	/* This signifies an endlink, if no arguments remain, return
1118	true, otherwise return false. */
1119	res = !more_const_call_expr_args_p (iter: &iter);
1120	goto end;
1121	case POINTER_TYPE:
1122	/* The actual argument must be nonnull when either the whole
1123	called function has been declared nonnull, or when the formal
1124	argument corresponding to the actual argument has been. */
1125	if (argmap
1126	&& (bitmap_empty_p (map: argmap) || bitmap_bit_p (argmap, argno)))
1127	{
1128	arg = next_const_call_expr_arg (iter: &iter);
1129	if (!validate_arg (arg, code) || integer_zerop (arg))
1130	goto end;
1131	break;
1132	}
1133	/* FALLTHRU */
1134	default:
1135	/* If no parameters remain or the parameter's code does not
1136	match the specified code, return false. Otherwise continue
1137	checking any remaining arguments. */
1138	arg = next_const_call_expr_arg (iter: &iter);
1139	if (!validate_arg (arg, code))
1140	goto end;
1141	break;
1142	}
1143	}
1144
1145	/* We need gotos here since we can only have one VA_CLOSE in a
1146	function. */
1147	end: ;
1148	va_end (ap);
1149
1150	BITMAP_FREE (argmap);
1151
1152	return res;
1153	}
1154
1155	/* Expand a call to __builtin_nonlocal_goto. We're passed the target label
1156	and the address of the save area. */
1157
1158	static rtx
1159	expand_builtin_nonlocal_goto (tree exp)
1160	{
1161	tree t_label, t_save_area;
1162	rtx r_label, r_save_area, r_fp, r_sp;
1163	rtx_insn *insn;
1164
1165	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
1166	return NULL_RTX;
1167
1168	t_label = CALL_EXPR_ARG (exp, 0);
1169	t_save_area = CALL_EXPR_ARG (exp, 1);
1170
1171	r_label = expand_normal (exp: t_label);
1172	r_label = convert_memory_address (Pmode, r_label);
1173	r_save_area = expand_normal (exp: t_save_area);
1174	r_save_area = convert_memory_address (Pmode, r_save_area);
1175	/* Copy the address of the save location to a register just in case it was
1176	based on the frame pointer. */
1177	r_save_area = copy_to_reg (r_save_area);
1178	r_fp = gen_rtx_MEM (Pmode, r_save_area);
1179	r_sp = gen_rtx_MEM (STACK_SAVEAREA_MODE (SAVE_NONLOCAL),
1180	plus_constant (Pmode, r_save_area,
1181	GET_MODE_SIZE (Pmode)));
1182
1183	crtl->has_nonlocal_goto = 1;
1184
1185	/* ??? We no longer need to pass the static chain value, afaik. */
1186	if (targetm.have_nonlocal_goto ())
1187	emit_insn (targetm.gen_nonlocal_goto (const0_rtx, r_label, r_sp, r_fp));
1188	else
1189	{
1190	emit_clobber (gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode)));
1191	emit_clobber (gen_rtx_MEM (BLKmode, hard_frame_pointer_rtx));
1192
1193	r_label = copy_to_reg (r_label);
1194
1195	/* Restore the frame pointer and stack pointer. We must use a
1196	temporary since the setjmp buffer may be a local. */
1197	r_fp = copy_to_reg (r_fp);
1198	emit_stack_restore (SAVE_NONLOCAL, r_sp);
1199
1200	/* Ensure the frame pointer move is not optimized. */
1201	emit_insn (gen_blockage ());
1202	emit_clobber (hard_frame_pointer_rtx);
1203	emit_clobber (frame_pointer_rtx);
1204	emit_move_insn (hard_frame_pointer_rtx, r_fp);
1205
1206	/* USE of hard_frame_pointer_rtx added for consistency;
1207	not clear if really needed. */
1208	emit_use (hard_frame_pointer_rtx);
1209	emit_use (stack_pointer_rtx);
1210
1211	/* If the architecture is using a GP register, we must
1212	conservatively assume that the target function makes use of it.
1213	The prologue of functions with nonlocal gotos must therefore
1214	initialize the GP register to the appropriate value, and we
1215	must then make sure that this value is live at the point
1216	of the jump. (Note that this doesn't necessarily apply
1217	to targets with a nonlocal_goto pattern; they are free
1218	to implement it in their own way. Note also that this is
1219	a no-op if the GP register is a global invariant.) */
1220	unsigned regnum = PIC_OFFSET_TABLE_REGNUM;
1221	if (regnum != INVALID_REGNUM && fixed_regs[regnum])
1222	emit_use (pic_offset_table_rtx);
1223
1224	emit_indirect_jump (r_label);
1225	}
1226
1227	/* Search backwards to the jump insn and mark it as a
1228	non-local goto. */
1229	for (insn = get_last_insn (); insn; insn = PREV_INSN (insn))
1230	{
1231	if (JUMP_P (insn))
1232	{
1233	add_reg_note (insn, REG_NON_LOCAL_GOTO, const0_rtx);
1234	break;
1235	}
1236	else if (CALL_P (insn))
1237	break;
1238	}
1239
1240	return const0_rtx;
1241	}
1242
1243	/* __builtin_update_setjmp_buf is passed a pointer to an array of five words
1244	(not all will be used on all machines) that was passed to __builtin_setjmp.
1245	It updates the stack pointer in that block to the current value. This is
1246	also called directly by the SJLJ exception handling code. */
1247
1248	void
1249	expand_builtin_update_setjmp_buf (rtx buf_addr)
1250	{
1251	machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
1252	buf_addr = convert_memory_address (Pmode, buf_addr);
1253	rtx stack_save
1254	= gen_rtx_MEM (sa_mode,
1255	memory_address
1256	(sa_mode,
1257	plus_constant (Pmode, buf_addr,
1258	2 * GET_MODE_SIZE (Pmode))));
1259
1260	emit_stack_save (SAVE_NONLOCAL, &stack_save);
1261	}
1262
1263	/* Expand a call to __builtin_prefetch. For a target that does not support
1264	data prefetch, evaluate the memory address argument in case it has side
1265	effects. */
1266
1267	static void
1268	expand_builtin_prefetch (tree exp)
1269	{
1270	tree arg0, arg1, arg2;
1271	int nargs;
1272	rtx op0, op1, op2;
1273
1274	if (!validate_arglist (callexpr: exp, POINTER_TYPE, 0))
1275	return;
1276
1277	arg0 = CALL_EXPR_ARG (exp, 0);
1278
1279	/* Arguments 1 and 2 are optional; argument 1 (read/write) defaults to
1280	zero (read) and argument 2 (locality) defaults to 3 (high degree of
1281	locality). */
1282	nargs = call_expr_nargs (exp);
1283	if (nargs > 1)
1284	arg1 = CALL_EXPR_ARG (exp, 1);
1285	else
1286	arg1 = integer_zero_node;
1287	if (nargs > 2)
1288	arg2 = CALL_EXPR_ARG (exp, 2);
1289	else
1290	arg2 = integer_three_node;
1291
1292	/* Argument 0 is an address. */
1293	op0 = expand_expr (exp: arg0, NULL_RTX, Pmode, modifier: EXPAND_NORMAL);
1294
1295	/* Argument 1 (read/write flag) must be a compile-time constant int. */
1296	if (TREE_CODE (arg1) != INTEGER_CST)
1297	{
1298	error ("second argument to %<__builtin_prefetch%> must be a constant");
1299	arg1 = integer_zero_node;
1300	}
1301	op1 = expand_normal (exp: arg1);
1302	/* Argument 1 must be either zero or one. */
1303	if (INTVAL (op1) != 0 && INTVAL (op1) != 1)
1304	{
1305	warning (0, "invalid second argument to %<__builtin_prefetch%>;"
1306	" using zero");
1307	op1 = const0_rtx;
1308	}
1309
1310	/* Argument 2 (locality) must be a compile-time constant int. */
1311	if (TREE_CODE (arg2) != INTEGER_CST)
1312	{
1313	error ("third argument to %<__builtin_prefetch%> must be a constant");
1314	arg2 = integer_zero_node;
1315	}
1316	op2 = expand_normal (exp: arg2);
1317	/* Argument 2 must be 0, 1, 2, or 3. */
1318	if (INTVAL (op2) < 0 || INTVAL (op2) > 3)
1319	{
1320	warning (0, "invalid third argument to %<__builtin_prefetch%>; using zero");
1321	op2 = const0_rtx;
1322	}
1323
1324	if (targetm.have_prefetch ())
1325	{
1326	class expand_operand ops[3];
1327
1328	create_address_operand (op: &ops[0], value: op0);
1329	create_integer_operand (&ops[1], INTVAL (op1));
1330	create_integer_operand (&ops[2], INTVAL (op2));
1331	if (maybe_expand_insn (icode: targetm.code_for_prefetch, nops: 3, ops))
1332	return;
1333	}
1334
1335	/* Don't do anything with direct references to volatile memory, but
1336	generate code to handle other side effects. */
1337	if (!MEM_P (op0) && side_effects_p (op0))
1338	emit_insn (op0);
1339	}
1340
1341	/* Get a MEM rtx for expression EXP which is the address of an operand
1342	to be used in a string instruction (cmpstrsi, cpymemsi, ..). LEN is
1343	the maximum length of the block of memory that might be accessed or
1344	NULL if unknown. */
1345
1346	rtx
1347	get_memory_rtx (tree exp, tree len)
1348	{
1349	tree orig_exp = exp, base;
1350	rtx addr, mem;
1351
1352	gcc_checking_assert
1353	(ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)))));
1354
1355	/* When EXP is not resolved SAVE_EXPR, MEM_ATTRS can be still derived
1356	from its expression, for expr->a.b only <variable>.a.b is recorded. */
1357	if (TREE_CODE (exp) == SAVE_EXPR && !SAVE_EXPR_RESOLVED_P (exp))
1358	exp = TREE_OPERAND (exp, 0);
1359
1360	addr = expand_expr (exp: orig_exp, NULL_RTX, mode: ptr_mode, modifier: EXPAND_NORMAL);
1361	mem = gen_rtx_MEM (BLKmode, memory_address (BLKmode, addr));
1362
1363	/* Get an expression we can use to find the attributes to assign to MEM.
1364	First remove any nops. */
1365	while (CONVERT_EXPR_P (exp)
1366	&& POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (exp, 0))))
1367	exp = TREE_OPERAND (exp, 0);
1368
1369	/* Build a MEM_REF representing the whole accessed area as a byte blob,
1370	(as builtin stringops may alias with anything). */
1371	exp = fold_build2 (MEM_REF,
1372	build_array_type (char_type_node,
1373	build_range_type (sizetype,
1374	size_one_node, len)),
1375	exp, build_int_cst (ptr_type_node, 0));
1376
1377	/* If the MEM_REF has no acceptable address, try to get the base object
1378	from the original address we got, and build an all-aliasing
1379	unknown-sized access to that one. */
1380	if (is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
1381	set_mem_attributes (mem, exp, 0);
1382	else if (TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
1383	&& (base = get_base_address (TREE_OPERAND (TREE_OPERAND (exp, 0),
1384	0))))
1385	{
1386	unsigned int align = get_pointer_alignment (TREE_OPERAND (exp, 0));
1387	exp = build_fold_addr_expr (base);
1388	exp = fold_build2 (MEM_REF,
1389	build_array_type (char_type_node,
1390	build_range_type (sizetype,
1391	size_zero_node,
1392	NULL)),
1393	exp, build_int_cst (ptr_type_node, 0));
1394	set_mem_attributes (mem, exp, 0);
1395	/* Since we stripped parts make sure the offset is unknown and the
1396	alignment is computed from the original address. */
1397	clear_mem_offset (mem);
1398	set_mem_align (mem, align);
1399	}
1400	set_mem_alias_set (mem, 0);
1401	return mem;
1402	}
1403
1404	/* Built-in functions to perform an untyped call and return. */
1405
1406	/* Wrapper that implicitly applies a delta when getting or setting the
1407	enclosed value. */
1408	template <typename T>
1409	class delta_type
1410	{
1411	T &value; T const delta;
1412	public:
1413	delta_type (T &val, T dlt) : value (val), delta (dlt) {}
1414	operator T () const { return value + delta; }
1415	T operator = (T val) const { value = val - delta; return val; }
1416	};
1417
1418	#define saved_apply_args_size \
1419	(delta_type<int> (this_target_builtins->x_apply_args_size_plus_one, -1))
1420	#define apply_args_mode \
1421	(this_target_builtins->x_apply_args_mode)
1422	#define saved_apply_result_size \
1423	(delta_type<int> (this_target_builtins->x_apply_result_size_plus_one, -1))
1424	#define apply_result_mode \
1425	(this_target_builtins->x_apply_result_mode)
1426
1427	/* Return the size required for the block returned by __builtin_apply_args,
1428	and initialize apply_args_mode. */
1429
1430	static int
1431	apply_args_size (void)
1432	{
1433	int size = saved_apply_args_size;
1434	int align;
1435	unsigned int regno;
1436
1437	/* The values computed by this function never change. */
1438	if (size < 0)
1439	{
1440	/* The first value is the incoming arg-pointer. */
1441	size = GET_MODE_SIZE (Pmode);
1442
1443	/* The second value is the structure value address unless this is
1444	passed as an "invisible" first argument. */
1445	if (targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 0))
1446	size += GET_MODE_SIZE (Pmode);
1447
1448	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1449	if (FUNCTION_ARG_REGNO_P (regno))
1450	{
1451	fixed_size_mode mode = targetm.calls.get_raw_arg_mode (regno);
1452
1453	if (mode != VOIDmode)
1454	{
1455	align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
1456	if (size % align != 0)
1457	size = CEIL (size, align) * align;
1458	size += GET_MODE_SIZE (mode);
1459	apply_args_mode[regno] = mode;
1460	}
1461	else
1462	apply_args_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
1463	}
1464	else
1465	apply_args_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
1466
1467	saved_apply_args_size = size;
1468	}
1469	return size;
1470	}
1471
1472	/* Return the size required for the block returned by __builtin_apply,
1473	and initialize apply_result_mode. */
1474
1475	static int
1476	apply_result_size (void)
1477	{
1478	int size = saved_apply_result_size;
1479	int align, regno;
1480
1481	/* The values computed by this function never change. */
1482	if (size < 0)
1483	{
1484	size = 0;
1485
1486	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1487	if (targetm.calls.function_value_regno_p (regno))
1488	{
1489	fixed_size_mode mode = targetm.calls.get_raw_result_mode (regno);
1490
1491	if (mode != VOIDmode)
1492	{
1493	align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
1494	if (size % align != 0)
1495	size = CEIL (size, align) * align;
1496	size += GET_MODE_SIZE (mode);
1497	apply_result_mode[regno] = mode;
1498	}
1499	else
1500	apply_result_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
1501	}
1502	else
1503	apply_result_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
1504
1505	/* Allow targets that use untyped_call and untyped_return to override
1506	the size so that machine-specific information can be stored here. */
1507	#ifdef APPLY_RESULT_SIZE
1508	size = APPLY_RESULT_SIZE;
1509	#endif
1510
1511	saved_apply_result_size = size;
1512	}
1513	return size;
1514	}
1515
1516	/* Create a vector describing the result block RESULT. If SAVEP is true,
1517	the result block is used to save the values; otherwise it is used to
1518	restore the values. */
1519
1520	static rtx
1521	result_vector (int savep, rtx result)
1522	{
1523	int regno, size, align, nelts;
1524	fixed_size_mode mode;
1525	rtx reg, mem;
1526	rtx *savevec = XALLOCAVEC (rtx, FIRST_PSEUDO_REGISTER);
1527
1528	size = nelts = 0;
1529	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1530	if ((mode = apply_result_mode[regno]) != VOIDmode)
1531	{
1532	align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
1533	if (size % align != 0)
1534	size = CEIL (size, align) * align;
1535	reg = gen_rtx_REG (mode, savep ? regno : INCOMING_REGNO (regno));
1536	mem = adjust_address (result, mode, size);
1537	savevec[nelts++] = (savep
1538	? gen_rtx_SET (mem, reg)
1539	: gen_rtx_SET (reg, mem));
1540	size += GET_MODE_SIZE (mode);
1541	}
1542	return gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelts, savevec));
1543	}
1544
1545	/* Save the state required to perform an untyped call with the same
1546	arguments as were passed to the current function. */
1547
1548	static rtx
1549	expand_builtin_apply_args_1 (void)
1550	{
1551	rtx registers, tem;
1552	int size, align, regno;
1553	fixed_size_mode mode;
1554	rtx struct_incoming_value = targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 1);
1555
1556	/* Create a block where the arg-pointer, structure value address,
1557	and argument registers can be saved. */
1558	registers = assign_stack_local (BLKmode, apply_args_size (), -1);
1559
1560	/* Walk past the arg-pointer and structure value address. */
1561	size = GET_MODE_SIZE (Pmode);
1562	if (targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 0))
1563	size += GET_MODE_SIZE (Pmode);
1564
1565	/* Save each register used in calling a function to the block. */
1566	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1567	if ((mode = apply_args_mode[regno]) != VOIDmode)
1568	{
1569	align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
1570	if (size % align != 0)
1571	size = CEIL (size, align) * align;
1572
1573	tem = gen_rtx_REG (mode, INCOMING_REGNO (regno));
1574
1575	emit_move_insn (adjust_address (registers, mode, size), tem);
1576	size += GET_MODE_SIZE (mode);
1577	}
1578
1579	/* Save the arg pointer to the block. */
1580	tem = copy_to_reg (crtl->args.internal_arg_pointer);
1581	/* We need the pointer as the caller actually passed them to us, not
1582	as we might have pretended they were passed. Make sure it's a valid
1583	operand, as emit_move_insn isn't expected to handle a PLUS. */
1584	if (STACK_GROWS_DOWNWARD)
1585	tem
1586	= force_operand (plus_constant (Pmode, tem,
1587	crtl->args.pretend_args_size),
1588	NULL_RTX);
1589	emit_move_insn (adjust_address (registers, Pmode, 0), tem);
1590
1591	size = GET_MODE_SIZE (Pmode);
1592
1593	/* Save the structure value address unless this is passed as an
1594	"invisible" first argument. */
1595	if (struct_incoming_value)
1596	emit_move_insn (adjust_address (registers, Pmode, size),
1597	copy_to_reg (struct_incoming_value));
1598
1599	/* Return the address of the block. */
1600	return copy_addr_to_reg (XEXP (registers, 0));
1601	}
1602
1603	/* __builtin_apply_args returns block of memory allocated on
1604	the stack into which is stored the arg pointer, structure
1605	value address, static chain, and all the registers that might
1606	possibly be used in performing a function call. The code is
1607	moved to the start of the function so the incoming values are
1608	saved. */
1609
1610	static rtx
1611	expand_builtin_apply_args (void)
1612	{
1613	/* Don't do __builtin_apply_args more than once in a function.
1614	Save the result of the first call and reuse it. */
1615	if (apply_args_value != 0)
1616	return apply_args_value;
1617	{
1618	/* When this function is called, it means that registers must be
1619	saved on entry to this function. So we migrate the
1620	call to the first insn of this function. */
1621	rtx temp;
1622
1623	start_sequence ();
1624	temp = expand_builtin_apply_args_1 ();
1625	rtx_insn *seq = get_insns ();
1626	end_sequence ();
1627
1628	apply_args_value = temp;
1629
1630	/* Put the insns after the NOTE that starts the function.
1631	If this is inside a start_sequence, make the outer-level insn
1632	chain current, so the code is placed at the start of the
1633	function. If internal_arg_pointer is a non-virtual pseudo,
1634	it needs to be placed after the function that initializes
1635	that pseudo. */
1636	push_topmost_sequence ();
1637	if (REG_P (crtl->args.internal_arg_pointer)
1638	&& REGNO (crtl->args.internal_arg_pointer) > LAST_VIRTUAL_REGISTER)
1639	emit_insn_before (seq, parm_birth_insn);
1640	else
1641	emit_insn_before (seq, NEXT_INSN (insn: entry_of_function ()));
1642	pop_topmost_sequence ();
1643	return temp;
1644	}
1645	}
1646
1647	/* Perform an untyped call and save the state required to perform an
1648	untyped return of whatever value was returned by the given function. */
1649
1650	static rtx
1651	expand_builtin_apply (rtx function, rtx arguments, rtx argsize)
1652	{
1653	int size, align, regno;
1654	fixed_size_mode mode;
1655	rtx incoming_args, result, reg, dest, src;
1656	rtx_call_insn *call_insn;
1657	rtx old_stack_level = 0;
1658	rtx call_fusage = 0;
1659	rtx struct_value = targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 0);
1660
1661	arguments = convert_memory_address (Pmode, arguments);
1662
1663	/* Create a block where the return registers can be saved. */
1664	result = assign_stack_local (BLKmode, apply_result_size (), -1);
1665
1666	/* Fetch the arg pointer from the ARGUMENTS block. */
1667	incoming_args = gen_reg_rtx (Pmode);
1668	emit_move_insn (incoming_args, gen_rtx_MEM (Pmode, arguments));
1669	if (!STACK_GROWS_DOWNWARD)
1670	incoming_args = expand_simple_binop (Pmode, MINUS, incoming_args, argsize,
1671	incoming_args, 0, OPTAB_LIB_WIDEN);
1672
1673	/* Push a new argument block and copy the arguments. Do not allow
1674	the (potential) memcpy call below to interfere with our stack
1675	manipulations. */
1676	do_pending_stack_adjust ();
1677	NO_DEFER_POP;
1678
1679	/* Save the stack with nonlocal if available. */
1680	if (targetm.have_save_stack_nonlocal ())
1681	emit_stack_save (SAVE_NONLOCAL, &old_stack_level);
1682	else
1683	emit_stack_save (SAVE_BLOCK, &old_stack_level);
1684
1685	/* Allocate a block of memory onto the stack and copy the memory
1686	arguments to the outgoing arguments address. We can pass TRUE
1687	as the 4th argument because we just saved the stack pointer
1688	and will restore it right after the call. */
1689	allocate_dynamic_stack_space (argsize, 0, BIGGEST_ALIGNMENT, -1, true);
1690
1691	/* Set DRAP flag to true, even though allocate_dynamic_stack_space
1692	may have already set current_function_calls_alloca to true.
1693	current_function_calls_alloca won't be set if argsize is zero,
1694	so we have to guarantee need_drap is true here. */
1695	if (SUPPORTS_STACK_ALIGNMENT)
1696	crtl->need_drap = true;
1697
1698	dest = virtual_outgoing_args_rtx;
1699	if (!STACK_GROWS_DOWNWARD)
1700	{
1701	if (CONST_INT_P (argsize))
1702	dest = plus_constant (Pmode, dest, -INTVAL (argsize));
1703	else
1704	dest = gen_rtx_PLUS (Pmode, dest, negate_rtx (Pmode, argsize));
1705	}
1706	dest = gen_rtx_MEM (BLKmode, dest);
1707	set_mem_align (dest, PARM_BOUNDARY);
1708	src = gen_rtx_MEM (BLKmode, incoming_args);
1709	set_mem_align (src, PARM_BOUNDARY);
1710	emit_block_move (dest, src, argsize, BLOCK_OP_NORMAL);
1711
1712	/* Refer to the argument block. */
1713	apply_args_size ();
1714	arguments = gen_rtx_MEM (BLKmode, arguments);
1715	set_mem_align (arguments, PARM_BOUNDARY);
1716
1717	/* Walk past the arg-pointer and structure value address. */
1718	size = GET_MODE_SIZE (Pmode);
1719	if (struct_value)
1720	size += GET_MODE_SIZE (Pmode);
1721
1722	/* Restore each of the registers previously saved. Make USE insns
1723	for each of these registers for use in making the call. */
1724	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1725	if ((mode = apply_args_mode[regno]) != VOIDmode)
1726	{
1727	align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
1728	if (size % align != 0)
1729	size = CEIL (size, align) * align;
1730	reg = gen_rtx_REG (mode, regno);
1731	emit_move_insn (reg, adjust_address (arguments, mode, size));
1732	use_reg (fusage: &call_fusage, reg);
1733	size += GET_MODE_SIZE (mode);
1734	}
1735
1736	/* Restore the structure value address unless this is passed as an
1737	"invisible" first argument. */
1738	size = GET_MODE_SIZE (Pmode);
1739	if (struct_value)
1740	{
1741	rtx value = gen_reg_rtx (Pmode);
1742	emit_move_insn (value, adjust_address (arguments, Pmode, size));
1743	emit_move_insn (struct_value, value);
1744	if (REG_P (struct_value))
1745	use_reg (fusage: &call_fusage, reg: struct_value);
1746	}
1747
1748	/* All arguments and registers used for the call are set up by now! */
1749	function = prepare_call_address (NULL, function, NULL, &call_fusage, 0, 0);
1750
1751	/* Ensure address is valid. SYMBOL_REF is already valid, so no need,
1752	and we don't want to load it into a register as an optimization,
1753	because prepare_call_address already did it if it should be done. */
1754	if (GET_CODE (function) != SYMBOL_REF)
1755	function = memory_address (FUNCTION_MODE, function);
1756
1757	/* Generate the actual call instruction and save the return value. */
1758	if (targetm.have_untyped_call ())
1759	{
1760	rtx mem = gen_rtx_MEM (FUNCTION_MODE, function);
1761	rtx_insn *seq = targetm.gen_untyped_call (mem, result,
1762	result_vector (savep: 1, result));
1763	for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
1764	if (CALL_P (insn))
1765	add_reg_note (insn, REG_UNTYPED_CALL, NULL_RTX);
1766	emit_insn (seq);
1767	}
1768	else if (targetm.have_call_value ())
1769	{
1770	rtx valreg = 0;
1771
1772	/* Locate the unique return register. It is not possible to
1773	express a call that sets more than one return register using
1774	call_value; use untyped_call for that. In fact, untyped_call
1775	only needs to save the return registers in the given block. */
1776	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1777	if ((mode = apply_result_mode[regno]) != VOIDmode)
1778	{
1779	gcc_assert (!valreg); /* have_untyped_call required. */
1780
1781	valreg = gen_rtx_REG (mode, regno);
1782	}
1783
1784	emit_insn (targetm.gen_call_value (valreg,
1785	gen_rtx_MEM (FUNCTION_MODE, function),
1786	const0_rtx, NULL_RTX, const0_rtx));
1787
1788	emit_move_insn (adjust_address (result, GET_MODE (valreg), 0), valreg);
1789	}
1790	else
1791	gcc_unreachable ();
1792
1793	/* Find the CALL insn we just emitted, and attach the register usage
1794	information. */
1795	call_insn = last_call_insn ();
1796	add_function_usage_to (call_insn, call_fusage);
1797
1798	/* Restore the stack. */
1799	if (targetm.have_save_stack_nonlocal ())
1800	emit_stack_restore (SAVE_NONLOCAL, old_stack_level);
1801	else
1802	emit_stack_restore (SAVE_BLOCK, old_stack_level);
1803	fixup_args_size_notes (call_insn, get_last_insn (), 0);
1804
1805	OK_DEFER_POP;
1806
1807	/* Return the address of the result block. */
1808	result = copy_addr_to_reg (XEXP (result, 0));
1809	return convert_memory_address (ptr_mode, result);
1810	}
1811
1812	/* Perform an untyped return. */
1813
1814	static void
1815	expand_builtin_return (rtx result)
1816	{
1817	int size, align, regno;
1818	fixed_size_mode mode;
1819	rtx reg;
1820	rtx_insn *call_fusage = 0;
1821
1822	result = convert_memory_address (Pmode, result);
1823
1824	apply_result_size ();
1825	result = gen_rtx_MEM (BLKmode, result);
1826
1827	if (targetm.have_untyped_return ())
1828	{
1829	rtx vector = result_vector (savep: 0, result);
1830	emit_jump_insn (targetm.gen_untyped_return (result, vector));
1831	emit_barrier ();
1832	return;
1833	}
1834
1835	/* Restore the return value and note that each value is used. */
1836	size = 0;
1837	for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1838	if ((mode = apply_result_mode[regno]) != VOIDmode)
1839	{
1840	align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
1841	if (size % align != 0)
1842	size = CEIL (size, align) * align;
1843	reg = gen_rtx_REG (mode, INCOMING_REGNO (regno));
1844	emit_move_insn (reg, adjust_address (result, mode, size));
1845
1846	push_to_sequence (call_fusage);
1847	emit_use (reg);
1848	call_fusage = get_insns ();
1849	end_sequence ();
1850	size += GET_MODE_SIZE (mode);
1851	}
1852
1853	/* Put the USE insns before the return. */
1854	emit_insn (call_fusage);
1855
1856	/* Return whatever values was restored by jumping directly to the end
1857	of the function. */
1858	expand_naked_return ();
1859	}
1860
1861	/* Used by expand_builtin_classify_type and fold_builtin_classify_type. */
1862
1863	int
1864	type_to_class (tree type)
1865	{
1866	switch (TREE_CODE (type))
1867	{
1868	case VOID_TYPE: return void_type_class;
1869	case INTEGER_TYPE: return integer_type_class;
1870	case ENUMERAL_TYPE: return enumeral_type_class;
1871	case BOOLEAN_TYPE: return boolean_type_class;
1872	case POINTER_TYPE: return pointer_type_class;
1873	case REFERENCE_TYPE: return reference_type_class;
1874	case OFFSET_TYPE: return offset_type_class;
1875	case REAL_TYPE: return real_type_class;
1876	case COMPLEX_TYPE: return complex_type_class;
1877	case FUNCTION_TYPE: return function_type_class;
1878	case METHOD_TYPE: return method_type_class;
1879	case RECORD_TYPE: return record_type_class;
1880	case UNION_TYPE:
1881	case QUAL_UNION_TYPE: return union_type_class;
1882	case ARRAY_TYPE: return (TYPE_STRING_FLAG (type)
1883	? string_type_class : array_type_class);
1884	case LANG_TYPE: return lang_type_class;
1885	case OPAQUE_TYPE: return opaque_type_class;
1886	case BITINT_TYPE: return bitint_type_class;
1887	case VECTOR_TYPE: return vector_type_class;
1888	default: return no_type_class;
1889	}
1890	}
1891
1892	/* Expand a call EXP to __builtin_classify_type. */
1893
1894	static rtx
1895	expand_builtin_classify_type (tree exp)
1896	{
1897	if (call_expr_nargs (exp))
1898	return GEN_INT (type_to_class (TREE_TYPE (CALL_EXPR_ARG (exp, 0))));
1899	return GEN_INT (no_type_class);
1900	}
1901
1902	/* This helper macro, meant to be used in mathfn_built_in below, determines
1903	which among a set of builtin math functions is appropriate for a given type
1904	mode. The `F' (float) and `L' (long double) are automatically generated
1905	from the 'double' case. If a function supports the _Float<N> and _Float<N>X
1906	types, there are additional types that are considered with 'F32', 'F64',
1907	'F128', etc. suffixes. */
1908	#define CASE_MATHFN(MATHFN) \
1909	CASE_CFN_##MATHFN: \
1910	fcode = BUILT_IN_##MATHFN; fcodef = BUILT_IN_##MATHFN##F ; \
1911	fcodel = BUILT_IN_##MATHFN##L ; break;
1912	/* Similar to the above, but also add support for the _Float<N> and _Float<N>X
1913	types. */
1914	#define CASE_MATHFN_FLOATN(MATHFN) \
1915	CASE_CFN_##MATHFN: \
1916	fcode = BUILT_IN_##MATHFN; fcodef = BUILT_IN_##MATHFN##F ; \
1917	fcodel = BUILT_IN_##MATHFN##L ; fcodef16 = BUILT_IN_##MATHFN##F16 ; \
1918	fcodef32 = BUILT_IN_##MATHFN##F32; fcodef64 = BUILT_IN_##MATHFN##F64 ; \
1919	fcodef128 = BUILT_IN_##MATHFN##F128 ; fcodef32x = BUILT_IN_##MATHFN##F32X ; \
1920	fcodef64x = BUILT_IN_##MATHFN##F64X ; fcodef128x = BUILT_IN_##MATHFN##F128X ;\
1921	break;
1922	/* Similar to above, but appends _R after any F/L suffix. */
1923	#define CASE_MATHFN_REENT(MATHFN) \
1924	case CFN_BUILT_IN_##MATHFN##_R: \
1925	case CFN_BUILT_IN_##MATHFN##F_R: \
1926	case CFN_BUILT_IN_##MATHFN##L_R: \
1927	fcode = BUILT_IN_##MATHFN##_R; fcodef = BUILT_IN_##MATHFN##F_R ; \
1928	fcodel = BUILT_IN_##MATHFN##L_R ; break;
1929
1930	/* Return a function equivalent to FN but operating on floating-point
1931	values of type TYPE, or END_BUILTINS if no such function exists.
1932	This is purely an operation on function codes; it does not guarantee
1933	that the target actually has an implementation of the function. */
1934
1935	static built_in_function
1936	mathfn_built_in_2 (tree type, combined_fn fn)
1937	{
1938	tree mtype;
1939	built_in_function fcode, fcodef, fcodel;
1940	built_in_function fcodef16 = END_BUILTINS;
1941	built_in_function fcodef32 = END_BUILTINS;
1942	built_in_function fcodef64 = END_BUILTINS;
1943	built_in_function fcodef128 = END_BUILTINS;
1944	built_in_function fcodef32x = END_BUILTINS;
1945	built_in_function fcodef64x = END_BUILTINS;
1946	built_in_function fcodef128x = END_BUILTINS;
1947
1948	/* If <math.h> has been included somehow, HUGE_VAL and NAN definitions
1949	break the uses below. */
1950	#undef HUGE_VAL
1951	#undef NAN
1952
1953	switch (fn)
1954	{
1955	#define SEQ_OF_CASE_MATHFN \
1956	CASE_MATHFN_FLOATN (ACOS) \
1957	CASE_MATHFN_FLOATN (ACOSH) \
1958	CASE_MATHFN_FLOATN (ASIN) \
1959	CASE_MATHFN_FLOATN (ASINH) \
1960	CASE_MATHFN_FLOATN (ATAN) \
1961	CASE_MATHFN_FLOATN (ATAN2) \
1962	CASE_MATHFN_FLOATN (ATANH) \
1963	CASE_MATHFN_FLOATN (CBRT) \
1964	CASE_MATHFN_FLOATN (CEIL) \
1965	CASE_MATHFN (CEXPI) \
1966	CASE_MATHFN_FLOATN (COPYSIGN) \
1967	CASE_MATHFN_FLOATN (COS) \
1968	CASE_MATHFN_FLOATN (COSH) \
1969	CASE_MATHFN (DREM) \
1970	CASE_MATHFN_FLOATN (ERF) \
1971	CASE_MATHFN_FLOATN (ERFC) \
1972	CASE_MATHFN_FLOATN (EXP) \
1973	CASE_MATHFN (EXP10) \
1974	CASE_MATHFN_FLOATN (EXP2) \
1975	CASE_MATHFN_FLOATN (EXPM1) \
1976	CASE_MATHFN_FLOATN (FABS) \
1977	CASE_MATHFN_FLOATN (FDIM) \
1978	CASE_MATHFN_FLOATN (FLOOR) \
1979	CASE_MATHFN_FLOATN (FMA) \
1980	CASE_MATHFN_FLOATN (FMAX) \
1981	CASE_MATHFN_FLOATN (FMIN) \
1982	CASE_MATHFN_FLOATN (FMOD) \
1983	CASE_MATHFN_FLOATN (FREXP) \
1984	CASE_MATHFN (GAMMA) \
1985	CASE_MATHFN_REENT (GAMMA) /* GAMMA_R */ \
1986	CASE_MATHFN_FLOATN (HUGE_VAL) \
1987	CASE_MATHFN_FLOATN (HYPOT) \
1988	CASE_MATHFN_FLOATN (ILOGB) \
1989	CASE_MATHFN (ICEIL) \
1990	CASE_MATHFN (IFLOOR) \
1991	CASE_MATHFN_FLOATN (INF) \
1992	CASE_MATHFN (IRINT) \
1993	CASE_MATHFN (IROUND) \
1994	CASE_MATHFN (ISINF) \
1995	CASE_MATHFN (J0) \
1996	CASE_MATHFN (J1) \
1997	CASE_MATHFN (JN) \
1998	CASE_MATHFN (LCEIL) \
1999	CASE_MATHFN_FLOATN (LDEXP) \
2000	CASE_MATHFN (LFLOOR) \
2001	CASE_MATHFN_FLOATN (LGAMMA) \
2002	CASE_MATHFN_REENT (LGAMMA) /* LGAMMA_R */ \
2003	CASE_MATHFN (LLCEIL) \
2004	CASE_MATHFN (LLFLOOR) \
2005	CASE_MATHFN_FLOATN (LLRINT) \
2006	CASE_MATHFN_FLOATN (LLROUND) \
2007	CASE_MATHFN_FLOATN (LOG) \
2008	CASE_MATHFN_FLOATN (LOG10) \
2009	CASE_MATHFN_FLOATN (LOG1P) \
2010	CASE_MATHFN_FLOATN (LOG2) \
2011	CASE_MATHFN_FLOATN (LOGB) \
2012	CASE_MATHFN_FLOATN (LRINT) \
2013	CASE_MATHFN_FLOATN (LROUND) \
2014	CASE_MATHFN_FLOATN (MODF) \
2015	CASE_MATHFN_FLOATN (NAN) \
2016	CASE_MATHFN_FLOATN (NANS) \
2017	CASE_MATHFN_FLOATN (NEARBYINT) \
2018	CASE_MATHFN_FLOATN (NEXTAFTER) \
2019	CASE_MATHFN (NEXTTOWARD) \
2020	CASE_MATHFN_FLOATN (POW) \
2021	CASE_MATHFN (POWI) \
2022	CASE_MATHFN (POW10) \
2023	CASE_MATHFN_FLOATN (REMAINDER) \
2024	CASE_MATHFN_FLOATN (REMQUO) \
2025	CASE_MATHFN_FLOATN (RINT) \
2026	CASE_MATHFN_FLOATN (ROUND) \
2027	CASE_MATHFN_FLOATN (ROUNDEVEN) \
2028	CASE_MATHFN (SCALB) \
2029	CASE_MATHFN_FLOATN (SCALBLN) \
2030	CASE_MATHFN_FLOATN (SCALBN) \
2031	CASE_MATHFN (SIGNBIT) \
2032	CASE_MATHFN (SIGNIFICAND) \
2033	CASE_MATHFN_FLOATN (SIN) \
2034	CASE_MATHFN (SINCOS) \
2035	CASE_MATHFN_FLOATN (SINH) \
2036	CASE_MATHFN_FLOATN (SQRT) \
2037	CASE_MATHFN_FLOATN (TAN) \
2038	CASE_MATHFN_FLOATN (TANH) \
2039	CASE_MATHFN_FLOATN (TGAMMA) \
2040	CASE_MATHFN_FLOATN (TRUNC) \
2041	CASE_MATHFN (Y0) \
2042	CASE_MATHFN (Y1) \
2043	CASE_MATHFN (YN)
2044
2045	SEQ_OF_CASE_MATHFN
2046
2047	default:
2048	return END_BUILTINS;
2049	}
2050
2051	mtype = TYPE_MAIN_VARIANT (type);
2052	if (mtype == double_type_node)
2053	return fcode;
2054	else if (mtype == float_type_node)
2055	return fcodef;
2056	else if (mtype == long_double_type_node)
2057	return fcodel;
2058	else if (mtype == float16_type_node)
2059	return fcodef16;
2060	else if (mtype == float32_type_node)
2061	return fcodef32;
2062	else if (mtype == float64_type_node)
2063	return fcodef64;
2064	else if (mtype == float128_type_node)
2065	return fcodef128;
2066	else if (mtype == float32x_type_node)
2067	return fcodef32x;
2068	else if (mtype == float64x_type_node)
2069	return fcodef64x;
2070	else if (mtype == float128x_type_node)
2071	return fcodef128x;
2072	else
2073	return END_BUILTINS;
2074	}
2075
2076	#undef CASE_MATHFN
2077	#undef CASE_MATHFN_FLOATN
2078	#undef CASE_MATHFN_REENT
2079
2080	/* Return mathematic function equivalent to FN but operating directly on TYPE,
2081	if available. If IMPLICIT_P is true use the implicit builtin declaration,
2082	otherwise use the explicit declaration. If we can't do the conversion,
2083	return null. */
2084
2085	static tree
2086	mathfn_built_in_1 (tree type, combined_fn fn, bool implicit_p)
2087	{
2088	built_in_function fcode2 = mathfn_built_in_2 (type, fn);
2089	if (fcode2 == END_BUILTINS)
2090	return NULL_TREE;
2091
2092	if (implicit_p && !builtin_decl_implicit_p (fncode: fcode2))
2093	return NULL_TREE;
2094
2095	return builtin_decl_explicit (fncode: fcode2);
2096	}
2097
2098	/* Like mathfn_built_in_1, but always use the implicit array. */
2099
2100	tree
2101	mathfn_built_in (tree type, combined_fn fn)
2102	{
2103	return mathfn_built_in_1 (type, fn, /*implicit=*/ implicit_p: 1);
2104	}
2105
2106	/* Like mathfn_built_in_1, but always use the explicit array. */
2107
2108	tree
2109	mathfn_built_in_explicit (tree type, combined_fn fn)
2110	{
2111	return mathfn_built_in_1 (type, fn, /*implicit=*/ implicit_p: 0);
2112	}
2113
2114	/* Like mathfn_built_in_1, but take a built_in_function and
2115	always use the implicit array. */
2116
2117	tree
2118	mathfn_built_in (tree type, enum built_in_function fn)
2119	{
2120	return mathfn_built_in_1 (type, fn: as_combined_fn (fn), /*implicit=*/ implicit_p: 1);
2121	}
2122
2123	/* Return the type associated with a built in function, i.e., the one
2124	to be passed to mathfn_built_in to get the type-specific
2125	function. */
2126
2127	tree
2128	mathfn_built_in_type (combined_fn fn)
2129	{
2130	#define CASE_MATHFN(MATHFN) \
2131	case CFN_BUILT_IN_##MATHFN: \
2132	return double_type_node; \
2133	case CFN_BUILT_IN_##MATHFN##F: \
2134	return float_type_node; \
2135	case CFN_BUILT_IN_##MATHFN##L: \
2136	return long_double_type_node;
2137
2138	#define CASE_MATHFN_FLOATN(MATHFN) \
2139	CASE_MATHFN(MATHFN) \
2140	case CFN_BUILT_IN_##MATHFN##F16: \
2141	return float16_type_node; \
2142	case CFN_BUILT_IN_##MATHFN##F32: \
2143	return float32_type_node; \
2144	case CFN_BUILT_IN_##MATHFN##F64: \
2145	return float64_type_node; \
2146	case CFN_BUILT_IN_##MATHFN##F128: \
2147	return float128_type_node; \
2148	case CFN_BUILT_IN_##MATHFN##F32X: \
2149	return float32x_type_node; \
2150	case CFN_BUILT_IN_##MATHFN##F64X: \
2151	return float64x_type_node; \
2152	case CFN_BUILT_IN_##MATHFN##F128X: \
2153	return float128x_type_node;
2154
2155	/* Similar to above, but appends _R after any F/L suffix. */
2156	#define CASE_MATHFN_REENT(MATHFN) \
2157	case CFN_BUILT_IN_##MATHFN##_R: \
2158	return double_type_node; \
2159	case CFN_BUILT_IN_##MATHFN##F_R: \
2160	return float_type_node; \
2161	case CFN_BUILT_IN_##MATHFN##L_R: \
2162	return long_double_type_node;
2163
2164	switch (fn)
2165	{
2166	SEQ_OF_CASE_MATHFN
2167
2168	default:
2169	return NULL_TREE;
2170	}
2171
2172	#undef CASE_MATHFN
2173	#undef CASE_MATHFN_FLOATN
2174	#undef CASE_MATHFN_REENT
2175	#undef SEQ_OF_CASE_MATHFN
2176	}
2177
2178	/* Check whether there is an internal function associated with function FN
2179	and return type RETURN_TYPE. Return the function if so, otherwise return
2180	IFN_LAST.
2181
2182	Note that this function only tests whether the function is defined in
2183	internals.def, not whether it is actually available on the target. */
2184
2185	static internal_fn
2186	associated_internal_fn (built_in_function fn, tree return_type)
2187	{
2188	switch (fn)
2189	{
2190	#define DEF_INTERNAL_FLT_FN(NAME, FLAGS, OPTAB, TYPE) \
2191	CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME;
2192	#define DEF_INTERNAL_FLT_FLOATN_FN(NAME, FLAGS, OPTAB, TYPE) \
2193	CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME; \
2194	CASE_FLT_FN_FLOATN_NX (BUILT_IN_##NAME): return IFN_##NAME;
2195	#define DEF_INTERNAL_INT_FN(NAME, FLAGS, OPTAB, TYPE) \
2196	CASE_INT_FN (BUILT_IN_##NAME): return IFN_##NAME;
2197	#include "internal-fn.def"
2198
2199	CASE_FLT_FN (BUILT_IN_POW10):
2200	return IFN_EXP10;
2201
2202	CASE_FLT_FN (BUILT_IN_DREM):
2203	return IFN_REMAINDER;
2204
2205	CASE_FLT_FN (BUILT_IN_SCALBN):
2206	CASE_FLT_FN (BUILT_IN_SCALBLN):
2207	if (REAL_MODE_FORMAT (TYPE_MODE (return_type))->b == 2)
2208	return IFN_LDEXP;
2209	return IFN_LAST;
2210
2211	default:
2212	return IFN_LAST;
2213	}
2214	}
2215
2216	/* If BUILT_IN_NORMAL function FNDECL has an associated internal function,
2217	return its code, otherwise return IFN_LAST. Note that this function
2218	only tests whether the function is defined in internals.def, not whether
2219	it is actually available on the target. */
2220
2221	internal_fn
2222	associated_internal_fn (tree fndecl)
2223	{
2224	gcc_checking_assert (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL);
2225	return associated_internal_fn (fn: DECL_FUNCTION_CODE (decl: fndecl),
2226	TREE_TYPE (TREE_TYPE (fndecl)));
2227	}
2228
2229	/* Check whether there is an internal function associated with function CFN
2230	and return type RETURN_TYPE. Return the function if so, otherwise return
2231	IFN_LAST.
2232
2233	Note that this function only tests whether the function is defined in
2234	internals.def, not whether it is actually available on the target. */
2235
2236	internal_fn
2237	associated_internal_fn (combined_fn cfn, tree return_type)
2238	{
2239	if (internal_fn_p (code: cfn))
2240	return as_internal_fn (code: cfn);
2241	return associated_internal_fn (fn: as_builtin_fn (code: cfn), return_type);
2242	}
2243
2244	/* If CALL is a call to a BUILT_IN_NORMAL function that could be replaced
2245	on the current target by a call to an internal function, return the
2246	code of that internal function, otherwise return IFN_LAST. The caller
2247	is responsible for ensuring that any side-effects of the built-in
2248	call are dealt with correctly. E.g. if CALL sets errno, the caller
2249	must decide that the errno result isn't needed or make it available
2250	in some other way. */
2251
2252	internal_fn
2253	replacement_internal_fn (gcall *call)
2254	{
2255	if (gimple_call_builtin_p (call, BUILT_IN_NORMAL))
2256	{
2257	internal_fn ifn = associated_internal_fn (fndecl: gimple_call_fndecl (gs: call));
2258	if (ifn != IFN_LAST)
2259	{
2260	tree_pair types = direct_internal_fn_types (ifn, call);
2261	optimization_type opt_type = bb_optimization_type (gimple_bb (g: call));
2262	if (direct_internal_fn_supported_p (ifn, types, opt_type))
2263	return ifn;
2264	}
2265	}
2266	return IFN_LAST;
2267	}
2268
2269	/* Expand a call to the builtin trinary math functions (fma).
2270	Return NULL_RTX if a normal call should be emitted rather than expanding the
2271	function in-line. EXP is the expression that is a call to the builtin
2272	function; if convenient, the result should be placed in TARGET.
2273	SUBTARGET may be used as the target for computing one of EXP's
2274	operands. */
2275
2276	static rtx
2277	expand_builtin_mathfn_ternary (tree exp, rtx target, rtx subtarget)
2278	{
2279	optab builtin_optab;
2280	rtx op0, op1, op2, result;
2281	rtx_insn *insns;
2282	tree fndecl = get_callee_fndecl (exp);
2283	tree arg0, arg1, arg2;
2284	machine_mode mode;
2285
2286	if (!validate_arglist (callexpr: exp, REAL_TYPE, REAL_TYPE, REAL_TYPE, VOID_TYPE))
2287	return NULL_RTX;
2288
2289	arg0 = CALL_EXPR_ARG (exp, 0);
2290	arg1 = CALL_EXPR_ARG (exp, 1);
2291	arg2 = CALL_EXPR_ARG (exp, 2);
2292
2293	switch (DECL_FUNCTION_CODE (decl: fndecl))
2294	{
2295	CASE_FLT_FN (BUILT_IN_FMA):
2296	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
2297	builtin_optab = fma_optab; break;
2298	default:
2299	gcc_unreachable ();
2300	}
2301
2302	/* Make a suitable register to place result in. */
2303	mode = TYPE_MODE (TREE_TYPE (exp));
2304
2305	/* Before working hard, check whether the instruction is available. */
2306	if (optab_handler (op: builtin_optab, mode) == CODE_FOR_nothing)
2307	return NULL_RTX;
2308
2309	result = gen_reg_rtx (mode);
2310
2311	/* Always stabilize the argument list. */
2312	CALL_EXPR_ARG (exp, 0) = arg0 = builtin_save_expr (exp: arg0);
2313	CALL_EXPR_ARG (exp, 1) = arg1 = builtin_save_expr (exp: arg1);
2314	CALL_EXPR_ARG (exp, 2) = arg2 = builtin_save_expr (exp: arg2);
2315
2316	op0 = expand_expr (exp: arg0, target: subtarget, VOIDmode, modifier: EXPAND_NORMAL);
2317	op1 = expand_normal (exp: arg1);
2318	op2 = expand_normal (exp: arg2);
2319
2320	start_sequence ();
2321
2322	/* Compute into RESULT.
2323	Set RESULT to wherever the result comes back. */
2324	result = expand_ternary_op (mode, ternary_optab: builtin_optab, op0, op1, op2,
2325	target: result, unsignedp: 0);
2326
2327	/* If we were unable to expand via the builtin, stop the sequence
2328	(without outputting the insns) and call to the library function
2329	with the stabilized argument list. */
2330	if (result == 0)
2331	{
2332	end_sequence ();
2333	return expand_call (exp, target, target == const0_rtx);
2334	}
2335
2336	/* Output the entire sequence. */
2337	insns = get_insns ();
2338	end_sequence ();
2339	emit_insn (insns);
2340
2341	return result;
2342	}
2343
2344	/* Expand a call to the builtin sin and cos math functions.
2345	Return NULL_RTX if a normal call should be emitted rather than expanding the
2346	function in-line. EXP is the expression that is a call to the builtin
2347	function; if convenient, the result should be placed in TARGET.
2348	SUBTARGET may be used as the target for computing one of EXP's
2349	operands. */
2350
2351	static rtx
2352	expand_builtin_mathfn_3 (tree exp, rtx target, rtx subtarget)
2353	{
2354	optab builtin_optab;
2355	rtx op0;
2356	rtx_insn *insns;
2357	tree fndecl = get_callee_fndecl (exp);
2358	machine_mode mode;
2359	tree arg;
2360
2361	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
2362	return NULL_RTX;
2363
2364	arg = CALL_EXPR_ARG (exp, 0);
2365
2366	switch (DECL_FUNCTION_CODE (decl: fndecl))
2367	{
2368	CASE_FLT_FN (BUILT_IN_SIN):
2369	CASE_FLT_FN (BUILT_IN_COS):
2370	builtin_optab = sincos_optab; break;
2371	default:
2372	gcc_unreachable ();
2373	}
2374
2375	/* Make a suitable register to place result in. */
2376	mode = TYPE_MODE (TREE_TYPE (exp));
2377
2378	/* Check if sincos insn is available, otherwise fallback
2379	to sin or cos insn. */
2380	if (optab_handler (op: builtin_optab, mode) == CODE_FOR_nothing)
2381	switch (DECL_FUNCTION_CODE (decl: fndecl))
2382	{
2383	CASE_FLT_FN (BUILT_IN_SIN):
2384	builtin_optab = sin_optab; break;
2385	CASE_FLT_FN (BUILT_IN_COS):
2386	builtin_optab = cos_optab; break;
2387	default:
2388	gcc_unreachable ();
2389	}
2390
2391	/* Before working hard, check whether the instruction is available. */
2392	if (optab_handler (op: builtin_optab, mode) != CODE_FOR_nothing)
2393	{
2394	rtx result = gen_reg_rtx (mode);
2395
2396	/* Wrap the computation of the argument in a SAVE_EXPR, as we may
2397	need to expand the argument again. This way, we will not perform
2398	side-effects more the once. */
2399	CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (exp: arg);
2400
2401	op0 = expand_expr (exp: arg, target: subtarget, VOIDmode, modifier: EXPAND_NORMAL);
2402
2403	start_sequence ();
2404
2405	/* Compute into RESULT.
2406	Set RESULT to wherever the result comes back. */
2407	if (builtin_optab == sincos_optab)
2408	{
2409	int ok;
2410
2411	switch (DECL_FUNCTION_CODE (decl: fndecl))
2412	{
2413	CASE_FLT_FN (BUILT_IN_SIN):
2414	ok = expand_twoval_unop (builtin_optab, op0, 0, result, 0);
2415	break;
2416	CASE_FLT_FN (BUILT_IN_COS):
2417	ok = expand_twoval_unop (builtin_optab, op0, result, 0, 0);
2418	break;
2419	default:
2420	gcc_unreachable ();
2421	}
2422	gcc_assert (ok);
2423	}
2424	else
2425	result = expand_unop (mode, builtin_optab, op0, result, 0);
2426
2427	if (result != 0)
2428	{
2429	/* Output the entire sequence. */
2430	insns = get_insns ();
2431	end_sequence ();
2432	emit_insn (insns);
2433	return result;
2434	}
2435
2436	/* If we were unable to expand via the builtin, stop the sequence
2437	(without outputting the insns) and call to the library function
2438	with the stabilized argument list. */
2439	end_sequence ();
2440	}
2441
2442	return expand_call (exp, target, target == const0_rtx);
2443	}
2444
2445	/* Given an interclass math builtin decl FNDECL and it's argument ARG
2446	return an RTL instruction code that implements the functionality.
2447	If that isn't possible or available return CODE_FOR_nothing. */
2448
2449	static enum insn_code
2450	interclass_mathfn_icode (tree arg, tree fndecl)
2451	{
2452	bool errno_set = false;
2453	optab builtin_optab = unknown_optab;
2454	machine_mode mode;
2455
2456	switch (DECL_FUNCTION_CODE (decl: fndecl))
2457	{
2458	CASE_FLT_FN (BUILT_IN_ILOGB):
2459	errno_set = true; builtin_optab = ilogb_optab; break;
2460	CASE_FLT_FN (BUILT_IN_ISINF):
2461	builtin_optab = isinf_optab; break;
2462	case BUILT_IN_ISNORMAL:
2463	case BUILT_IN_ISFINITE:
2464	CASE_FLT_FN (BUILT_IN_FINITE):
2465	case BUILT_IN_FINITED32:
2466	case BUILT_IN_FINITED64:
2467	case BUILT_IN_FINITED128:
2468	case BUILT_IN_ISINFD32:
2469	case BUILT_IN_ISINFD64:
2470	case BUILT_IN_ISINFD128:
2471	/* These builtins have no optabs (yet). */
2472	break;
2473	default:
2474	gcc_unreachable ();
2475	}
2476
2477	/* There's no easy way to detect the case we need to set EDOM. */
2478	if (flag_errno_math && errno_set)
2479	return CODE_FOR_nothing;
2480
2481	/* Optab mode depends on the mode of the input argument. */
2482	mode = TYPE_MODE (TREE_TYPE (arg));
2483
2484	if (builtin_optab)
2485	return optab_handler (op: builtin_optab, mode);
2486	return CODE_FOR_nothing;
2487	}
2488
2489	/* Expand a call to one of the builtin math functions that operate on
2490	floating point argument and output an integer result (ilogb, isinf,
2491	isnan, etc).
2492	Return 0 if a normal call should be emitted rather than expanding the
2493	function in-line. EXP is the expression that is a call to the builtin
2494	function; if convenient, the result should be placed in TARGET. */
2495
2496	static rtx
2497	expand_builtin_interclass_mathfn (tree exp, rtx target)
2498	{
2499	enum insn_code icode = CODE_FOR_nothing;
2500	rtx op0;
2501	tree fndecl = get_callee_fndecl (exp);
2502	machine_mode mode;
2503	tree arg;
2504
2505	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
2506	return NULL_RTX;
2507
2508	arg = CALL_EXPR_ARG (exp, 0);
2509	icode = interclass_mathfn_icode (arg, fndecl);
2510	mode = TYPE_MODE (TREE_TYPE (arg));
2511
2512	if (icode != CODE_FOR_nothing)
2513	{
2514	class expand_operand ops[1];
2515	rtx_insn *last = get_last_insn ();
2516	tree orig_arg = arg;
2517
2518	/* Wrap the computation of the argument in a SAVE_EXPR, as we may
2519	need to expand the argument again. This way, we will not perform
2520	side-effects more the once. */
2521	CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (exp: arg);
2522
2523	op0 = expand_expr (exp: arg, NULL_RTX, VOIDmode, modifier: EXPAND_NORMAL);
2524
2525	if (mode != GET_MODE (op0))
2526	op0 = convert_to_mode (mode, op0, 0);
2527
2528	create_output_operand (op: &ops[0], x: target, TYPE_MODE (TREE_TYPE (exp)));
2529	if (maybe_legitimize_operands (icode, opno: 0, nops: 1, ops)
2530	&& maybe_emit_unop_insn (icode, ops[0].value, op0, UNKNOWN))
2531	return ops[0].value;
2532
2533	delete_insns_since (last);
2534	CALL_EXPR_ARG (exp, 0) = orig_arg;
2535	}
2536
2537	return NULL_RTX;
2538	}
2539
2540	/* Expand a call to the builtin sincos math function.
2541	Return NULL_RTX if a normal call should be emitted rather than expanding the
2542	function in-line. EXP is the expression that is a call to the builtin
2543	function. */
2544
2545	static rtx
2546	expand_builtin_sincos (tree exp)
2547	{
2548	rtx op0, op1, op2, target1, target2;
2549	machine_mode mode;
2550	tree arg, sinp, cosp;
2551	int result;
2552	location_t loc = EXPR_LOCATION (exp);
2553	tree alias_type, alias_off;
2554
2555	if (!validate_arglist (callexpr: exp, REAL_TYPE,
2556	POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
2557	return NULL_RTX;
2558
2559	arg = CALL_EXPR_ARG (exp, 0);
2560	sinp = CALL_EXPR_ARG (exp, 1);
2561	cosp = CALL_EXPR_ARG (exp, 2);
2562
2563	/* Make a suitable register to place result in. */
2564	mode = TYPE_MODE (TREE_TYPE (arg));
2565
2566	/* Check if sincos insn is available, otherwise emit the call. */
2567	if (optab_handler (op: sincos_optab, mode) == CODE_FOR_nothing)
2568	return NULL_RTX;
2569
2570	target1 = gen_reg_rtx (mode);
2571	target2 = gen_reg_rtx (mode);
2572
2573	op0 = expand_normal (exp: arg);
2574	alias_type = build_pointer_type_for_mode (TREE_TYPE (arg), ptr_mode, true);
2575	alias_off = build_int_cst (alias_type, 0);
2576	op1 = expand_normal (exp: fold_build2_loc (loc, MEM_REF, TREE_TYPE (arg),
2577	sinp, alias_off));
2578	op2 = expand_normal (exp: fold_build2_loc (loc, MEM_REF, TREE_TYPE (arg),
2579	cosp, alias_off));
2580
2581	/* Compute into target1 and target2.
2582	Set TARGET to wherever the result comes back. */
2583	result = expand_twoval_unop (sincos_optab, op0, target2, target1, 0);
2584	gcc_assert (result);
2585
2586	/* Move target1 and target2 to the memory locations indicated
2587	by op1 and op2. */
2588	emit_move_insn (op1, target1);
2589	emit_move_insn (op2, target2);
2590
2591	return const0_rtx;
2592	}
2593
2594	/* Expand call EXP to the fegetround builtin (from C99 fenv.h), returning the
2595	result and setting it in TARGET. Otherwise return NULL_RTX on failure. */
2596	static rtx
2597	expand_builtin_fegetround (tree exp, rtx target, machine_mode target_mode)
2598	{
2599	if (!validate_arglist (callexpr: exp, VOID_TYPE))
2600	return NULL_RTX;
2601
2602	insn_code icode = direct_optab_handler (op: fegetround_optab, SImode);
2603	if (icode == CODE_FOR_nothing)
2604	return NULL_RTX;
2605
2606	if (target == 0
2607	|| GET_MODE (target) != target_mode
2608	|| !(*insn_data[icode].operand[0].predicate) (target, target_mode))
2609	target = gen_reg_rtx (target_mode);
2610
2611	rtx pat = GEN_FCN (icode) (target);
2612	if (!pat)
2613	return NULL_RTX;
2614	emit_insn (pat);
2615
2616	return target;
2617	}
2618
2619	/* Expand call EXP to either feclearexcept or feraiseexcept builtins (from C99
2620	fenv.h), returning the result and setting it in TARGET. Otherwise return
2621	NULL_RTX on failure. */
2622	static rtx
2623	expand_builtin_feclear_feraise_except (tree exp, rtx target,
2624	machine_mode target_mode, optab op_optab)
2625	{
2626	if (!validate_arglist (callexpr: exp, INTEGER_TYPE, VOID_TYPE))
2627	return NULL_RTX;
2628	rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
2629
2630	insn_code icode = direct_optab_handler (op: op_optab, SImode);
2631	if (icode == CODE_FOR_nothing)
2632	return NULL_RTX;
2633
2634	if (!(*insn_data[icode].operand[1].predicate) (op0, GET_MODE (op0)))
2635	return NULL_RTX;
2636
2637	if (target == 0
2638	|| GET_MODE (target) != target_mode
2639	|| !(*insn_data[icode].operand[0].predicate) (target, target_mode))
2640	target = gen_reg_rtx (target_mode);
2641
2642	rtx pat = GEN_FCN (icode) (target, op0);
2643	if (!pat)
2644	return NULL_RTX;
2645	emit_insn (pat);
2646
2647	return target;
2648	}
2649
2650	/* Expand a call to the internal cexpi builtin to the sincos math function.
2651	EXP is the expression that is a call to the builtin function; if convenient,
2652	the result should be placed in TARGET. */
2653
2654	static rtx
2655	expand_builtin_cexpi (tree exp, rtx target)
2656	{
2657	tree fndecl = get_callee_fndecl (exp);
2658	tree arg, type;
2659	machine_mode mode;
2660	rtx op0, op1, op2;
2661	location_t loc = EXPR_LOCATION (exp);
2662
2663	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
2664	return NULL_RTX;
2665
2666	arg = CALL_EXPR_ARG (exp, 0);
2667	type = TREE_TYPE (arg);
2668	mode = TYPE_MODE (TREE_TYPE (arg));
2669
2670	/* Try expanding via a sincos optab, fall back to emitting a libcall
2671	to sincos or cexp. We are sure we have sincos or cexp because cexpi
2672	is only generated from sincos, cexp or if we have either of them. */
2673	if (optab_handler (op: sincos_optab, mode) != CODE_FOR_nothing)
2674	{
2675	op1 = gen_reg_rtx (mode);
2676	op2 = gen_reg_rtx (mode);
2677
2678	op0 = expand_expr (exp: arg, NULL_RTX, VOIDmode, modifier: EXPAND_NORMAL);
2679
2680	/* Compute into op1 and op2. */
2681	expand_twoval_unop (sincos_optab, op0, op2, op1, 0);
2682	}
2683	else if (targetm.libc_has_function (function_sincos, type))
2684	{
2685	tree call, fn = NULL_TREE;
2686	tree top1, top2;
2687	rtx op1a, op2a;
2688
2689	if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPIF)
2690	fn = builtin_decl_explicit (fncode: BUILT_IN_SINCOSF);
2691	else if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPI)
2692	fn = builtin_decl_explicit (fncode: BUILT_IN_SINCOS);
2693	else if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPIL)
2694	fn = builtin_decl_explicit (fncode: BUILT_IN_SINCOSL);
2695	else
2696	gcc_unreachable ();
2697
2698	op1 = assign_temp (TREE_TYPE (arg), 1, 1);
2699	op2 = assign_temp (TREE_TYPE (arg), 1, 1);
2700	op1a = copy_addr_to_reg (XEXP (op1, 0));
2701	op2a = copy_addr_to_reg (XEXP (op2, 0));
2702	top1 = make_tree (build_pointer_type (TREE_TYPE (arg)), op1a);
2703	top2 = make_tree (build_pointer_type (TREE_TYPE (arg)), op2a);
2704
2705	/* Make sure not to fold the sincos call again. */
2706	call = build1 (ADDR_EXPR, build_pointer_type (TREE_TYPE (fn)), fn);
2707	expand_normal (exp: build_call_nary (TREE_TYPE (TREE_TYPE (fn)),
2708	call, 3, arg, top1, top2));
2709	}
2710	else
2711	{
2712	tree call, fn = NULL_TREE, narg;
2713	tree ctype = build_complex_type (type);
2714
2715	if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPIF)
2716	fn = builtin_decl_explicit (fncode: BUILT_IN_CEXPF);
2717	else if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPI)
2718	fn = builtin_decl_explicit (fncode: BUILT_IN_CEXP);
2719	else if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPIL)
2720	fn = builtin_decl_explicit (fncode: BUILT_IN_CEXPL);
2721	else
2722	gcc_unreachable ();
2723
2724	/* If we don't have a decl for cexp create one. This is the
2725	friendliest fallback if the user calls __builtin_cexpi
2726	without full target C99 function support. */
2727	if (fn == NULL_TREE)
2728	{
2729	tree fntype;
2730	const char *name = NULL;
2731
2732	if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPIF)
2733	name = "cexpf";
2734	else if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPI)
2735	name = "cexp";
2736	else if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_CEXPIL)
2737	name = "cexpl";
2738
2739	fntype = build_function_type_list (ctype, ctype, NULL_TREE);
2740	fn = build_fn_decl (name, fntype);
2741	}
2742
2743	narg = fold_build2_loc (loc, COMPLEX_EXPR, ctype,
2744	build_real (type, dconst0), arg);
2745
2746	/* Make sure not to fold the cexp call again. */
2747	call = build1 (ADDR_EXPR, build_pointer_type (TREE_TYPE (fn)), fn);
2748	return expand_expr (exp: build_call_nary (ctype, call, 1, narg),
2749	target, VOIDmode, modifier: EXPAND_NORMAL);
2750	}
2751
2752	/* Now build the proper return type. */
2753	return expand_expr (exp: build2 (COMPLEX_EXPR, build_complex_type (type),
2754	make_tree (TREE_TYPE (arg), op2),
2755	make_tree (TREE_TYPE (arg), op1)),
2756	target, VOIDmode, modifier: EXPAND_NORMAL);
2757	}
2758
2759	/* Conveniently construct a function call expression. FNDECL names the
2760	function to be called, N is the number of arguments, and the "..."
2761	parameters are the argument expressions. Unlike build_call_exr
2762	this doesn't fold the call, hence it will always return a CALL_EXPR. */
2763
2764	static tree
2765	build_call_nofold_loc (location_t loc, tree fndecl, int n, ...)
2766	{
2767	va_list ap;
2768	tree fntype = TREE_TYPE (fndecl);
2769	tree fn = build1 (ADDR_EXPR, build_pointer_type (fntype), fndecl);
2770
2771	va_start (ap, n);
2772	fn = build_call_valist (TREE_TYPE (fntype), fn, n, ap);
2773	va_end (ap);
2774	SET_EXPR_LOCATION (fn, loc);
2775	return fn;
2776	}
2777
2778	/* Expand the __builtin_issignaling builtin. This needs to handle
2779	all floating point formats that do support NaNs (for those that
2780	don't it just sets target to 0). */
2781
2782	static rtx
2783	expand_builtin_issignaling (tree exp, rtx target)
2784	{
2785	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
2786	return NULL_RTX;
2787
2788	tree arg = CALL_EXPR_ARG (exp, 0);
2789	scalar_float_mode fmode = SCALAR_FLOAT_TYPE_MODE (TREE_TYPE (arg));
2790	const struct real_format *fmt = REAL_MODE_FORMAT (fmode);
2791
2792	/* Expand the argument yielding a RTX expression. */
2793	rtx temp = expand_normal (exp: arg);
2794
2795	/* If mode doesn't support NaN, always return 0.
2796	Don't use !HONOR_SNANS (fmode) here, so there is some possibility of
2797	__builtin_issignaling working without -fsignaling-nans. Especially
2798	when -fno-signaling-nans is the default.
2799	On the other side, MODE_HAS_NANS (fmode) is unnecessary, with
2800	-ffinite-math-only even __builtin_isnan or __builtin_fpclassify
2801	fold to 0 or non-NaN/Inf classification. */
2802	if (!HONOR_NANS (fmode))
2803	{
2804	emit_move_insn (target, const0_rtx);
2805	return target;
2806	}
2807
2808	/* Check if the back end provides an insn that handles issignaling for the
2809	argument's mode. */
2810	enum insn_code icode = optab_handler (op: issignaling_optab, mode: fmode);
2811	if (icode != CODE_FOR_nothing)
2812	{
2813	rtx_insn *last = get_last_insn ();
2814	rtx this_target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
2815	if (maybe_emit_unop_insn (icode, this_target, temp, UNKNOWN))
2816	return this_target;
2817	delete_insns_since (last);
2818	}
2819
2820	if (DECIMAL_FLOAT_MODE_P (fmode))
2821	{
2822	scalar_int_mode imode;
2823	rtx hi;
2824	switch (fmt->ieee_bits)
2825	{
2826	case 32:
2827	case 64:
2828	imode = int_mode_for_mode (fmode).require ();
2829	temp = gen_lowpart (imode, temp);
2830	break;
2831	case 128:
2832	imode = int_mode_for_size (size: 64, limit: 1).require ();
2833	hi = NULL_RTX;
2834	/* For decimal128, TImode support isn't always there and even when
2835	it is, working on the DImode high part is usually better. */
2836	if (!MEM_P (temp))
2837	{
2838	if (rtx t = simplify_gen_subreg (outermode: imode, op: temp, innermode: fmode,
2839	byte: subreg_highpart_offset (outermode: imode,
2840	innermode: fmode)))
2841	hi = t;
2842	else
2843	{
2844	scalar_int_mode imode2;
2845	if (int_mode_for_mode (fmode).exists (mode: &imode2))
2846	{
2847	rtx temp2 = gen_lowpart (imode2, temp);
2848	poly_uint64 off = subreg_highpart_offset (outermode: imode, innermode: imode2);
2849	if (rtx t = simplify_gen_subreg (outermode: imode, op: temp2,
2850	innermode: imode2, byte: off))
2851	hi = t;
2852	}
2853	}
2854	if (!hi)
2855	{
2856	rtx mem = assign_stack_temp (fmode, GET_MODE_SIZE (mode: fmode));
2857	emit_move_insn (mem, temp);
2858	temp = mem;
2859	}
2860	}
2861	if (!hi)
2862	{
2863	poly_int64 offset
2864	= subreg_highpart_offset (outermode: imode, GET_MODE (temp));
2865	hi = adjust_address (temp, imode, offset);
2866	}
2867	temp = hi;
2868	break;
2869	default:
2870	gcc_unreachable ();
2871	}
2872	/* In all of decimal{32,64,128}, there is MSB sign bit and sNaN
2873	have 6 bits below it all set. */
2874	rtx val
2875	= GEN_INT (HOST_WIDE_INT_C (0x3f) << (GET_MODE_BITSIZE (imode) - 7));
2876	temp = expand_binop (imode, and_optab, temp, val,
2877	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2878	temp = emit_store_flag_force (target, EQ, temp, val, imode, 1, 1);
2879	return temp;
2880	}
2881
2882	/* Only PDP11 has these defined differently but doesn't support NaNs. */
2883	gcc_assert (FLOAT_WORDS_BIG_ENDIAN == WORDS_BIG_ENDIAN);
2884	gcc_assert (fmt->signbit_ro > 0 && fmt->b == 2);
2885	gcc_assert (MODE_COMPOSITE_P (fmode)
2886	|| (fmt->pnan == fmt->p
2887	&& fmt->signbit_ro == fmt->signbit_rw));
2888
2889	switch (fmt->p)
2890	{
2891	case 106: /* IBM double double */
2892	/* For IBM double double, recurse on the most significant double. */
2893	gcc_assert (MODE_COMPOSITE_P (fmode));
2894	temp = convert_modes (DFmode, oldmode: fmode, x: temp, unsignedp: 0);
2895	fmode = DFmode;
2896	fmt = REAL_MODE_FORMAT (DFmode);
2897	/* FALLTHRU */
2898	case 8: /* bfloat */
2899	case 11: /* IEEE half */
2900	case 24: /* IEEE single */
2901	case 53: /* IEEE double or Intel extended with rounding to double */
2902	if (fmt->p == 53 && fmt->signbit_ro == 79)
2903	goto extended;
2904	{
2905	scalar_int_mode imode = int_mode_for_mode (fmode).require ();
2906	temp = gen_lowpart (imode, temp);
2907	rtx val = GEN_INT ((HOST_WIDE_INT_M1U << (fmt->p - 2))
2908	& ~(HOST_WIDE_INT_M1U << fmt->signbit_ro));
2909	if (fmt->qnan_msb_set)
2910	{
2911	rtx mask = GEN_INT (~(HOST_WIDE_INT_M1U << fmt->signbit_ro));
2912	rtx bit = GEN_INT (HOST_WIDE_INT_1U << (fmt->p - 2));
2913	/* For non-MIPS/PA IEEE single/double/half or bfloat, expand to:
2914	((temp ^ bit) & mask) > val. */
2915	temp = expand_binop (imode, xor_optab, temp, bit,
2916	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2917	temp = expand_binop (imode, and_optab, temp, mask,
2918	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2919	temp = emit_store_flag_force (target, GTU, temp, val, imode,
2920	1, 1);
2921	}
2922	else
2923	{
2924	/* For MIPS/PA IEEE single/double, expand to:
2925	(temp & val) == val. */
2926	temp = expand_binop (imode, and_optab, temp, val,
2927	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2928	temp = emit_store_flag_force (target, EQ, temp, val, imode,
2929	1, 1);
2930	}
2931	}
2932	break;
2933	case 113: /* IEEE quad */
2934	{
2935	rtx hi = NULL_RTX, lo = NULL_RTX;
2936	scalar_int_mode imode = int_mode_for_size (size: 64, limit: 1).require ();
2937	/* For IEEE quad, TImode support isn't always there and even when
2938	it is, working on DImode parts is usually better. */
2939	if (!MEM_P (temp))
2940	{
2941	hi = simplify_gen_subreg (outermode: imode, op: temp, innermode: fmode,
2942	byte: subreg_highpart_offset (outermode: imode, innermode: fmode));
2943	lo = simplify_gen_subreg (outermode: imode, op: temp, innermode: fmode,
2944	byte: subreg_lowpart_offset (outermode: imode, innermode: fmode));
2945	if (!hi || !lo)
2946	{
2947	scalar_int_mode imode2;
2948	if (int_mode_for_mode (fmode).exists (mode: &imode2))
2949	{
2950	rtx temp2 = gen_lowpart (imode2, temp);
2951	hi = simplify_gen_subreg (outermode: imode, op: temp2, innermode: imode2,
2952	byte: subreg_highpart_offset (outermode: imode,
2953	innermode: imode2));
2954	lo = simplify_gen_subreg (outermode: imode, op: temp2, innermode: imode2,
2955	byte: subreg_lowpart_offset (outermode: imode,
2956	innermode: imode2));
2957	}
2958	}
2959	if (!hi || !lo)
2960	{
2961	rtx mem = assign_stack_temp (fmode, GET_MODE_SIZE (mode: fmode));
2962	emit_move_insn (mem, temp);
2963	temp = mem;
2964	}
2965	}
2966	if (!hi || !lo)
2967	{
2968	poly_int64 offset
2969	= subreg_highpart_offset (outermode: imode, GET_MODE (temp));
2970	hi = adjust_address (temp, imode, offset);
2971	offset = subreg_lowpart_offset (outermode: imode, GET_MODE (temp));
2972	lo = adjust_address (temp, imode, offset);
2973	}
2974	rtx val = GEN_INT ((HOST_WIDE_INT_M1U << (fmt->p - 2 - 64))
2975	& ~(HOST_WIDE_INT_M1U << (fmt->signbit_ro - 64)));
2976	if (fmt->qnan_msb_set)
2977	{
2978	rtx mask = GEN_INT (~(HOST_WIDE_INT_M1U << (fmt->signbit_ro
2979	- 64)));
2980	rtx bit = GEN_INT (HOST_WIDE_INT_1U << (fmt->p - 2 - 64));
2981	/* For non-MIPS/PA IEEE quad, expand to:
2982	(((hi ^ bit) | ((lo | -lo) >> 63)) & mask) > val. */
2983	rtx nlo = expand_unop (imode, neg_optab, lo, NULL_RTX, 0);
2984	lo = expand_binop (imode, ior_optab, lo, nlo,
2985	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2986	lo = expand_shift (RSHIFT_EXPR, imode, lo, 63, NULL_RTX, 1);
2987	temp = expand_binop (imode, xor_optab, hi, bit,
2988	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2989	temp = expand_binop (imode, ior_optab, temp, lo,
2990	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2991	temp = expand_binop (imode, and_optab, temp, mask,
2992	NULL_RTX, 1, OPTAB_LIB_WIDEN);
2993	temp = emit_store_flag_force (target, GTU, temp, val, imode,
2994	1, 1);
2995	}
2996	else
2997	{
2998	/* For MIPS/PA IEEE quad, expand to:
2999	(hi & val) == val. */
3000	temp = expand_binop (imode, and_optab, hi, val,
3001	NULL_RTX, 1, OPTAB_LIB_WIDEN);
3002	temp = emit_store_flag_force (target, EQ, temp, val, imode,
3003	1, 1);
3004	}
3005	}
3006	break;
3007	case 64: /* Intel or Motorola extended */
3008	extended:
3009	{
3010	rtx ex, hi, lo;
3011	scalar_int_mode imode = int_mode_for_size (size: 32, limit: 1).require ();
3012	scalar_int_mode iemode = int_mode_for_size (size: 16, limit: 1).require ();
3013	if (!MEM_P (temp))
3014	{
3015	rtx mem = assign_stack_temp (fmode, GET_MODE_SIZE (mode: fmode));
3016	emit_move_insn (mem, temp);
3017	temp = mem;
3018	}
3019	if (fmt->signbit_ro == 95)
3020	{
3021	/* Motorola, always big endian, with 16-bit gap in between
3022	16-bit sign+exponent and 64-bit mantissa. */
3023	ex = adjust_address (temp, iemode, 0);
3024	hi = adjust_address (temp, imode, 4);
3025	lo = adjust_address (temp, imode, 8);
3026	}
3027	else if (!WORDS_BIG_ENDIAN)
3028	{
3029	/* Intel little endian, 64-bit mantissa followed by 16-bit
3030	sign+exponent and then either 16 or 48 bits of gap. */
3031	ex = adjust_address (temp, iemode, 8);
3032	hi = adjust_address (temp, imode, 4);
3033	lo = adjust_address (temp, imode, 0);
3034	}
3035	else
3036	{
3037	/* Big endian Itanium. */
3038	ex = adjust_address (temp, iemode, 0);
3039	hi = adjust_address (temp, imode, 2);
3040	lo = adjust_address (temp, imode, 6);
3041	}
3042	rtx val = GEN_INT (HOST_WIDE_INT_M1U << 30);
3043	gcc_assert (fmt->qnan_msb_set);
3044	rtx mask = GEN_INT (0x7fff);
3045	rtx bit = GEN_INT (HOST_WIDE_INT_1U << 30);
3046	/* For Intel/Motorola extended format, expand to:
3047	(ex & mask) == mask && ((hi ^ bit) | ((lo | -lo) >> 31)) > val. */
3048	rtx nlo = expand_unop (imode, neg_optab, lo, NULL_RTX, 0);
3049	lo = expand_binop (imode, ior_optab, lo, nlo,
3050	NULL_RTX, 1, OPTAB_LIB_WIDEN);
3051	lo = expand_shift (RSHIFT_EXPR, imode, lo, 31, NULL_RTX, 1);
3052	temp = expand_binop (imode, xor_optab, hi, bit,
3053	NULL_RTX, 1, OPTAB_LIB_WIDEN);
3054	temp = expand_binop (imode, ior_optab, temp, lo,
3055	NULL_RTX, 1, OPTAB_LIB_WIDEN);
3056	temp = emit_store_flag_force (target, GTU, temp, val, imode, 1, 1);
3057	ex = expand_binop (iemode, and_optab, ex, mask,
3058	NULL_RTX, 1, OPTAB_LIB_WIDEN);
3059	ex = emit_store_flag_force (gen_reg_rtx (GET_MODE (temp)), EQ,
3060	ex, mask, iemode, 1, 1);
3061	temp = expand_binop (GET_MODE (temp), and_optab, temp, ex,
3062	NULL_RTX, 1, OPTAB_LIB_WIDEN);
3063	}
3064	break;
3065	default:
3066	gcc_unreachable ();
3067	}
3068
3069	return temp;
3070	}
3071
3072	/* Expand a call to one of the builtin rounding functions gcc defines
3073	as an extension (lfloor and lceil). As these are gcc extensions we
3074	do not need to worry about setting errno to EDOM.
3075	If expanding via optab fails, lower expression to (int)(floor(x)).
3076	EXP is the expression that is a call to the builtin function;
3077	if convenient, the result should be placed in TARGET. */
3078
3079	static rtx
3080	expand_builtin_int_roundingfn (tree exp, rtx target)
3081	{
3082	convert_optab builtin_optab;
3083	rtx op0, tmp;
3084	rtx_insn *insns;
3085	tree fndecl = get_callee_fndecl (exp);
3086	enum built_in_function fallback_fn;
3087	tree fallback_fndecl;
3088	machine_mode mode;
3089	tree arg;
3090
3091	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
3092	return NULL_RTX;
3093
3094	arg = CALL_EXPR_ARG (exp, 0);
3095
3096	switch (DECL_FUNCTION_CODE (decl: fndecl))
3097	{
3098	CASE_FLT_FN (BUILT_IN_ICEIL):
3099	CASE_FLT_FN (BUILT_IN_LCEIL):
3100	CASE_FLT_FN (BUILT_IN_LLCEIL):
3101	builtin_optab = lceil_optab;
3102	fallback_fn = BUILT_IN_CEIL;
3103	break;
3104
3105	CASE_FLT_FN (BUILT_IN_IFLOOR):
3106	CASE_FLT_FN (BUILT_IN_LFLOOR):
3107	CASE_FLT_FN (BUILT_IN_LLFLOOR):
3108	builtin_optab = lfloor_optab;
3109	fallback_fn = BUILT_IN_FLOOR;
3110	break;
3111
3112	default:
3113	gcc_unreachable ();
3114	}
3115
3116	/* Make a suitable register to place result in. */
3117	mode = TYPE_MODE (TREE_TYPE (exp));
3118
3119	target = gen_reg_rtx (mode);
3120
3121	/* Wrap the computation of the argument in a SAVE_EXPR, as we may
3122	need to expand the argument again. This way, we will not perform
3123	side-effects more the once. */
3124	CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (exp: arg);
3125
3126	op0 = expand_expr (exp: arg, NULL, VOIDmode, modifier: EXPAND_NORMAL);
3127
3128	start_sequence ();
3129
3130	/* Compute into TARGET. */
3131	if (expand_sfix_optab (target, op0, builtin_optab))
3132	{
3133	/* Output the entire sequence. */
3134	insns = get_insns ();
3135	end_sequence ();
3136	emit_insn (insns);
3137	return target;
3138	}
3139
3140	/* If we were unable to expand via the builtin, stop the sequence
3141	(without outputting the insns). */
3142	end_sequence ();
3143
3144	/* Fall back to floating point rounding optab. */
3145	fallback_fndecl = mathfn_built_in (TREE_TYPE (arg), fn: fallback_fn);
3146
3147	/* For non-C99 targets we may end up without a fallback fndecl here
3148	if the user called __builtin_lfloor directly. In this case emit
3149	a call to the floor/ceil variants nevertheless. This should result
3150	in the best user experience for not full C99 targets. */
3151	if (fallback_fndecl == NULL_TREE)
3152	{
3153	tree fntype;
3154	const char *name = NULL;
3155
3156	switch (DECL_FUNCTION_CODE (decl: fndecl))
3157	{
3158	case BUILT_IN_ICEIL:
3159	case BUILT_IN_LCEIL:
3160	case BUILT_IN_LLCEIL:
3161	name = "ceil";
3162	break;
3163	case BUILT_IN_ICEILF:
3164	case BUILT_IN_LCEILF:
3165	case BUILT_IN_LLCEILF:
3166	name = "ceilf";
3167	break;
3168	case BUILT_IN_ICEILL:
3169	case BUILT_IN_LCEILL:
3170	case BUILT_IN_LLCEILL:
3171	name = "ceill";
3172	break;
3173	case BUILT_IN_IFLOOR:
3174	case BUILT_IN_LFLOOR:
3175	case BUILT_IN_LLFLOOR:
3176	name = "floor";
3177	break;
3178	case BUILT_IN_IFLOORF:
3179	case BUILT_IN_LFLOORF:
3180	case BUILT_IN_LLFLOORF:
3181	name = "floorf";
3182	break;
3183	case BUILT_IN_IFLOORL:
3184	case BUILT_IN_LFLOORL:
3185	case BUILT_IN_LLFLOORL:
3186	name = "floorl";
3187	break;
3188	default:
3189	gcc_unreachable ();
3190	}
3191
3192	fntype = build_function_type_list (TREE_TYPE (arg),
3193	TREE_TYPE (arg), NULL_TREE);
3194	fallback_fndecl = build_fn_decl (name, fntype);
3195	}
3196
3197	exp = build_call_nofold_loc (EXPR_LOCATION (exp), fndecl: fallback_fndecl, n: 1, arg);
3198
3199	tmp = expand_normal (exp);
3200	tmp = maybe_emit_group_store (tmp, TREE_TYPE (exp));
3201
3202	/* Truncate the result of floating point optab to integer
3203	via expand_fix (). */
3204	target = gen_reg_rtx (mode);
3205	expand_fix (target, tmp, 0);
3206
3207	return target;
3208	}
3209
3210	/* Expand a call to one of the builtin math functions doing integer
3211	conversion (lrint).
3212	Return 0 if a normal call should be emitted rather than expanding the
3213	function in-line. EXP is the expression that is a call to the builtin
3214	function; if convenient, the result should be placed in TARGET. */
3215
3216	static rtx
3217	expand_builtin_int_roundingfn_2 (tree exp, rtx target)
3218	{
3219	convert_optab builtin_optab;
3220	rtx op0;
3221	rtx_insn *insns;
3222	tree fndecl = get_callee_fndecl (exp);
3223	tree arg;
3224	machine_mode mode;
3225	enum built_in_function fallback_fn = BUILT_IN_NONE;
3226
3227	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
3228	return NULL_RTX;
3229
3230	arg = CALL_EXPR_ARG (exp, 0);
3231
3232	switch (DECL_FUNCTION_CODE (decl: fndecl))
3233	{
3234	CASE_FLT_FN (BUILT_IN_IRINT):
3235	fallback_fn = BUILT_IN_LRINT;
3236	gcc_fallthrough ();
3237	CASE_FLT_FN (BUILT_IN_LRINT):
3238	CASE_FLT_FN (BUILT_IN_LLRINT):
3239	builtin_optab = lrint_optab;
3240	break;
3241
3242	CASE_FLT_FN (BUILT_IN_IROUND):
3243	fallback_fn = BUILT_IN_LROUND;
3244	gcc_fallthrough ();
3245	CASE_FLT_FN (BUILT_IN_LROUND):
3246	CASE_FLT_FN (BUILT_IN_LLROUND):
3247	builtin_optab = lround_optab;
3248	break;
3249
3250	default:
3251	gcc_unreachable ();
3252	}
3253
3254	/* There's no easy way to detect the case we need to set EDOM. */
3255	if (flag_errno_math && fallback_fn == BUILT_IN_NONE)
3256	return NULL_RTX;
3257
3258	/* Make a suitable register to place result in. */
3259	mode = TYPE_MODE (TREE_TYPE (exp));
3260
3261	/* There's no easy way to detect the case we need to set EDOM. */
3262	if (!flag_errno_math)
3263	{
3264	rtx result = gen_reg_rtx (mode);
3265
3266	/* Wrap the computation of the argument in a SAVE_EXPR, as we may
3267	need to expand the argument again. This way, we will not perform
3268	side-effects more the once. */
3269	CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (exp: arg);
3270
3271	op0 = expand_expr (exp: arg, NULL, VOIDmode, modifier: EXPAND_NORMAL);
3272
3273	start_sequence ();
3274
3275	if (expand_sfix_optab (result, op0, builtin_optab))
3276	{
3277	/* Output the entire sequence. */
3278	insns = get_insns ();
3279	end_sequence ();
3280	emit_insn (insns);
3281	return result;
3282	}
3283
3284	/* If we were unable to expand via the builtin, stop the sequence
3285	(without outputting the insns) and call to the library function
3286	with the stabilized argument list. */
3287	end_sequence ();
3288	}
3289
3290	if (fallback_fn != BUILT_IN_NONE)
3291	{
3292	/* Fall back to rounding to long int. Use implicit_p 0 - for non-C99
3293	targets, (int) round (x) should never be transformed into
3294	BUILT_IN_IROUND and if __builtin_iround is called directly, emit
3295	a call to lround in the hope that the target provides at least some
3296	C99 functions. This should result in the best user experience for
3297	not full C99 targets.
3298	As scalar float conversions with same mode are useless in GIMPLE,
3299	we can end up e.g. with _Float32 argument passed to float builtin,
3300	try to get the type from the builtin prototype first. */
3301	tree fallback_fndecl = NULL_TREE;
3302	if (tree argtypes = TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
3303	fallback_fndecl
3304	= mathfn_built_in_1 (TREE_VALUE (argtypes),
3305	fn: as_combined_fn (fn: fallback_fn), implicit_p: 0);
3306	if (fallback_fndecl == NULL_TREE)
3307	fallback_fndecl
3308	= mathfn_built_in_1 (TREE_TYPE (arg),
3309	fn: as_combined_fn (fn: fallback_fn), implicit_p: 0);
3310	if (fallback_fndecl)
3311	{
3312	exp = build_call_nofold_loc (EXPR_LOCATION (exp),
3313	fndecl: fallback_fndecl, n: 1, arg);
3314
3315	target = expand_call (exp, NULL_RTX, target == const0_rtx);
3316	target = maybe_emit_group_store (target, TREE_TYPE (exp));
3317	return convert_to_mode (mode, target, 0);
3318	}
3319	}
3320
3321	return expand_call (exp, target, target == const0_rtx);
3322	}
3323
3324	/* Expand a call to the powi built-in mathematical function. Return NULL_RTX if
3325	a normal call should be emitted rather than expanding the function
3326	in-line. EXP is the expression that is a call to the builtin
3327	function; if convenient, the result should be placed in TARGET. */
3328
3329	static rtx
3330	expand_builtin_powi (tree exp, rtx target)
3331	{
3332	tree arg0, arg1;
3333	rtx op0, op1;
3334	machine_mode mode;
3335	machine_mode mode2;
3336
3337	if (! validate_arglist (callexpr: exp, REAL_TYPE, INTEGER_TYPE, VOID_TYPE))
3338	return NULL_RTX;
3339
3340	arg0 = CALL_EXPR_ARG (exp, 0);
3341	arg1 = CALL_EXPR_ARG (exp, 1);
3342	mode = TYPE_MODE (TREE_TYPE (exp));
3343
3344	/* Emit a libcall to libgcc. */
3345
3346	/* Mode of the 2nd argument must match that of an int. */
3347	mode2 = int_mode_for_size (INT_TYPE_SIZE, limit: 0).require ();
3348
3349	if (target == NULL_RTX)
3350	target = gen_reg_rtx (mode);
3351
3352	op0 = expand_expr (exp: arg0, NULL_RTX, mode, modifier: EXPAND_NORMAL);
3353	if (GET_MODE (op0) != mode)
3354	op0 = convert_to_mode (mode, op0, 0);
3355	op1 = expand_expr (exp: arg1, NULL_RTX, mode: mode2, modifier: EXPAND_NORMAL);
3356	if (GET_MODE (op1) != mode2)
3357	op1 = convert_to_mode (mode2, op1, 0);
3358
3359	target = emit_library_call_value (fun: optab_libfunc (powi_optab, mode),
3360	value: target, fn_type: LCT_CONST, outmode: mode,
3361	arg1: op0, arg1_mode: mode, arg2: op1, arg2_mode: mode2);
3362
3363	return target;
3364	}
3365
3366	/* Expand expression EXP which is a call to the strlen builtin. Return
3367	NULL_RTX if we failed and the caller should emit a normal call, otherwise
3368	try to get the result in TARGET, if convenient. */
3369
3370	static rtx
3371	expand_builtin_strlen (tree exp, rtx target,
3372	machine_mode target_mode)
3373	{
3374	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
3375	return NULL_RTX;
3376
3377	tree src = CALL_EXPR_ARG (exp, 0);
3378
3379	/* If the length can be computed at compile-time, return it. */
3380	if (tree len = c_strlen (arg: src, only_value: 0))
3381	return expand_expr (exp: len, target, mode: target_mode, modifier: EXPAND_NORMAL);
3382
3383	/* If the length can be computed at compile-time and is constant
3384	integer, but there are side-effects in src, evaluate
3385	src for side-effects, then return len.
3386	E.g. x = strlen (i++ ? "xfoo" + 1 : "bar");
3387	can be optimized into: i++; x = 3; */
3388	tree len = c_strlen (arg: src, only_value: 1);
3389	if (len && TREE_CODE (len) == INTEGER_CST)
3390	{
3391	expand_expr (exp: src, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
3392	return expand_expr (exp: len, target, mode: target_mode, modifier: EXPAND_NORMAL);
3393	}
3394
3395	unsigned int align = get_pointer_alignment (exp: src) / BITS_PER_UNIT;
3396
3397	/* If SRC is not a pointer type, don't do this operation inline. */
3398	if (align == 0)
3399	return NULL_RTX;
3400
3401	/* Bail out if we can't compute strlen in the right mode. */
3402	machine_mode insn_mode;
3403	enum insn_code icode = CODE_FOR_nothing;
3404	FOR_EACH_MODE_FROM (insn_mode, target_mode)
3405	{
3406	icode = optab_handler (op: strlen_optab, mode: insn_mode);
3407	if (icode != CODE_FOR_nothing)
3408	break;
3409	}
3410	if (insn_mode == VOIDmode)
3411	return NULL_RTX;
3412
3413	/* Make a place to hold the source address. We will not expand
3414	the actual source until we are sure that the expansion will
3415	not fail -- there are trees that cannot be expanded twice. */
3416	rtx src_reg = gen_reg_rtx (Pmode);
3417
3418	/* Mark the beginning of the strlen sequence so we can emit the
3419	source operand later. */
3420	rtx_insn *before_strlen = get_last_insn ();
3421
3422	class expand_operand ops[4];
3423	create_output_operand (op: &ops[0], x: target, mode: insn_mode);
3424	create_fixed_operand (op: &ops[1], x: gen_rtx_MEM (BLKmode, src_reg));
3425	create_integer_operand (&ops[2], 0);
3426	create_integer_operand (&ops[3], align);
3427	if (!maybe_expand_insn (icode, nops: 4, ops))
3428	return NULL_RTX;
3429
3430	/* Check to see if the argument was declared attribute nonstring
3431	and if so, issue a warning since at this point it's not known
3432	to be nul-terminated. */
3433	maybe_warn_nonstring_arg (get_callee_fndecl (exp), exp);
3434
3435	/* Now that we are assured of success, expand the source. */
3436	start_sequence ();
3437	rtx pat = expand_expr (exp: src, target: src_reg, Pmode, modifier: EXPAND_NORMAL);
3438	if (pat != src_reg)
3439	{
3440	#ifdef POINTERS_EXTEND_UNSIGNED
3441	if (GET_MODE (pat) != Pmode)
3442	pat = convert_to_mode (Pmode, pat,
3443	POINTERS_EXTEND_UNSIGNED);
3444	#endif
3445	emit_move_insn (src_reg, pat);
3446	}
3447	pat = get_insns ();
3448	end_sequence ();
3449
3450	if (before_strlen)
3451	emit_insn_after (pat, before_strlen);
3452	else
3453	emit_insn_before (pat, get_insns ());
3454
3455	/* Return the value in the proper mode for this function. */
3456	if (GET_MODE (ops[0].value) == target_mode)
3457	target = ops[0].value;
3458	else if (target != 0)
3459	convert_move (target, ops[0].value, 0);
3460	else
3461	target = convert_to_mode (target_mode, ops[0].value, 0);
3462
3463	return target;
3464	}
3465
3466	/* Expand call EXP to the strnlen built-in, returning the result
3467	and setting it in TARGET. Otherwise return NULL_RTX on failure. */
3468
3469	static rtx
3470	expand_builtin_strnlen (tree exp, rtx target, machine_mode target_mode)
3471	{
3472	if (!validate_arglist (callexpr: exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
3473	return NULL_RTX;
3474
3475	tree src = CALL_EXPR_ARG (exp, 0);
3476	tree bound = CALL_EXPR_ARG (exp, 1);
3477
3478	if (!bound)
3479	return NULL_RTX;
3480
3481	location_t loc = UNKNOWN_LOCATION;
3482	if (EXPR_HAS_LOCATION (exp))
3483	loc = EXPR_LOCATION (exp);
3484
3485	/* FIXME: Change c_strlen() to return sizetype instead of ssizetype
3486	so these conversions aren't necessary. */
3487	c_strlen_data lendata = { };
3488	tree len = c_strlen (arg: src, only_value: 0, data: &lendata, eltsize: 1);
3489	if (len)
3490	len = fold_convert_loc (loc, TREE_TYPE (bound), len);
3491
3492	if (TREE_CODE (bound) == INTEGER_CST)
3493	{
3494	if (!len)
3495	return NULL_RTX;
3496
3497	len = fold_build2_loc (loc, MIN_EXPR, size_type_node, len, bound);
3498	return expand_expr (exp: len, target, mode: target_mode, modifier: EXPAND_NORMAL);
3499	}
3500
3501	if (TREE_CODE (bound) != SSA_NAME)
3502	return NULL_RTX;
3503
3504	wide_int min, max;
3505	value_range r;
3506	get_global_range_query ()->range_of_expr (r, expr: bound);
3507	if (r.varying_p () || r.undefined_p ())
3508	return NULL_RTX;
3509	min = r.lower_bound ();
3510	max = r.upper_bound ();
3511
3512	if (!len || TREE_CODE (len) != INTEGER_CST)
3513	{
3514	bool exact;
3515	lendata.decl = unterminated_array (src, &len, &exact);
3516	if (!lendata.decl)
3517	return NULL_RTX;
3518	}
3519
3520	if (lendata.decl)
3521	return NULL_RTX;
3522
3523	if (wi::gtu_p (x: min, y: wi::to_wide (t: len)))
3524	return expand_expr (exp: len, target, mode: target_mode, modifier: EXPAND_NORMAL);
3525
3526	len = fold_build2_loc (loc, MIN_EXPR, TREE_TYPE (len), len, bound);
3527	return expand_expr (exp: len, target, mode: target_mode, modifier: EXPAND_NORMAL);
3528	}
3529
3530	/* Callback routine for store_by_pieces. Read GET_MODE_BITSIZE (MODE)
3531	bytes from bytes at DATA + OFFSET and return it reinterpreted as
3532	a target constant. */
3533
3534	static rtx
3535	builtin_memcpy_read_str (void *data, void *, HOST_WIDE_INT offset,
3536	fixed_size_mode mode)
3537	{
3538	/* The REPresentation pointed to by DATA need not be a nul-terminated
3539	string but the caller guarantees it's large enough for MODE. */
3540	const char *rep = (const char *) data;
3541
3542	return c_readstr (str: rep + offset, mode, /*nul_terminated=*/null_terminated_p: false);
3543	}
3544
3545	/* LEN specify length of the block of memcpy/memset operation.
3546	Figure out its range and put it into MIN_SIZE/MAX_SIZE.
3547	In some cases we can make very likely guess on max size, then we
3548	set it into PROBABLE_MAX_SIZE. */
3549
3550	static void
3551	determine_block_size (tree len, rtx len_rtx,
3552	unsigned HOST_WIDE_INT *min_size,
3553	unsigned HOST_WIDE_INT *max_size,
3554	unsigned HOST_WIDE_INT *probable_max_size)
3555	{
3556	if (CONST_INT_P (len_rtx))
3557	{
3558	*min_size = *max_size = *probable_max_size = UINTVAL (len_rtx);
3559	return;
3560	}
3561	else
3562	{
3563	wide_int min, max;
3564	enum value_range_kind range_type = VR_UNDEFINED;
3565
3566	/* Determine bounds from the type. */
3567	if (tree_fits_uhwi_p (TYPE_MIN_VALUE (TREE_TYPE (len))))
3568	*min_size = tree_to_uhwi (TYPE_MIN_VALUE (TREE_TYPE (len)));
3569	else
3570	*min_size = 0;
3571	if (tree_fits_uhwi_p (TYPE_MAX_VALUE (TREE_TYPE (len))))
3572	*probable_max_size = *max_size
3573	= tree_to_uhwi (TYPE_MAX_VALUE (TREE_TYPE (len)));
3574	else
3575	*probable_max_size = *max_size = GET_MODE_MASK (GET_MODE (len_rtx));
3576
3577	if (TREE_CODE (len) == SSA_NAME)
3578	{
3579	value_range r;
3580	tree tmin, tmax;
3581	get_global_range_query ()->range_of_expr (r, expr: len);
3582	range_type = get_legacy_range (r, min&: tmin, max&: tmax);
3583	if (range_type != VR_UNDEFINED)
3584	{
3585	min = wi::to_wide (t: tmin);
3586	max = wi::to_wide (t: tmax);
3587	}
3588	}
3589	if (range_type == VR_RANGE)
3590	{
3591	if (wi::fits_uhwi_p (x: min) && *min_size < min.to_uhwi ())
3592	*min_size = min.to_uhwi ();
3593	if (wi::fits_uhwi_p (x: max) && *max_size > max.to_uhwi ())
3594	*probable_max_size = *max_size = max.to_uhwi ();
3595	}
3596	else if (range_type == VR_ANTI_RANGE)
3597	{
3598	/* Code like
3599
3600	int n;
3601	if (n < 100)
3602	memcpy (a, b, n)
3603
3604	Produce anti range allowing negative values of N. We still
3605	can use the information and make a guess that N is not negative.
3606	*/
3607	if (!wi::leu_p (x: max, y: 1 << 30) && wi::fits_uhwi_p (x: min))
3608	*probable_max_size = min.to_uhwi () - 1;
3609	}
3610	}
3611	gcc_checking_assert (*max_size <=
3612	(unsigned HOST_WIDE_INT)
3613	GET_MODE_MASK (GET_MODE (len_rtx)));
3614	}
3615
3616	/* Expand a call EXP to the memcpy builtin.
3617	Return NULL_RTX if we failed, the caller should emit a normal call,
3618	otherwise try to get the result in TARGET, if convenient (and in
3619	mode MODE if that's convenient). */
3620
3621	static rtx
3622	expand_builtin_memcpy (tree exp, rtx target)
3623	{
3624	if (!validate_arglist (callexpr: exp,
3625	POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
3626	return NULL_RTX;
3627
3628	tree dest = CALL_EXPR_ARG (exp, 0);
3629	tree src = CALL_EXPR_ARG (exp, 1);
3630	tree len = CALL_EXPR_ARG (exp, 2);
3631
3632	return expand_builtin_memory_copy_args (dest, src, len, target, exp,
3633	/*retmode=*/ RETURN_BEGIN, might_overlap: false);
3634	}
3635
3636	/* Check a call EXP to the memmove built-in for validity.
3637	Return NULL_RTX on both success and failure. */
3638
3639	static rtx
3640	expand_builtin_memmove (tree exp, rtx target)
3641	{
3642	if (!validate_arglist (callexpr: exp,
3643	POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
3644	return NULL_RTX;
3645
3646	tree dest = CALL_EXPR_ARG (exp, 0);
3647	tree src = CALL_EXPR_ARG (exp, 1);
3648	tree len = CALL_EXPR_ARG (exp, 2);
3649
3650	return expand_builtin_memory_copy_args (dest, src, len, target, exp,
3651	/*retmode=*/ RETURN_BEGIN, might_overlap: true);
3652	}
3653
3654	/* Expand a call EXP to the mempcpy builtin.
3655	Return NULL_RTX if we failed; the caller should emit a normal call,
3656	otherwise try to get the result in TARGET, if convenient (and in
3657	mode MODE if that's convenient). */
3658
3659	static rtx
3660	expand_builtin_mempcpy (tree exp, rtx target)
3661	{
3662	if (!validate_arglist (callexpr: exp,
3663	POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
3664	return NULL_RTX;
3665
3666	tree dest = CALL_EXPR_ARG (exp, 0);
3667	tree src = CALL_EXPR_ARG (exp, 1);
3668	tree len = CALL_EXPR_ARG (exp, 2);
3669
3670	/* Policy does not generally allow using compute_objsize (which
3671	is used internally by check_memop_size) to change code generation
3672	or drive optimization decisions.
3673
3674	In this instance it is safe because the code we generate has
3675	the same semantics regardless of the return value of
3676	check_memop_sizes. Exactly the same amount of data is copied
3677	and the return value is exactly the same in both cases.
3678
3679	Furthermore, check_memop_size always uses mode 0 for the call to
3680	compute_objsize, so the imprecise nature of compute_objsize is
3681	avoided. */
3682
3683	/* Avoid expanding mempcpy into memcpy when the call is determined
3684	to overflow the buffer. This also prevents the same overflow
3685	from being diagnosed again when expanding memcpy. */
3686
3687	return expand_builtin_mempcpy_args (dest, src, len,
3688	target, exp, /*retmode=*/ RETURN_END);
3689	}
3690
3691	/* Helper function to do the actual work for expand of memory copy family
3692	functions (memcpy, mempcpy, stpcpy). Expansing should assign LEN bytes
3693	of memory from SRC to DEST and assign to TARGET if convenient. Return
3694	value is based on RETMODE argument. */
3695
3696	static rtx
3697	expand_builtin_memory_copy_args (tree dest, tree src, tree len,
3698	rtx target, tree exp, memop_ret retmode,
3699	bool might_overlap)
3700	{
3701	unsigned int src_align = get_pointer_alignment (exp: src);
3702	unsigned int dest_align = get_pointer_alignment (exp: dest);
3703	rtx dest_mem, src_mem, dest_addr, len_rtx;
3704	HOST_WIDE_INT expected_size = -1;
3705	unsigned int expected_align = 0;
3706	unsigned HOST_WIDE_INT min_size;
3707	unsigned HOST_WIDE_INT max_size;
3708	unsigned HOST_WIDE_INT probable_max_size;
3709
3710	bool is_move_done;
3711
3712	/* If DEST is not a pointer type, call the normal function. */
3713	if (dest_align == 0)
3714	return NULL_RTX;
3715
3716	/* If either SRC is not a pointer type, don't do this
3717	operation in-line. */
3718	if (src_align == 0)
3719	return NULL_RTX;
3720
3721	if (currently_expanding_gimple_stmt)
3722	stringop_block_profile (currently_expanding_gimple_stmt,
3723	&expected_align, &expected_size);
3724
3725	if (expected_align < dest_align)
3726	expected_align = dest_align;
3727	dest_mem = get_memory_rtx (exp: dest, len);
3728	set_mem_align (dest_mem, dest_align);
3729	len_rtx = expand_normal (exp: len);
3730	determine_block_size (len, len_rtx, min_size: &min_size, max_size: &max_size,
3731	probable_max_size: &probable_max_size);
3732
3733	/* Try to get the byte representation of the constant SRC points to,
3734	with its byte size in NBYTES. */
3735	unsigned HOST_WIDE_INT nbytes;
3736	const char *rep = getbyterep (src, &nbytes);
3737
3738	/* If the function's constant bound LEN_RTX is less than or equal
3739	to the byte size of the representation of the constant argument,
3740	and if block move would be done by pieces, we can avoid loading
3741	the bytes from memory and only store the computed constant.
3742	This works in the overlap (memmove) case as well because
3743	store_by_pieces just generates a series of stores of constants
3744	from the representation returned by getbyterep(). */
3745	if (rep
3746	&& CONST_INT_P (len_rtx)
3747	&& (unsigned HOST_WIDE_INT) INTVAL (len_rtx) <= nbytes
3748	&& can_store_by_pieces (INTVAL (len_rtx), builtin_memcpy_read_str,
3749	CONST_CAST (char *, rep),
3750	dest_align, false))
3751	{
3752	dest_mem = store_by_pieces (dest_mem, INTVAL (len_rtx),
3753	builtin_memcpy_read_str,
3754	CONST_CAST (char *, rep),
3755	dest_align, false, retmode);
3756	dest_mem = force_operand (XEXP (dest_mem, 0), target);
3757	dest_mem = convert_memory_address (ptr_mode, dest_mem);
3758	return dest_mem;
3759	}
3760
3761	src_mem = get_memory_rtx (exp: src, len);
3762	set_mem_align (src_mem, src_align);
3763
3764	/* Copy word part most expediently. */
3765	enum block_op_methods method = BLOCK_OP_NORMAL;
3766	if (CALL_EXPR_TAILCALL (exp)
3767	&& (retmode == RETURN_BEGIN || target == const0_rtx))
3768	method = BLOCK_OP_TAILCALL;
3769	bool use_mempcpy_call = (targetm.libc_has_fast_function (BUILT_IN_MEMPCPY)
3770	&& retmode == RETURN_END
3771	&& !might_overlap
3772	&& target != const0_rtx);
3773	if (use_mempcpy_call)
3774	method = BLOCK_OP_NO_LIBCALL_RET;
3775	dest_addr = emit_block_move_hints (dest_mem, src_mem, len_rtx, method,
3776	expected_align, expected_size,
3777	min_size, max_size, probable_max_size,
3778	bail_out_libcall: use_mempcpy_call, is_move_done: &is_move_done,
3779	might_overlap, ctz_size: tree_ctz (len));
3780
3781	/* Bail out when a mempcpy call would be expanded as libcall and when
3782	we have a target that provides a fast implementation
3783	of mempcpy routine. */
3784	if (!is_move_done)
3785	return NULL_RTX;
3786
3787	if (dest_addr == pc_rtx)
3788	return NULL_RTX;
3789
3790	if (dest_addr == 0)
3791	{
3792	dest_addr = force_operand (XEXP (dest_mem, 0), target);
3793	dest_addr = convert_memory_address (ptr_mode, dest_addr);
3794	}
3795
3796	if (retmode != RETURN_BEGIN && target != const0_rtx)
3797	{
3798	dest_addr = gen_rtx_PLUS (ptr_mode, dest_addr, len_rtx);
3799	/* stpcpy pointer to last byte. */
3800	if (retmode == RETURN_END_MINUS_ONE)
3801	dest_addr = gen_rtx_MINUS (ptr_mode, dest_addr, const1_rtx);
3802	}
3803
3804	return dest_addr;
3805	}
3806
3807	static rtx
3808	expand_builtin_mempcpy_args (tree dest, tree src, tree len,
3809	rtx target, tree orig_exp, memop_ret retmode)
3810	{
3811	return expand_builtin_memory_copy_args (dest, src, len, target, exp: orig_exp,
3812	retmode, might_overlap: false);
3813	}
3814
3815	/* Expand into a movstr instruction, if one is available. Return NULL_RTX if
3816	we failed, the caller should emit a normal call, otherwise try to
3817	get the result in TARGET, if convenient.
3818	Return value is based on RETMODE argument. */
3819
3820	static rtx
3821	expand_movstr (tree dest, tree src, rtx target, memop_ret retmode)
3822	{
3823	class expand_operand ops[3];
3824	rtx dest_mem;
3825	rtx src_mem;
3826
3827	if (!targetm.have_movstr ())
3828	return NULL_RTX;
3829
3830	dest_mem = get_memory_rtx (exp: dest, NULL);
3831	src_mem = get_memory_rtx (exp: src, NULL);
3832	if (retmode == RETURN_BEGIN)
3833	{
3834	target = force_reg (Pmode, XEXP (dest_mem, 0));
3835	dest_mem = replace_equiv_address (dest_mem, target);
3836	}
3837
3838	create_output_operand (op: &ops[0],
3839	x: retmode != RETURN_BEGIN ? target : NULL_RTX, Pmode);
3840	create_fixed_operand (op: &ops[1], x: dest_mem);
3841	create_fixed_operand (op: &ops[2], x: src_mem);
3842	if (!maybe_expand_insn (icode: targetm.code_for_movstr, nops: 3, ops))
3843	return NULL_RTX;
3844
3845	if (retmode != RETURN_BEGIN && target != const0_rtx)
3846	{
3847	target = ops[0].value;
3848	/* movstr is supposed to set end to the address of the NUL
3849	terminator. If the caller requested a mempcpy-like return value,
3850	adjust it. */
3851	if (retmode == RETURN_END)
3852	{
3853	rtx tem = plus_constant (GET_MODE (target),
3854	gen_lowpart (GET_MODE (target), target), 1);
3855	emit_move_insn (target, force_operand (tem, NULL_RTX));
3856	}
3857	}
3858	return target;
3859	}
3860
3861	/* Expand expression EXP, which is a call to the strcpy builtin. Return
3862	NULL_RTX if we failed the caller should emit a normal call, otherwise
3863	try to get the result in TARGET, if convenient (and in mode MODE if that's
3864	convenient). */
3865
3866	static rtx
3867	expand_builtin_strcpy (tree exp, rtx target)
3868	{
3869	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
3870	return NULL_RTX;
3871
3872	tree dest = CALL_EXPR_ARG (exp, 0);
3873	tree src = CALL_EXPR_ARG (exp, 1);
3874
3875	return expand_builtin_strcpy_args (exp, dest, src, target);
3876	}
3877
3878	/* Helper function to do the actual work for expand_builtin_strcpy. The
3879	arguments to the builtin_strcpy call DEST and SRC are broken out
3880	so that this can also be called without constructing an actual CALL_EXPR.
3881	The other arguments and return value are the same as for
3882	expand_builtin_strcpy. */
3883
3884	static rtx
3885	expand_builtin_strcpy_args (tree, tree dest, tree src, rtx target)
3886	{
3887	return expand_movstr (dest, src, target, /*retmode=*/ RETURN_BEGIN);
3888	}
3889
3890	/* Expand a call EXP to the stpcpy builtin.
3891	Return NULL_RTX if we failed the caller should emit a normal call,
3892	otherwise try to get the result in TARGET, if convenient (and in
3893	mode MODE if that's convenient). */
3894
3895	static rtx
3896	expand_builtin_stpcpy_1 (tree exp, rtx target, machine_mode mode)
3897	{
3898	tree dst, src;
3899	location_t loc = EXPR_LOCATION (exp);
3900
3901	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
3902	return NULL_RTX;
3903
3904	dst = CALL_EXPR_ARG (exp, 0);
3905	src = CALL_EXPR_ARG (exp, 1);
3906
3907	/* If return value is ignored, transform stpcpy into strcpy. */
3908	if (target == const0_rtx && builtin_decl_implicit (fncode: BUILT_IN_STRCPY))
3909	{
3910	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRCPY);
3911	tree result = build_call_nofold_loc (loc, fndecl: fn, n: 2, dst, src);
3912	return expand_expr (exp: result, target, mode, modifier: EXPAND_NORMAL);
3913	}
3914	else
3915	{
3916	tree len, lenp1;
3917	rtx ret;
3918
3919	/* Ensure we get an actual string whose length can be evaluated at
3920	compile-time, not an expression containing a string. This is
3921	because the latter will potentially produce pessimized code
3922	when used to produce the return value. */
3923	c_strlen_data lendata = { };
3924	if (!c_getstr (src)
3925	|| !(len = c_strlen (arg: src, only_value: 0, data: &lendata, eltsize: 1)))
3926	return expand_movstr (dest: dst, src, target,
3927	/*retmode=*/ RETURN_END_MINUS_ONE);
3928
3929	lenp1 = size_binop_loc (loc, PLUS_EXPR, len, ssize_int (1));
3930	ret = expand_builtin_mempcpy_args (dest: dst, src, len: lenp1,
3931	target, orig_exp: exp,
3932	/*retmode=*/ RETURN_END_MINUS_ONE);
3933
3934	if (ret)
3935	return ret;
3936
3937	if (TREE_CODE (len) == INTEGER_CST)
3938	{
3939	rtx len_rtx = expand_normal (exp: len);
3940
3941	if (CONST_INT_P (len_rtx))
3942	{
3943	ret = expand_builtin_strcpy_args (exp, dest: dst, src, target);
3944
3945	if (ret)
3946	{
3947	if (! target)
3948	{
3949	if (mode != VOIDmode)
3950	target = gen_reg_rtx (mode);
3951	else
3952	target = gen_reg_rtx (GET_MODE (ret));
3953	}
3954	if (GET_MODE (target) != GET_MODE (ret))
3955	ret = gen_lowpart (GET_MODE (target), ret);
3956
3957	ret = plus_constant (GET_MODE (ret), ret, INTVAL (len_rtx));
3958	ret = emit_move_insn (target, force_operand (ret, NULL_RTX));
3959	gcc_assert (ret);
3960
3961	return target;
3962	}
3963	}
3964	}
3965
3966	return expand_movstr (dest: dst, src, target,
3967	/*retmode=*/ RETURN_END_MINUS_ONE);
3968	}
3969	}
3970
3971	/* Expand a call EXP to the stpcpy builtin and diagnose uses of nonstring
3972	arguments while being careful to avoid duplicate warnings (which could
3973	be issued if the expander were to expand the call, resulting in it
3974	being emitted in expand_call(). */
3975
3976	static rtx
3977	expand_builtin_stpcpy (tree exp, rtx target, machine_mode mode)
3978	{
3979	if (rtx ret = expand_builtin_stpcpy_1 (exp, target, mode))
3980	{
3981	/* The call has been successfully expanded. Check for nonstring
3982	arguments and issue warnings as appropriate. */
3983	maybe_warn_nonstring_arg (get_callee_fndecl (exp), exp);
3984	return ret;
3985	}
3986
3987	return NULL_RTX;
3988	}
3989
3990	/* Callback routine for store_by_pieces. Read GET_MODE_BITSIZE (MODE)
3991	bytes from constant string DATA + OFFSET and return it as target
3992	constant. */
3993
3994	rtx
3995	builtin_strncpy_read_str (void *data, void *, HOST_WIDE_INT offset,
3996	fixed_size_mode mode)
3997	{
3998	const char *str = (const char *) data;
3999
4000	if ((unsigned HOST_WIDE_INT) offset > strlen (s: str))
4001	return const0_rtx;
4002
4003	return c_readstr (str: str + offset, mode);
4004	}
4005
4006	/* Helper to check the sizes of sequences and the destination of calls
4007	to __builtin_strncat and __builtin___strncat_chk. Returns true on
4008	success (no overflow or invalid sizes), false otherwise. */
4009
4010	static bool
4011	check_strncat_sizes (tree exp, tree objsize)
4012	{
4013	tree dest = CALL_EXPR_ARG (exp, 0);
4014	tree src = CALL_EXPR_ARG (exp, 1);
4015	tree maxread = CALL_EXPR_ARG (exp, 2);
4016
4017	/* Try to determine the range of lengths that the source expression
4018	refers to. */
4019	c_strlen_data lendata = { };
4020	get_range_strlen (src, &lendata, /* eltsize = */ 1);
4021
4022	/* Try to verify that the destination is big enough for the shortest
4023	string. */
4024
4025	access_data data (nullptr, exp, access_read_write, maxread, true);
4026	if (!objsize && warn_stringop_overflow)
4027	{
4028	/* If it hasn't been provided by __strncat_chk, try to determine
4029	the size of the destination object into which the source is
4030	being copied. */
4031	objsize = compute_objsize (ptr: dest, warn_stringop_overflow - 1, pref: &data.dst);
4032	}
4033
4034	/* Add one for the terminating nul. */
4035	tree srclen = (lendata.minlen
4036	? fold_build2 (PLUS_EXPR, size_type_node, lendata.minlen,
4037	size_one_node)
4038	: NULL_TREE);
4039
4040	/* The strncat function copies at most MAXREAD bytes and always appends
4041	the terminating nul so the specified upper bound should never be equal
4042	to (or greater than) the size of the destination. */
4043	if (tree_fits_uhwi_p (maxread) && tree_fits_uhwi_p (objsize)
4044	&& tree_int_cst_equal (objsize, maxread))
4045	{
4046	location_t loc = EXPR_LOCATION (exp);
4047	warning_at (loc, OPT_Wstringop_overflow_,
4048	"%qD specified bound %E equals destination size",
4049	get_callee_fndecl (exp), maxread);
4050
4051	return false;
4052	}
4053
4054	if (!srclen
4055	|| (maxread && tree_fits_uhwi_p (maxread)
4056	&& tree_fits_uhwi_p (srclen)
4057	&& tree_int_cst_lt (t1: maxread, t2: srclen)))
4058	srclen = maxread;
4059
4060	/* The number of bytes to write is LEN but check_access will alsoa
4061	check SRCLEN if LEN's value isn't known. */
4062	return check_access (exp, /*dstwrite=*/NULL_TREE, maxread, srclen,
4063	objsize, data.mode, &data);
4064	}
4065
4066	/* Expand expression EXP, which is a call to the strncpy builtin. Return
4067	NULL_RTX if we failed the caller should emit a normal call. */
4068
4069	static rtx
4070	expand_builtin_strncpy (tree exp, rtx target)
4071	{
4072	location_t loc = EXPR_LOCATION (exp);
4073
4074	if (!validate_arglist (callexpr: exp,
4075	POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
4076	return NULL_RTX;
4077	tree dest = CALL_EXPR_ARG (exp, 0);
4078	tree src = CALL_EXPR_ARG (exp, 1);
4079	/* The number of bytes to write (not the maximum). */
4080	tree len = CALL_EXPR_ARG (exp, 2);
4081
4082	/* The length of the source sequence. */
4083	tree slen = c_strlen (arg: src, only_value: 1);
4084
4085	/* We must be passed a constant len and src parameter. */
4086	if (!tree_fits_uhwi_p (len) || !slen || !tree_fits_uhwi_p (slen))
4087	return NULL_RTX;
4088
4089	slen = size_binop_loc (loc, PLUS_EXPR, slen, ssize_int (1));
4090
4091	/* We're required to pad with trailing zeros if the requested
4092	len is greater than strlen(s2)+1. In that case try to
4093	use store_by_pieces, if it fails, punt. */
4094	if (tree_int_cst_lt (t1: slen, t2: len))
4095	{
4096	unsigned int dest_align = get_pointer_alignment (exp: dest);
4097	const char *p = c_getstr (src);
4098	rtx dest_mem;
4099
4100	if (!p || dest_align == 0 || !tree_fits_uhwi_p (len)
4101	|| !can_store_by_pieces (tree_to_uhwi (len),
4102	builtin_strncpy_read_str,
4103	CONST_CAST (char *, p),
4104	dest_align, false))
4105	return NULL_RTX;
4106
4107	dest_mem = get_memory_rtx (exp: dest, len);
4108	store_by_pieces (dest_mem, tree_to_uhwi (len),
4109	builtin_strncpy_read_str,
4110	CONST_CAST (char *, p), dest_align, false,
4111	RETURN_BEGIN);
4112	dest_mem = force_operand (XEXP (dest_mem, 0), target);
4113	dest_mem = convert_memory_address (ptr_mode, dest_mem);
4114	return dest_mem;
4115	}
4116
4117	return NULL_RTX;
4118	}
4119
4120	/* Return the RTL of a register in MODE generated from PREV in the
4121	previous iteration. */
4122
4123	static rtx
4124	gen_memset_value_from_prev (by_pieces_prev *prev, fixed_size_mode mode)
4125	{
4126	rtx target = nullptr;
4127	if (prev != nullptr && prev->data != nullptr)
4128	{
4129	/* Use the previous data in the same mode. */
4130	if (prev->mode == mode)
4131	return prev->data;
4132
4133	fixed_size_mode prev_mode = prev->mode;
4134
4135	/* Don't use the previous data to write QImode if it is in a
4136	vector mode. */
4137	if (VECTOR_MODE_P (prev_mode) && mode == QImode)
4138	return target;
4139
4140	rtx prev_rtx = prev->data;
4141
4142	if (REG_P (prev_rtx)
4143	&& HARD_REGISTER_P (prev_rtx)
4144	&& lowpart_subreg_regno (REGNO (prev_rtx), prev_mode, mode) < 0)
4145	{
4146	/* This case occurs when PREV_MODE is a vector and when
4147	MODE is too small to store using vector operations.
4148	After register allocation, the code will need to move the
4149	lowpart of the vector register into a non-vector register.
4150
4151	Also, the target has chosen to use a hard register
4152	instead of going with the default choice of using a
4153	pseudo register. We should respect that choice and try to
4154	avoid creating a pseudo register with the same mode as the
4155	current hard register.
4156
4157	In principle, we could just use a lowpart MODE subreg of
4158	the vector register. However, the vector register mode might
4159	be too wide for non-vector registers, and we already know
4160	that the non-vector mode is too small for vector registers.
4161	It's therefore likely that we'd need to spill to memory in
4162	the vector mode and reload the non-vector value from there.
4163
4164	Try to avoid that by reducing the vector register to the
4165	smallest size that it can hold. This should increase the
4166	chances that non-vector registers can hold both the inner
4167	and outer modes of the subreg that we generate later. */
4168	machine_mode m;
4169	fixed_size_mode candidate;
4170	FOR_EACH_MODE_IN_CLASS (m, GET_MODE_CLASS (mode))
4171	if (is_a<fixed_size_mode> (m, result: &candidate))
4172	{
4173	if (GET_MODE_SIZE (mode: candidate)
4174	>= GET_MODE_SIZE (mode: prev_mode))
4175	break;
4176	if (GET_MODE_SIZE (mode: candidate) >= GET_MODE_SIZE (mode)
4177	&& lowpart_subreg_regno (REGNO (prev_rtx),
4178	prev_mode, candidate) >= 0)
4179	{
4180	target = lowpart_subreg (outermode: candidate, op: prev_rtx,
4181	innermode: prev_mode);
4182	prev_rtx = target;
4183	prev_mode = candidate;
4184	break;
4185	}
4186	}
4187	if (target == nullptr)
4188	prev_rtx = copy_to_reg (prev_rtx);
4189	}
4190
4191	target = lowpart_subreg (outermode: mode, op: prev_rtx, innermode: prev_mode);
4192	}
4193	return target;
4194	}
4195
4196	/* Callback routine for store_by_pieces. Read GET_MODE_BITSIZE (MODE)
4197	bytes from constant string DATA + OFFSET and return it as target
4198	constant. If PREV isn't nullptr, it has the RTL info from the
4199	previous iteration. */
4200
4201	rtx
4202	builtin_memset_read_str (void *data, void *prev,
4203	HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
4204	fixed_size_mode mode)
4205	{
4206	const char *c = (const char *) data;
4207	unsigned int size = GET_MODE_SIZE (mode);
4208
4209	rtx target = gen_memset_value_from_prev (prev: (by_pieces_prev *) prev,
4210	mode);
4211	if (target != nullptr)
4212	return target;
4213	rtx src = gen_int_mode (*c, QImode);
4214
4215	if (VECTOR_MODE_P (mode))
4216	{
4217	gcc_assert (GET_MODE_INNER (mode) == QImode);
4218
4219	rtx const_vec = gen_const_vec_duplicate (mode, src);
4220	if (prev == NULL)
4221	/* Return CONST_VECTOR when called by a query function. */
4222	return const_vec;
4223
4224	/* Use the move expander with CONST_VECTOR. */
4225	target = gen_reg_rtx (mode);
4226	emit_move_insn (target, const_vec);
4227	return target;
4228	}
4229
4230	char *p = XALLOCAVEC (char, size);
4231
4232	memset (s: p, c: *c, n: size);
4233
4234	return c_readstr (str: p, mode);
4235	}
4236
4237	/* Callback routine for store_by_pieces. Return the RTL of a register
4238	containing GET_MODE_SIZE (MODE) consecutive copies of the unsigned
4239	char value given in the RTL register data. For example, if mode is
4240	4 bytes wide, return the RTL for 0x01010101*data. If PREV isn't
4241	nullptr, it has the RTL info from the previous iteration. */
4242
4243	static rtx
4244	builtin_memset_gen_str (void *data, void *prev,
4245	HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
4246	fixed_size_mode mode)
4247	{
4248	rtx target, coeff;
4249	size_t size;
4250	char *p;
4251
4252	size = GET_MODE_SIZE (mode);
4253	if (size == 1)
4254	return (rtx) data;
4255
4256	target = gen_memset_value_from_prev (prev: (by_pieces_prev *) prev, mode);
4257	if (target != nullptr)
4258	return target;
4259
4260	if (VECTOR_MODE_P (mode))
4261	{
4262	gcc_assert (GET_MODE_INNER (mode) == QImode);
4263
4264	/* vec_duplicate_optab is a precondition to pick a vector mode for
4265	the memset expander. */
4266	insn_code icode = optab_handler (op: vec_duplicate_optab, mode);
4267
4268	target = gen_reg_rtx (mode);
4269	class expand_operand ops[2];
4270	create_output_operand (op: &ops[0], x: target, mode);
4271	create_input_operand (op: &ops[1], value: (rtx) data, QImode);
4272	expand_insn (icode, nops: 2, ops);
4273	if (!rtx_equal_p (target, ops[0].value))
4274	emit_move_insn (target, ops[0].value);
4275
4276	return target;
4277	}
4278
4279	p = XALLOCAVEC (char, size);
4280	memset (s: p, c: 1, n: size);
4281	coeff = c_readstr (str: p, mode);
4282
4283	target = convert_to_mode (mode, (rtx) data, 1);
4284	target = expand_mult (mode, target, coeff, NULL_RTX, 1);
4285	return force_reg (mode, target);
4286	}
4287
4288	/* Expand expression EXP, which is a call to the memset builtin. Return
4289	NULL_RTX if we failed the caller should emit a normal call, otherwise
4290	try to get the result in TARGET, if convenient (and in mode MODE if that's
4291	convenient). */
4292
4293	rtx
4294	expand_builtin_memset (tree exp, rtx target, machine_mode mode)
4295	{
4296	if (!validate_arglist (callexpr: exp,
4297	POINTER_TYPE, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE))
4298	return NULL_RTX;
4299
4300	tree dest = CALL_EXPR_ARG (exp, 0);
4301	tree val = CALL_EXPR_ARG (exp, 1);
4302	tree len = CALL_EXPR_ARG (exp, 2);
4303
4304	return expand_builtin_memset_args (dest, val, len, target, mode, exp);
4305	}
4306
4307	/* Check that store_by_pieces allows BITS + LEN (so that we don't
4308	expand something too unreasonably long), and every power of 2 in
4309	BITS. It is assumed that LEN has already been tested by
4310	itself. */
4311	static bool
4312	can_store_by_multiple_pieces (unsigned HOST_WIDE_INT bits,
4313	by_pieces_constfn constfun,
4314	void *constfundata, unsigned int align,
4315	bool memsetp,
4316	unsigned HOST_WIDE_INT len)
4317	{
4318	if (bits
4319	&& !can_store_by_pieces (bits + len, constfun, constfundata,
4320	align, memsetp))
4321	return false;
4322
4323	/* BITS set are expected to be generally in the low range and
4324	contiguous. We do NOT want to repeat the test above in case BITS
4325	has a single bit set, so we terminate the loop when BITS == BIT.
4326	In the unlikely case that BITS has the MSB set, also terminate in
4327	case BIT gets shifted out. */
4328	for (unsigned HOST_WIDE_INT bit = 1; bit < bits && bit; bit <<= 1)
4329	{
4330	if ((bits & bit) == 0)
4331	continue;
4332
4333	if (!can_store_by_pieces (bit, constfun, constfundata,
4334	align, memsetp))
4335	return false;
4336	}
4337
4338	return true;
4339	}
4340
4341	/* Try to store VAL (or, if NULL_RTX, VALC) in LEN bytes starting at TO.
4342	Return TRUE if successful, FALSE otherwise. TO is assumed to be
4343	aligned at an ALIGN-bits boundary. LEN must be a multiple of
4344	1<<CTZ_LEN between MIN_LEN and MAX_LEN.
4345
4346	The strategy is to issue one store_by_pieces for each power of two,
4347	from most to least significant, guarded by a test on whether there
4348	are at least that many bytes left to copy in LEN.
4349
4350	??? Should we skip some powers of two in favor of loops? Maybe start
4351	at the max of TO/LEN/word alignment, at least when optimizing for
4352	size, instead of ensuring O(log len) dynamic compares? */
4353
4354	bool
4355	try_store_by_multiple_pieces (rtx to, rtx len, unsigned int ctz_len,
4356	unsigned HOST_WIDE_INT min_len,
4357	unsigned HOST_WIDE_INT max_len,
4358	rtx val, char valc, unsigned int align)
4359	{
4360	int max_bits = floor_log2 (x: max_len);
4361	int min_bits = floor_log2 (x: min_len);
4362	int sctz_len = ctz_len;
4363
4364	gcc_checking_assert (sctz_len >= 0);
4365
4366	if (val)
4367	valc = 1;
4368
4369	/* Bits more significant than TST_BITS are part of the shared prefix
4370	in the binary representation of both min_len and max_len. Since
4371	they're identical, we don't need to test them in the loop. */
4372	int tst_bits = (max_bits != min_bits ? max_bits
4373	: floor_log2 (x: max_len ^ min_len));
4374
4375	/* Save the pre-blksize values. */
4376	int orig_max_bits = max_bits;
4377	int orig_tst_bits = tst_bits;
4378
4379	/* Check whether it's profitable to start by storing a fixed BLKSIZE
4380	bytes, to lower max_bits. In the unlikely case of a constant LEN
4381	(implied by identical MAX_LEN and MIN_LEN), we want to issue a
4382	single store_by_pieces, but otherwise, select the minimum multiple
4383	of the ALIGN (in bytes) and of the MCD of the possible LENs, that
4384	brings MAX_LEN below TST_BITS, if that's lower than min_len. */
4385	unsigned HOST_WIDE_INT blksize;
4386	if (max_len > min_len)
4387	{
4388	unsigned HOST_WIDE_INT alrng = MAX (HOST_WIDE_INT_1U << ctz_len,
4389	align / BITS_PER_UNIT);
4390	blksize = max_len - (HOST_WIDE_INT_1U << tst_bits) + alrng;
4391	blksize &= ~(alrng - 1);
4392	}
4393	else if (max_len == min_len)
4394	blksize = max_len;
4395	else
4396	/* Huh, max_len < min_len? Punt. See pr100843.c. */
4397	return false;
4398	if (min_len >= blksize
4399	/* ??? Maybe try smaller fixed-prefix blksizes before
4400	punting? */
4401	&& can_store_by_pieces (blksize, builtin_memset_read_str,
4402	&valc, align, true))
4403	{
4404	min_len -= blksize;
4405	min_bits = floor_log2 (x: min_len);
4406	max_len -= blksize;
4407	max_bits = floor_log2 (x: max_len);
4408
4409	tst_bits = (max_bits != min_bits ? max_bits
4410	: floor_log2 (x: max_len ^ min_len));
4411	}
4412	else
4413	blksize = 0;
4414
4415	/* Check that we can use store by pieces for the maximum store count
4416	we may issue (initial fixed-size block, plus conditional
4417	power-of-two-sized from max_bits to ctz_len. */
4418	unsigned HOST_WIDE_INT xlenest = blksize;
4419	if (max_bits >= 0)
4420	xlenest += ((HOST_WIDE_INT_1U << max_bits) * 2
4421	- (HOST_WIDE_INT_1U << ctz_len));
4422	bool max_loop = false;
4423	bool use_store_by_pieces = true;
4424	/* Skip the test in case of overflow in xlenest. It shouldn't
4425	happen because of the way max_bits and blksize are related, but
4426	it doesn't hurt to test. */
4427	if (blksize > xlenest
4428	|| !can_store_by_multiple_pieces (bits: xlenest - blksize,
4429	constfun: builtin_memset_read_str,
4430	constfundata: &valc, align, memsetp: true, len: blksize))
4431	{
4432	if (!(flag_inline_stringops & ILSOP_MEMSET))
4433	return false;
4434
4435	for (max_bits = orig_max_bits;
4436	max_bits >= sctz_len;
4437	--max_bits)
4438	{
4439	xlenest = ((HOST_WIDE_INT_1U << max_bits) * 2
4440	- (HOST_WIDE_INT_1U << ctz_len));
4441	/* Check that blksize plus the bits to be stored as blocks
4442	sized at powers of two can be stored by pieces. This is
4443	like the test above, but with smaller max_bits. Skip
4444	orig_max_bits (it would be redundant). Also skip in case
4445	of overflow. */
4446	if (max_bits < orig_max_bits
4447	&& xlenest + blksize >= xlenest
4448	&& can_store_by_multiple_pieces (bits: xlenest,
4449	constfun: builtin_memset_read_str,
4450	constfundata: &valc, align, memsetp: true, len: blksize))
4451	{
4452	max_loop = true;
4453	break;
4454	}
4455	if (blksize
4456	&& can_store_by_multiple_pieces (bits: xlenest,
4457	constfun: builtin_memset_read_str,
4458	constfundata: &valc, align, memsetp: true, len: 0))
4459	{
4460	max_len += blksize;
4461	min_len += blksize;
4462	tst_bits = orig_tst_bits;
4463	blksize = 0;
4464	max_loop = true;
4465	break;
4466	}
4467	if (max_bits == sctz_len)
4468	{
4469	/* We'll get here if can_store_by_pieces refuses to
4470	store even a single QImode. We'll fall back to
4471	QImode stores then. */
4472	if (!sctz_len)
4473	{
4474	blksize = 0;
4475	max_loop = true;
4476	use_store_by_pieces = false;
4477	break;
4478	}
4479	--sctz_len;
4480	--ctz_len;
4481	}
4482	}
4483	if (!max_loop)
4484	return false;
4485	/* If the boundaries are such that min and max may run a
4486	different number of trips in the initial loop, the remainder
4487	needs not be between the moduli, so set tst_bits to cover all
4488	bits. Otherwise, if the trip counts are the same, max_len
4489	has the common prefix, and the previously-computed tst_bits
4490	is usable. */
4491	if (max_len >> max_bits > min_len >> max_bits)
4492	tst_bits = max_bits;
4493	}
4494
4495	by_pieces_constfn constfun;
4496	void *constfundata;
4497	if (val)
4498	{
4499	constfun = builtin_memset_gen_str;
4500	constfundata = val = force_reg (TYPE_MODE (unsigned_char_type_node),
4501	val);
4502	}
4503	else
4504	{
4505	constfun = builtin_memset_read_str;
4506	constfundata = &valc;
4507	}
4508
4509	rtx ptr = copy_addr_to_reg (XEXP (to, 0));
4510	rtx rem = copy_to_mode_reg (ptr_mode, convert_to_mode (ptr_mode, len, 0));
4511	to = replace_equiv_address (to, ptr);
4512	set_mem_align (to, align);
4513
4514	if (blksize)
4515	{
4516	to = store_by_pieces (to, blksize,
4517	constfun, constfundata,
4518	align, true,
4519	max_len != 0 ? RETURN_END : RETURN_BEGIN);
4520	if (max_len == 0)
4521	return true;
4522
4523	/* Adjust PTR, TO and REM. Since TO's address is likely
4524	PTR+offset, we have to replace it. */
4525	emit_move_insn (ptr, force_operand (XEXP (to, 0), NULL_RTX));
4526	to = replace_equiv_address (to, ptr);
4527	rtx rem_minus_blksize = plus_constant (ptr_mode, rem, -blksize);
4528	emit_move_insn (rem, force_operand (rem_minus_blksize, NULL_RTX));
4529	}
4530
4531	/* Iterate over power-of-two block sizes from the maximum length to
4532	the least significant bit possibly set in the length. */
4533	for (int i = max_bits; i >= sctz_len; i--)
4534	{
4535	rtx_code_label *loop_label = NULL;
4536	rtx_code_label *label = NULL;
4537
4538	blksize = HOST_WIDE_INT_1U << i;
4539
4540	/* If we're past the bits shared between min_ and max_len, expand
4541	a test on the dynamic length, comparing it with the
4542	BLKSIZE. */
4543	if (i <= tst_bits)
4544	{
4545	label = gen_label_rtx ();
4546	emit_cmp_and_jump_insns (rem, GEN_INT (blksize), LT, NULL,
4547	ptr_mode, 1, label,
4548	prob: profile_probability::even ());
4549	}
4550	/* If we are at a bit that is in the prefix shared by min_ and
4551	max_len, skip the current BLKSIZE if the bit is clear, but do
4552	not skip the loop, even if it doesn't require
4553	prechecking. */
4554	else if ((max_len & blksize) == 0
4555	&& !(max_loop && i == max_bits))
4556	continue;
4557
4558	if (max_loop && i == max_bits)
4559	{
4560	loop_label = gen_label_rtx ();
4561	emit_label (loop_label);
4562	/* Since we may run this multiple times, don't assume we
4563	know anything about the offset. */
4564	clear_mem_offset (to);
4565	}
4566
4567	bool update_needed = i != sctz_len || loop_label;
4568	rtx next_ptr = NULL_RTX;
4569	if (!use_store_by_pieces)
4570	{
4571	gcc_checking_assert (blksize == 1);
4572	if (!val)
4573	val = gen_int_mode (valc, QImode);
4574	to = change_address (to, QImode, 0);
4575	emit_move_insn (to, val);
4576	if (update_needed)
4577	next_ptr = plus_constant (GET_MODE (ptr), ptr, blksize);
4578	}
4579	else
4580	{
4581	/* Issue a store of BLKSIZE bytes. */
4582	to = store_by_pieces (to, blksize,
4583	constfun, constfundata,
4584	align, true,
4585	update_needed ? RETURN_END : RETURN_BEGIN);
4586	next_ptr = XEXP (to, 0);
4587	}
4588	/* Adjust REM and PTR, unless this is the last iteration. */
4589	if (update_needed)
4590	{
4591	emit_move_insn (ptr, force_operand (next_ptr, NULL_RTX));
4592	to = replace_equiv_address (to, ptr);
4593	rtx rem_minus_blksize = plus_constant (ptr_mode, rem, -blksize);
4594	emit_move_insn (rem, force_operand (rem_minus_blksize, NULL_RTX));
4595	}
4596
4597	if (loop_label)
4598	emit_cmp_and_jump_insns (rem, GEN_INT (blksize), GE, NULL,
4599	ptr_mode, 1, loop_label,
4600	prob: profile_probability::likely ());
4601
4602	if (label)
4603	{
4604	emit_label (label);
4605
4606	/* Given conditional stores, the offset can no longer be
4607	known, so clear it. */
4608	clear_mem_offset (to);
4609	}
4610	}
4611
4612	return true;
4613	}
4614
4615	/* Helper function to do the actual work for expand_builtin_memset. The
4616	arguments to the builtin_memset call DEST, VAL, and LEN are broken out
4617	so that this can also be called without constructing an actual CALL_EXPR.
4618	The other arguments and return value are the same as for
4619	expand_builtin_memset. */
4620
4621	static rtx
4622	expand_builtin_memset_args (tree dest, tree val, tree len,
4623	rtx target, machine_mode mode, tree orig_exp)
4624	{
4625	tree fndecl, fn;
4626	enum built_in_function fcode;
4627	machine_mode val_mode;
4628	char c;
4629	unsigned int dest_align;
4630	rtx dest_mem, dest_addr, len_rtx;
4631	HOST_WIDE_INT expected_size = -1;
4632	unsigned int expected_align = 0;
4633	unsigned HOST_WIDE_INT min_size;
4634	unsigned HOST_WIDE_INT max_size;
4635	unsigned HOST_WIDE_INT probable_max_size;
4636
4637	dest_align = get_pointer_alignment (exp: dest);
4638
4639	/* If DEST is not a pointer type, don't do this operation in-line. */
4640	if (dest_align == 0)
4641	return NULL_RTX;
4642
4643	if (currently_expanding_gimple_stmt)
4644	stringop_block_profile (currently_expanding_gimple_stmt,
4645	&expected_align, &expected_size);
4646
4647	if (expected_align < dest_align)
4648	expected_align = dest_align;
4649
4650	/* If the LEN parameter is zero, return DEST. */
4651	if (integer_zerop (len))
4652	{
4653	/* Evaluate and ignore VAL in case it has side-effects. */
4654	expand_expr (exp: val, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
4655	return expand_expr (exp: dest, target, mode, modifier: EXPAND_NORMAL);
4656	}
4657
4658	/* Stabilize the arguments in case we fail. */
4659	dest = builtin_save_expr (exp: dest);
4660	val = builtin_save_expr (exp: val);
4661	len = builtin_save_expr (exp: len);
4662
4663	len_rtx = expand_normal (exp: len);
4664	determine_block_size (len, len_rtx, min_size: &min_size, max_size: &max_size,
4665	probable_max_size: &probable_max_size);
4666	dest_mem = get_memory_rtx (exp: dest, len);
4667	val_mode = TYPE_MODE (unsigned_char_type_node);
4668
4669	if (TREE_CODE (val) != INTEGER_CST
4670	|| target_char_cast (cst: val, p: &c))
4671	{
4672	rtx val_rtx;
4673
4674	val_rtx = expand_normal (exp: val);
4675	val_rtx = convert_to_mode (val_mode, val_rtx, 0);
4676
4677	/* Assume that we can memset by pieces if we can store
4678	* the coefficients by pieces (in the required modes).
4679	* We can't pass builtin_memset_gen_str as that emits RTL. */
4680	c = 1;
4681	if (tree_fits_uhwi_p (len)
4682	&& can_store_by_pieces (tree_to_uhwi (len),
4683	builtin_memset_read_str, &c, dest_align,
4684	true))
4685	{
4686	val_rtx = force_reg (val_mode, val_rtx);
4687	store_by_pieces (dest_mem, tree_to_uhwi (len),
4688	builtin_memset_gen_str, val_rtx, dest_align,
4689	true, RETURN_BEGIN);
4690	}
4691	else if (!set_storage_via_setmem (dest_mem, len_rtx, val_rtx,
4692	dest_align, expected_align,
4693	expected_size, min_size, max_size,
4694	probable_max_size)
4695	&& !try_store_by_multiple_pieces (to: dest_mem, len: len_rtx,
4696	ctz_len: tree_ctz (len),
4697	min_len: min_size, max_len: max_size,
4698	val: val_rtx, valc: 0,
4699	align: dest_align))
4700	goto do_libcall;
4701
4702	dest_mem = force_operand (XEXP (dest_mem, 0), NULL_RTX);
4703	dest_mem = convert_memory_address (ptr_mode, dest_mem);
4704	return dest_mem;
4705	}
4706
4707	if (c)
4708	{
4709	if (tree_fits_uhwi_p (len)
4710	&& can_store_by_pieces (tree_to_uhwi (len),
4711	builtin_memset_read_str, &c, dest_align,
4712	true))
4713	store_by_pieces (dest_mem, tree_to_uhwi (len),
4714	builtin_memset_read_str, &c, dest_align, true,
4715	RETURN_BEGIN);
4716	else if (!set_storage_via_setmem (dest_mem, len_rtx,
4717	gen_int_mode (c, val_mode),
4718	dest_align, expected_align,
4719	expected_size, min_size, max_size,
4720	probable_max_size)
4721	&& !try_store_by_multiple_pieces (to: dest_mem, len: len_rtx,
4722	ctz_len: tree_ctz (len),
4723	min_len: min_size, max_len: max_size,
4724	NULL_RTX, valc: c,
4725	align: dest_align))
4726	goto do_libcall;
4727
4728	dest_mem = force_operand (XEXP (dest_mem, 0), NULL_RTX);
4729	dest_mem = convert_memory_address (ptr_mode, dest_mem);
4730	return dest_mem;
4731	}
4732
4733	set_mem_align (dest_mem, dest_align);
4734	dest_addr = clear_storage_hints (dest_mem, len_rtx,
4735	CALL_EXPR_TAILCALL (orig_exp)
4736	? BLOCK_OP_TAILCALL : BLOCK_OP_NORMAL,
4737	expected_align, expected_size,
4738	min_size, max_size,
4739	probable_max_size, tree_ctz (len));
4740
4741	if (dest_addr == 0)
4742	{
4743	dest_addr = force_operand (XEXP (dest_mem, 0), NULL_RTX);
4744	dest_addr = convert_memory_address (ptr_mode, dest_addr);
4745	}
4746
4747	return dest_addr;
4748
4749	do_libcall:
4750	fndecl = get_callee_fndecl (orig_exp);
4751	fcode = DECL_FUNCTION_CODE (decl: fndecl);
4752	if (fcode == BUILT_IN_MEMSET)
4753	fn = build_call_nofold_loc (EXPR_LOCATION (orig_exp), fndecl, n: 3,
4754	dest, val, len);
4755	else if (fcode == BUILT_IN_BZERO)
4756	fn = build_call_nofold_loc (EXPR_LOCATION (orig_exp), fndecl, n: 2,
4757	dest, len);
4758	else
4759	gcc_unreachable ();
4760	gcc_assert (TREE_CODE (fn) == CALL_EXPR);
4761	CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (orig_exp);
4762	return expand_call (fn, target, target == const0_rtx);
4763	}
4764
4765	/* Expand expression EXP, which is a call to the bzero builtin. Return
4766	NULL_RTX if we failed the caller should emit a normal call. */
4767
4768	static rtx
4769	expand_builtin_bzero (tree exp)
4770	{
4771	if (!validate_arglist (callexpr: exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
4772	return NULL_RTX;
4773
4774	tree dest = CALL_EXPR_ARG (exp, 0);
4775	tree size = CALL_EXPR_ARG (exp, 1);
4776
4777	/* New argument list transforming bzero(ptr x, int y) to
4778	memset(ptr x, int 0, size_t y). This is done this way
4779	so that if it isn't expanded inline, we fallback to
4780	calling bzero instead of memset. */
4781
4782	location_t loc = EXPR_LOCATION (exp);
4783
4784	return expand_builtin_memset_args (dest, integer_zero_node,
4785	len: fold_convert_loc (loc,
4786	size_type_node, size),
4787	const0_rtx, VOIDmode, orig_exp: exp);
4788	}
4789
4790	/* Try to expand cmpstr operation ICODE with the given operands.
4791	Return the result rtx on success, otherwise return null. */
4792
4793	static rtx
4794	expand_cmpstr (insn_code icode, rtx target, rtx arg1_rtx, rtx arg2_rtx,
4795	HOST_WIDE_INT align)
4796	{
4797	machine_mode insn_mode = insn_data[icode].operand[0].mode;
4798
4799	if (target && (!REG_P (target) || HARD_REGISTER_P (target)))
4800	target = NULL_RTX;
4801
4802	class expand_operand ops[4];
4803	create_output_operand (op: &ops[0], x: target, mode: insn_mode);
4804	create_fixed_operand (op: &ops[1], x: arg1_rtx);
4805	create_fixed_operand (op: &ops[2], x: arg2_rtx);
4806	create_integer_operand (&ops[3], align);
4807	if (maybe_expand_insn (icode, nops: 4, ops))
4808	return ops[0].value;
4809	return NULL_RTX;
4810	}
4811
4812	/* Expand expression EXP, which is a call to the memcmp built-in function.
4813	Return NULL_RTX if we failed and the caller should emit a normal call,
4814	otherwise try to get the result in TARGET, if convenient.
4815	RESULT_EQ is true if we can relax the returned value to be either zero
4816	or nonzero, without caring about the sign. */
4817
4818	static rtx
4819	expand_builtin_memcmp (tree exp, rtx target, bool result_eq)
4820	{
4821	if (!validate_arglist (callexpr: exp,
4822	POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
4823	return NULL_RTX;
4824
4825	tree arg1 = CALL_EXPR_ARG (exp, 0);
4826	tree arg2 = CALL_EXPR_ARG (exp, 1);
4827	tree len = CALL_EXPR_ARG (exp, 2);
4828
4829	/* Due to the performance benefit, always inline the calls first
4830	when result_eq is false. */
4831	rtx result = NULL_RTX;
4832	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: get_callee_fndecl (exp));
4833	if (!result_eq && fcode != BUILT_IN_BCMP)
4834	{
4835	result = inline_expand_builtin_bytecmp (exp, target);
4836	if (result)
4837	return result;
4838	}
4839
4840	machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4841	location_t loc = EXPR_LOCATION (exp);
4842
4843	unsigned int arg1_align = get_pointer_alignment (exp: arg1) / BITS_PER_UNIT;
4844	unsigned int arg2_align = get_pointer_alignment (exp: arg2) / BITS_PER_UNIT;
4845
4846	/* If we don't have POINTER_TYPE, call the function. */
4847	if (arg1_align == 0 || arg2_align == 0)
4848	return NULL_RTX;
4849
4850	rtx arg1_rtx = get_memory_rtx (exp: arg1, len);
4851	rtx arg2_rtx = get_memory_rtx (exp: arg2, len);
4852	rtx len_rtx = expand_normal (exp: fold_convert_loc (loc, sizetype, len));
4853
4854	/* Set MEM_SIZE as appropriate. */
4855	if (CONST_INT_P (len_rtx))
4856	{
4857	set_mem_size (arg1_rtx, INTVAL (len_rtx));
4858	set_mem_size (arg2_rtx, INTVAL (len_rtx));
4859	}
4860
4861	by_pieces_constfn constfn = NULL;
4862
4863	/* Try to get the byte representation of the constant ARG2 (or, only
4864	when the function's result is used for equality to zero, ARG1)
4865	points to, with its byte size in NBYTES. */
4866	unsigned HOST_WIDE_INT nbytes;
4867	const char *rep = getbyterep (arg2, &nbytes);
4868	if (result_eq && rep == NULL)
4869	{
4870	/* For equality to zero the arguments are interchangeable. */
4871	rep = getbyterep (arg1, &nbytes);
4872	if (rep != NULL)
4873	std::swap (a&: arg1_rtx, b&: arg2_rtx);
4874	}
4875
4876	/* If the function's constant bound LEN_RTX is less than or equal
4877	to the byte size of the representation of the constant argument,
4878	and if block move would be done by pieces, we can avoid loading
4879	the bytes from memory and only store the computed constant result. */
4880	if (rep
4881	&& CONST_INT_P (len_rtx)
4882	&& (unsigned HOST_WIDE_INT) INTVAL (len_rtx) <= nbytes)
4883	constfn = builtin_memcpy_read_str;
4884
4885	result = emit_block_cmp_hints (arg1_rtx, arg2_rtx, len_rtx,
4886	TREE_TYPE (len), target,
4887	result_eq, constfn,
4888	CONST_CAST (char *, rep),
4889	ctz_len: tree_ctz (len));
4890
4891	if (result)
4892	{
4893	/* Return the value in the proper mode for this function. */
4894	if (GET_MODE (result) == mode)
4895	return result;
4896
4897	if (target != 0)
4898	{
4899	convert_move (target, result, 0);
4900	return target;
4901	}
4902
4903	return convert_to_mode (mode, result, 0);
4904	}
4905
4906	return NULL_RTX;
4907	}
4908
4909	/* Expand expression EXP, which is a call to the strcmp builtin. Return NULL_RTX
4910	if we failed the caller should emit a normal call, otherwise try to get
4911	the result in TARGET, if convenient. */
4912
4913	static rtx
4914	expand_builtin_strcmp (tree exp, ATTRIBUTE_UNUSED rtx target)
4915	{
4916	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
4917	return NULL_RTX;
4918
4919	tree arg1 = CALL_EXPR_ARG (exp, 0);
4920	tree arg2 = CALL_EXPR_ARG (exp, 1);
4921
4922	/* Due to the performance benefit, always inline the calls first. */
4923	rtx result = NULL_RTX;
4924	result = inline_expand_builtin_bytecmp (exp, target);
4925	if (result)
4926	return result;
4927
4928	insn_code cmpstr_icode = direct_optab_handler (op: cmpstr_optab, SImode);
4929	insn_code cmpstrn_icode = direct_optab_handler (op: cmpstrn_optab, SImode);
4930	if (cmpstr_icode == CODE_FOR_nothing && cmpstrn_icode == CODE_FOR_nothing)
4931	return NULL_RTX;
4932
4933	unsigned int arg1_align = get_pointer_alignment (exp: arg1) / BITS_PER_UNIT;
4934	unsigned int arg2_align = get_pointer_alignment (exp: arg2) / BITS_PER_UNIT;
4935
4936	/* If we don't have POINTER_TYPE, call the function. */
4937	if (arg1_align == 0 || arg2_align == 0)
4938	return NULL_RTX;
4939
4940	/* Stabilize the arguments in case gen_cmpstr(n)si fail. */
4941	arg1 = builtin_save_expr (exp: arg1);
4942	arg2 = builtin_save_expr (exp: arg2);
4943
4944	rtx arg1_rtx = get_memory_rtx (exp: arg1, NULL);
4945	rtx arg2_rtx = get_memory_rtx (exp: arg2, NULL);
4946
4947	/* Try to call cmpstrsi. */
4948	if (cmpstr_icode != CODE_FOR_nothing)
4949	result = expand_cmpstr (icode: cmpstr_icode, target, arg1_rtx, arg2_rtx,
4950	MIN (arg1_align, arg2_align));
4951
4952	/* Try to determine at least one length and call cmpstrnsi. */
4953	if (!result && cmpstrn_icode != CODE_FOR_nothing)
4954	{
4955	tree len;
4956	rtx arg3_rtx;
4957
4958	tree len1 = c_strlen (arg: arg1, only_value: 1);
4959	tree len2 = c_strlen (arg: arg2, only_value: 1);
4960
4961	if (len1)
4962	len1 = size_binop (PLUS_EXPR, ssize_int (1), len1);
4963	if (len2)
4964	len2 = size_binop (PLUS_EXPR, ssize_int (1), len2);
4965
4966	/* If we don't have a constant length for the first, use the length
4967	of the second, if we know it. We don't require a constant for
4968	this case; some cost analysis could be done if both are available
4969	but neither is constant. For now, assume they're equally cheap,
4970	unless one has side effects. If both strings have constant lengths,
4971	use the smaller. */
4972
4973	if (!len1)
4974	len = len2;
4975	else if (!len2)
4976	len = len1;
4977	else if (TREE_SIDE_EFFECTS (len1))
4978	len = len2;
4979	else if (TREE_SIDE_EFFECTS (len2))
4980	len = len1;
4981	else if (TREE_CODE (len1) != INTEGER_CST)
4982	len = len2;
4983	else if (TREE_CODE (len2) != INTEGER_CST)
4984	len = len1;
4985	else if (tree_int_cst_lt (t1: len1, t2: len2))
4986	len = len1;
4987	else
4988	len = len2;
4989
4990	/* If both arguments have side effects, we cannot optimize. */
4991	if (len && !TREE_SIDE_EFFECTS (len))
4992	{
4993	arg3_rtx = expand_normal (exp: len);
4994	result = expand_cmpstrn_or_cmpmem
4995	(cmpstrn_icode, target, arg1_rtx, arg2_rtx, TREE_TYPE (len),
4996	arg3_rtx, MIN (arg1_align, arg2_align));
4997	}
4998	}
4999
5000	tree fndecl = get_callee_fndecl (exp);
5001	if (result)
5002	{
5003	/* Return the value in the proper mode for this function. */
5004	machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
5005	if (GET_MODE (result) == mode)
5006	return result;
5007	if (target == 0)
5008	return convert_to_mode (mode, result, 0);
5009	convert_move (target, result, 0);
5010	return target;
5011	}
5012
5013	/* Expand the library call ourselves using a stabilized argument
5014	list to avoid re-evaluating the function's arguments twice. */
5015	tree fn = build_call_nofold_loc (EXPR_LOCATION (exp), fndecl, n: 2, arg1, arg2);
5016	copy_warning (fn, exp);
5017	gcc_assert (TREE_CODE (fn) == CALL_EXPR);
5018	CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (exp);
5019	return expand_call (fn, target, target == const0_rtx);
5020	}
5021
5022	/* Expand expression EXP, which is a call to the strncmp builtin. Return
5023	NULL_RTX if we failed the caller should emit a normal call, otherwise
5024	try to get the result in TARGET, if convenient. */
5025
5026	static rtx
5027	expand_builtin_strncmp (tree exp, ATTRIBUTE_UNUSED rtx target,
5028	ATTRIBUTE_UNUSED machine_mode mode)
5029	{
5030	if (!validate_arglist (callexpr: exp,
5031	POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
5032	return NULL_RTX;
5033
5034	tree arg1 = CALL_EXPR_ARG (exp, 0);
5035	tree arg2 = CALL_EXPR_ARG (exp, 1);
5036	tree arg3 = CALL_EXPR_ARG (exp, 2);
5037
5038	location_t loc = EXPR_LOCATION (exp);
5039	tree len1 = c_strlen (arg: arg1, only_value: 1);
5040	tree len2 = c_strlen (arg: arg2, only_value: 1);
5041
5042	/* Due to the performance benefit, always inline the calls first. */
5043	rtx result = NULL_RTX;
5044	result = inline_expand_builtin_bytecmp (exp, target);
5045	if (result)
5046	return result;
5047
5048	/* If c_strlen can determine an expression for one of the string
5049	lengths, and it doesn't have side effects, then emit cmpstrnsi
5050	using length MIN(strlen(string)+1, arg3). */
5051	insn_code cmpstrn_icode = direct_optab_handler (op: cmpstrn_optab, SImode);
5052	if (cmpstrn_icode == CODE_FOR_nothing)
5053	return NULL_RTX;
5054
5055	tree len;
5056
5057	unsigned int arg1_align = get_pointer_alignment (exp: arg1) / BITS_PER_UNIT;
5058	unsigned int arg2_align = get_pointer_alignment (exp: arg2) / BITS_PER_UNIT;
5059
5060	if (len1)
5061	len1 = size_binop_loc (loc, PLUS_EXPR, ssize_int (1), len1);
5062	if (len2)
5063	len2 = size_binop_loc (loc, PLUS_EXPR, ssize_int (1), len2);
5064
5065	tree len3 = fold_convert_loc (loc, sizetype, arg3);
5066
5067	/* If we don't have a constant length for the first, use the length
5068	of the second, if we know it. If neither string is constant length,
5069	use the given length argument. We don't require a constant for
5070	this case; some cost analysis could be done if both are available
5071	but neither is constant. For now, assume they're equally cheap,
5072	unless one has side effects. If both strings have constant lengths,
5073	use the smaller. */
5074
5075	if (!len1 && !len2)
5076	len = len3;
5077	else if (!len1)
5078	len = len2;
5079	else if (!len2)
5080	len = len1;
5081	else if (TREE_SIDE_EFFECTS (len1))
5082	len = len2;
5083	else if (TREE_SIDE_EFFECTS (len2))
5084	len = len1;
5085	else if (TREE_CODE (len1) != INTEGER_CST)
5086	len = len2;
5087	else if (TREE_CODE (len2) != INTEGER_CST)
5088	len = len1;
5089	else if (tree_int_cst_lt (t1: len1, t2: len2))
5090	len = len1;
5091	else
5092	len = len2;
5093
5094	/* If we are not using the given length, we must incorporate it here.
5095	The actual new length parameter will be MIN(len,arg3) in this case. */
5096	if (len != len3)
5097	{
5098	len = fold_convert_loc (loc, sizetype, len);
5099	len = fold_build2_loc (loc, MIN_EXPR, TREE_TYPE (len), len, len3);
5100	}
5101	rtx arg1_rtx = get_memory_rtx (exp: arg1, len);
5102	rtx arg2_rtx = get_memory_rtx (exp: arg2, len);
5103	rtx arg3_rtx = expand_normal (exp: len);
5104	result = expand_cmpstrn_or_cmpmem (cmpstrn_icode, target, arg1_rtx,
5105	arg2_rtx, TREE_TYPE (len), arg3_rtx,
5106	MIN (arg1_align, arg2_align));
5107
5108	tree fndecl = get_callee_fndecl (exp);
5109	if (result)
5110	{
5111	/* Return the value in the proper mode for this function. */
5112	mode = TYPE_MODE (TREE_TYPE (exp));
5113	if (GET_MODE (result) == mode)
5114	return result;
5115	if (target == 0)
5116	return convert_to_mode (mode, result, 0);
5117	convert_move (target, result, 0);
5118	return target;
5119	}
5120
5121	/* Expand the library call ourselves using a stabilized argument
5122	list to avoid re-evaluating the function's arguments twice. */
5123	tree call = build_call_nofold_loc (loc, fndecl, n: 3, arg1, arg2, len);
5124	copy_warning (call, exp);
5125	gcc_assert (TREE_CODE (call) == CALL_EXPR);
5126	CALL_EXPR_TAILCALL (call) = CALL_EXPR_TAILCALL (exp);
5127	return expand_call (call, target, target == const0_rtx);
5128	}
5129
5130	/* Expand a call to __builtin_saveregs, generating the result in TARGET,
5131	if that's convenient. */
5132
5133	rtx
5134	expand_builtin_saveregs (void)
5135	{
5136	rtx val;
5137	rtx_insn *seq;
5138
5139	/* Don't do __builtin_saveregs more than once in a function.
5140	Save the result of the first call and reuse it. */
5141	if (saveregs_value != 0)
5142	return saveregs_value;
5143
5144	/* When this function is called, it means that registers must be
5145	saved on entry to this function. So we migrate the call to the
5146	first insn of this function. */
5147
5148	start_sequence ();
5149
5150	/* Do whatever the machine needs done in this case. */
5151	val = targetm.calls.expand_builtin_saveregs ();
5152
5153	seq = get_insns ();
5154	end_sequence ();
5155
5156	saveregs_value = val;
5157
5158	/* Put the insns after the NOTE that starts the function. If this
5159	is inside a start_sequence, make the outer-level insn chain current, so
5160	the code is placed at the start of the function. */
5161	push_topmost_sequence ();
5162	emit_insn_after (seq, entry_of_function ());
5163	pop_topmost_sequence ();
5164
5165	return val;
5166	}
5167
5168	/* Expand a call to __builtin_next_arg. */
5169
5170	static rtx
5171	expand_builtin_next_arg (void)
5172	{
5173	/* Checking arguments is already done in fold_builtin_next_arg
5174	that must be called before this function. */
5175	return expand_binop (ptr_mode, add_optab,
5176	crtl->args.internal_arg_pointer,
5177	crtl->args.arg_offset_rtx,
5178	NULL_RTX, 0, OPTAB_LIB_WIDEN);
5179	}
5180
5181	/* Make it easier for the backends by protecting the valist argument
5182	from multiple evaluations. */
5183
5184	static tree
5185	stabilize_va_list_loc (location_t loc, tree valist, int needs_lvalue)
5186	{
5187	tree vatype = targetm.canonical_va_list_type (TREE_TYPE (valist));
5188
5189	/* The current way of determining the type of valist is completely
5190	bogus. We should have the information on the va builtin instead. */
5191	if (!vatype)
5192	vatype = targetm.fn_abi_va_list (cfun->decl);
5193
5194	if (TREE_CODE (vatype) == ARRAY_TYPE)
5195	{
5196	if (TREE_SIDE_EFFECTS (valist))
5197	valist = save_expr (valist);
5198
5199	/* For this case, the backends will be expecting a pointer to
5200	vatype, but it's possible we've actually been given an array
5201	(an actual TARGET_CANONICAL_VA_LIST_TYPE (valist)).
5202	So fix it. */
5203	if (TREE_CODE (TREE_TYPE (valist)) == ARRAY_TYPE)
5204	{
5205	tree p1 = build_pointer_type (TREE_TYPE (vatype));
5206	valist = build_fold_addr_expr_with_type_loc (loc, valist, p1);
5207	}
5208	}
5209	else
5210	{
5211	tree pt = build_pointer_type (vatype);
5212
5213	if (! needs_lvalue)
5214	{
5215	if (! TREE_SIDE_EFFECTS (valist))
5216	return valist;
5217
5218	valist = fold_build1_loc (loc, ADDR_EXPR, pt, valist);
5219	TREE_SIDE_EFFECTS (valist) = 1;
5220	}
5221
5222	if (TREE_SIDE_EFFECTS (valist))
5223	valist = save_expr (valist);
5224	valist = fold_build2_loc (loc, MEM_REF,
5225	vatype, valist, build_int_cst (pt, 0));
5226	}
5227
5228	return valist;
5229	}
5230
5231	/* The "standard" definition of va_list is void*. */
5232
5233	tree
5234	std_build_builtin_va_list (void)
5235	{
5236	return ptr_type_node;
5237	}
5238
5239	/* The "standard" abi va_list is va_list_type_node. */
5240
5241	tree
5242	std_fn_abi_va_list (tree fndecl ATTRIBUTE_UNUSED)
5243	{
5244	return va_list_type_node;
5245	}
5246
5247	/* The "standard" type of va_list is va_list_type_node. */
5248
5249	tree
5250	std_canonical_va_list_type (tree type)
5251	{
5252	tree wtype, htype;
5253
5254	wtype = va_list_type_node;
5255	htype = type;
5256
5257	if (TREE_CODE (wtype) == ARRAY_TYPE)
5258	{
5259	/* If va_list is an array type, the argument may have decayed
5260	to a pointer type, e.g. by being passed to another function.
5261	In that case, unwrap both types so that we can compare the
5262	underlying records. */
5263	if (TREE_CODE (htype) == ARRAY_TYPE
5264	|| POINTER_TYPE_P (htype))
5265	{
5266	wtype = TREE_TYPE (wtype);
5267	htype = TREE_TYPE (htype);
5268	}
5269	}
5270	if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
5271	return va_list_type_node;
5272
5273	return NULL_TREE;
5274	}
5275
5276	/* The "standard" implementation of va_start: just assign `nextarg' to
5277	the variable. */
5278
5279	void
5280	std_expand_builtin_va_start (tree valist, rtx nextarg)
5281	{
5282	rtx va_r = expand_expr (exp: valist, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5283	convert_move (va_r, nextarg, 0);
5284	}
5285
5286	/* Expand EXP, a call to __builtin_va_start. */
5287
5288	static rtx
5289	expand_builtin_va_start (tree exp)
5290	{
5291	rtx nextarg;
5292	tree valist;
5293	location_t loc = EXPR_LOCATION (exp);
5294
5295	if (call_expr_nargs (exp) < 2)
5296	{
5297	error_at (loc, "too few arguments to function %<va_start%>");
5298	return const0_rtx;
5299	}
5300
5301	if (fold_builtin_next_arg (exp, true))
5302	return const0_rtx;
5303
5304	nextarg = expand_builtin_next_arg ();
5305	valist = stabilize_va_list_loc (loc, CALL_EXPR_ARG (exp, 0), needs_lvalue: 1);
5306
5307	if (targetm.expand_builtin_va_start)
5308	targetm.expand_builtin_va_start (valist, nextarg);
5309	else
5310	std_expand_builtin_va_start (valist, nextarg);
5311
5312	return const0_rtx;
5313	}
5314
5315	/* Expand EXP, a call to __builtin_va_end. */
5316
5317	static rtx
5318	expand_builtin_va_end (tree exp)
5319	{
5320	tree valist = CALL_EXPR_ARG (exp, 0);
5321
5322	/* Evaluate for side effects, if needed. I hate macros that don't
5323	do that. */
5324	if (TREE_SIDE_EFFECTS (valist))
5325	expand_expr (exp: valist, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
5326
5327	return const0_rtx;
5328	}
5329
5330	/* Expand EXP, a call to __builtin_va_copy. We do this as a
5331	builtin rather than just as an assignment in stdarg.h because of the
5332	nastiness of array-type va_list types. */
5333
5334	static rtx
5335	expand_builtin_va_copy (tree exp)
5336	{
5337	tree dst, src, t;
5338	location_t loc = EXPR_LOCATION (exp);
5339
5340	dst = CALL_EXPR_ARG (exp, 0);
5341	src = CALL_EXPR_ARG (exp, 1);
5342
5343	dst = stabilize_va_list_loc (loc, valist: dst, needs_lvalue: 1);
5344	src = stabilize_va_list_loc (loc, valist: src, needs_lvalue: 0);
5345
5346	gcc_assert (cfun != NULL && cfun->decl != NULL_TREE);
5347
5348	if (TREE_CODE (targetm.fn_abi_va_list (cfun->decl)) != ARRAY_TYPE)
5349	{
5350	t = build2 (MODIFY_EXPR, targetm.fn_abi_va_list (cfun->decl), dst, src);
5351	TREE_SIDE_EFFECTS (t) = 1;
5352	expand_expr (exp: t, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
5353	}
5354	else
5355	{
5356	rtx dstb, srcb, size;
5357
5358	/* Evaluate to pointers. */
5359	dstb = expand_expr (exp: dst, NULL_RTX, Pmode, modifier: EXPAND_NORMAL);
5360	srcb = expand_expr (exp: src, NULL_RTX, Pmode, modifier: EXPAND_NORMAL);
5361	size = expand_expr (TYPE_SIZE_UNIT (targetm.fn_abi_va_list (cfun->decl)),
5362	NULL_RTX, VOIDmode, modifier: EXPAND_NORMAL);
5363
5364	dstb = convert_memory_address (Pmode, dstb);
5365	srcb = convert_memory_address (Pmode, srcb);
5366
5367	/* "Dereference" to BLKmode memories. */
5368	dstb = gen_rtx_MEM (BLKmode, dstb);
5369	set_mem_alias_set (dstb, get_alias_set (TREE_TYPE (TREE_TYPE (dst))));
5370	set_mem_align (dstb, TYPE_ALIGN (targetm.fn_abi_va_list (cfun->decl)));
5371	srcb = gen_rtx_MEM (BLKmode, srcb);
5372	set_mem_alias_set (srcb, get_alias_set (TREE_TYPE (TREE_TYPE (src))));
5373	set_mem_align (srcb, TYPE_ALIGN (targetm.fn_abi_va_list (cfun->decl)));
5374
5375	/* Copy. */
5376	emit_block_move (dstb, srcb, size, BLOCK_OP_NORMAL);
5377	}
5378
5379	return const0_rtx;
5380	}
5381
5382	/* Expand a call to one of the builtin functions __builtin_frame_address or
5383	__builtin_return_address. */
5384
5385	static rtx
5386	expand_builtin_frame_address (tree fndecl, tree exp)
5387	{
5388	/* The argument must be a nonnegative integer constant.
5389	It counts the number of frames to scan up the stack.
5390	The value is either the frame pointer value or the return
5391	address saved in that frame. */
5392	if (call_expr_nargs (exp) == 0)
5393	/* Warning about missing arg was already issued. */
5394	return const0_rtx;
5395	else if (! tree_fits_uhwi_p (CALL_EXPR_ARG (exp, 0)))
5396	{
5397	error ("invalid argument to %qD", fndecl);
5398	return const0_rtx;
5399	}
5400	else
5401	{
5402	/* Number of frames to scan up the stack. */
5403	unsigned HOST_WIDE_INT count = tree_to_uhwi (CALL_EXPR_ARG (exp, 0));
5404
5405	rtx tem = expand_builtin_return_addr (fndecl_code: DECL_FUNCTION_CODE (decl: fndecl), count);
5406
5407	/* Some ports cannot access arbitrary stack frames. */
5408	if (tem == NULL)
5409	{
5410	warning (0, "unsupported argument to %qD", fndecl);
5411	return const0_rtx;
5412	}
5413
5414	if (count)
5415	{
5416	/* Warn since no effort is made to ensure that any frame
5417	beyond the current one exists or can be safely reached. */
5418	warning (OPT_Wframe_address, "calling %qD with "
5419	"a nonzero argument is unsafe", fndecl);
5420	}
5421
5422	/* For __builtin_frame_address, return what we've got. */
5423	if (DECL_FUNCTION_CODE (decl: fndecl) == BUILT_IN_FRAME_ADDRESS)
5424	return tem;
5425
5426	if (!REG_P (tem)
5427	&& ! CONSTANT_P (tem))
5428	tem = copy_addr_to_reg (tem);
5429	return tem;
5430	}
5431	}
5432
5433	#if ! STACK_GROWS_DOWNWARD
5434	# define STACK_TOPS GT
5435	#else
5436	# define STACK_TOPS LT
5437	#endif
5438
5439	#ifdef POINTERS_EXTEND_UNSIGNED
5440	# define STACK_UNSIGNED POINTERS_EXTEND_UNSIGNED
5441	#else
5442	# define STACK_UNSIGNED true
5443	#endif
5444
5445	/* Expand a call to builtin function __builtin_stack_address. */
5446
5447	static rtx
5448	expand_builtin_stack_address ()
5449	{
5450	rtx ret = convert_to_mode (ptr_mode, copy_to_reg (stack_pointer_rtx),
5451	STACK_UNSIGNED);
5452
5453	#ifdef STACK_ADDRESS_OFFSET
5454	/* Unbias the stack pointer, bringing it to the boundary between the
5455	stack area claimed by the active function calling this builtin,
5456	and stack ranges that could get clobbered if it called another
5457	function. It should NOT encompass any stack red zone, that is
5458	used in leaf functions.
5459
5460	On SPARC, the register save area is *not* considered active or
5461	used by the active function, but rather as akin to the area in
5462	which call-preserved registers are saved by callees. This
5463	enables __strub_leave to clear what would otherwise overlap with
5464	its own register save area.
5465
5466	If the address is computed too high or too low, parts of a stack
5467	range that should be scrubbed may be left unscrubbed, scrubbing
5468	may corrupt active portions of the stack frame, and stack ranges
5469	may be doubly-scrubbed by caller and callee.
5470
5471	In order for it to be just right, the area delimited by
5472	@code{__builtin_stack_address} and @code{__builtin_frame_address
5473	(0)} should encompass caller's registers saved by the function,
5474	local on-stack variables and @code{alloca} stack areas.
5475	Accumulated outgoing on-stack arguments, preallocated as part of
5476	a function's own prologue, are to be regarded as part of the
5477	(caller) function's active area as well, whereas those pushed or
5478	allocated temporarily for a call are regarded as part of the
5479	callee's stack range, rather than the caller's. */
5480	ret = plus_constant (ptr_mode, ret, STACK_ADDRESS_OFFSET);
5481	#endif
5482
5483	return force_reg (ptr_mode, ret);
5484	}
5485
5486	/* Expand a call to builtin function __builtin_strub_enter. */
5487
5488	static rtx
5489	expand_builtin_strub_enter (tree exp)
5490	{
5491	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
5492	return NULL_RTX;
5493
5494	if (optimize < 1 || flag_no_inline)
5495	return NULL_RTX;
5496
5497	rtx stktop = expand_builtin_stack_address ();
5498
5499	tree wmptr = CALL_EXPR_ARG (exp, 0);
5500	tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
5501	tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
5502	build_int_cst (TREE_TYPE (wmptr), 0));
5503	rtx wmark = expand_expr (exp: wmtree, NULL_RTX, mode: ptr_mode, modifier: EXPAND_MEMORY);
5504
5505	emit_move_insn (wmark, stktop);
5506
5507	return const0_rtx;
5508	}
5509
5510	/* Expand a call to builtin function __builtin_strub_update. */
5511
5512	static rtx
5513	expand_builtin_strub_update (tree exp)
5514	{
5515	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
5516	return NULL_RTX;
5517
5518	if (optimize < 2 || flag_no_inline)
5519	return NULL_RTX;
5520
5521	rtx stktop = expand_builtin_stack_address ();
5522
5523	#ifdef RED_ZONE_SIZE
5524	/* Here's how the strub enter, update and leave functions deal with red zones.
5525
5526	If it weren't for red zones, update, called from within a strub context,
5527	would bump the watermark to the top of the stack. Enter and leave, running
5528	in the caller, would use the caller's top of stack address both to
5529	initialize the watermark passed to the callee, and to start strubbing the
5530	stack afterwards.
5531
5532	Ideally, we'd update the watermark so as to cover the used amount of red
5533	zone, and strub starting at the caller's other end of the (presumably
5534	unused) red zone. Normally, only leaf functions use the red zone, but at
5535	this point we can't tell whether a function is a leaf, nor can we tell how
5536	much of the red zone it uses. Furthermore, some strub contexts may have
5537	been inlined so that update and leave are called from the same stack frame,
5538	and the strub builtins may all have been inlined, turning a strub function
5539	into a leaf.
5540
5541	So cleaning the range from the caller's stack pointer (one end of the red
5542	zone) to the (potentially inlined) callee's (other end of the) red zone
5543	could scribble over the caller's own red zone.
5544
5545	We avoid this possibility by arranging for callers that are strub contexts
5546	to use their own watermark as the strub starting point. So, if A calls B,
5547	and B calls C, B will tell A to strub up to the end of B's red zone, and
5548	will strub itself only the part of C's stack frame and red zone that
5549	doesn't overlap with B's. With that, we don't need to know who's leaf and
5550	who isn't: inlined calls will shrink their strub window to zero, each
5551	remaining call will strub some portion of the stack, and eventually the
5552	strub context will return to a caller that isn't a strub context itself,
5553	that will therefore use its own stack pointer as the strub starting point.
5554	It's not a leaf, because strub contexts can't be inlined into non-strub
5555	contexts, so it doesn't use the red zone, and it will therefore correctly
5556	strub up the callee's stack frame up to the end of the callee's red zone.
5557	Neat! */
5558	if (true /* (flags_from_decl_or_type (current_function_decl) & ECF_LEAF) */)
5559	{
5560	poly_int64 red_zone_size = RED_ZONE_SIZE;
5561	#if STACK_GROWS_DOWNWARD
5562	red_zone_size = -red_zone_size;
5563	#endif
5564	stktop = plus_constant (ptr_mode, stktop, red_zone_size);
5565	stktop = force_reg (ptr_mode, stktop);
5566	}
5567	#endif
5568
5569	tree wmptr = CALL_EXPR_ARG (exp, 0);
5570	tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
5571	tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
5572	build_int_cst (TREE_TYPE (wmptr), 0));
5573	rtx wmark = expand_expr (exp: wmtree, NULL_RTX, mode: ptr_mode, modifier: EXPAND_MEMORY);
5574
5575	rtx wmarkr = force_reg (ptr_mode, wmark);
5576
5577	rtx_code_label *lab = gen_label_rtx ();
5578	do_compare_rtx_and_jump (stktop, wmarkr, STACK_TOPS, STACK_UNSIGNED,
5579	ptr_mode, NULL_RTX, lab, NULL,
5580	profile_probability::very_likely ());
5581	emit_move_insn (wmark, stktop);
5582
5583	/* If this is an inlined strub function, also bump the watermark for the
5584	enclosing function. This avoids a problem with the following scenario: A
5585	calls B and B calls C, and both B and C get inlined into A. B allocates
5586	temporary stack space before calling C. If we don't update A's watermark,
5587	we may use an outdated baseline for the post-C strub_leave, erasing B's
5588	temporary stack allocation. We only need this if we're fully expanding
5589	strub_leave inline. */
5590	tree xwmptr = (optimize > 2
5591	? strub_watermark_parm (fndecl: current_function_decl)
5592	: wmptr);
5593	if (wmptr != xwmptr)
5594	{
5595	wmptr = xwmptr;
5596	wmtype = TREE_TYPE (TREE_TYPE (wmptr));
5597	wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
5598	build_int_cst (TREE_TYPE (wmptr), 0));
5599	wmark = expand_expr (exp: wmtree, NULL_RTX, mode: ptr_mode, modifier: EXPAND_MEMORY);
5600	wmarkr = force_reg (ptr_mode, wmark);
5601
5602	do_compare_rtx_and_jump (stktop, wmarkr, STACK_TOPS, STACK_UNSIGNED,
5603	ptr_mode, NULL_RTX, lab, NULL,
5604	profile_probability::very_likely ());
5605	emit_move_insn (wmark, stktop);
5606	}
5607
5608	emit_label (lab);
5609
5610	return const0_rtx;
5611	}
5612
5613
5614	/* Expand a call to builtin function __builtin_strub_leave. */
5615
5616	static rtx
5617	expand_builtin_strub_leave (tree exp)
5618	{
5619	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
5620	return NULL_RTX;
5621
5622	if (optimize < 2 || optimize_size || flag_no_inline)
5623	return NULL_RTX;
5624
5625	rtx stktop = NULL_RTX;
5626
5627	if (tree wmptr = (optimize
5628	? strub_watermark_parm (fndecl: current_function_decl)
5629	: NULL_TREE))
5630	{
5631	tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
5632	tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
5633	build_int_cst (TREE_TYPE (wmptr), 0));
5634	rtx wmark = expand_expr (exp: wmtree, NULL_RTX, mode: ptr_mode, modifier: EXPAND_MEMORY);
5635	stktop = force_reg (ptr_mode, wmark);
5636	}
5637
5638	if (!stktop)
5639	stktop = expand_builtin_stack_address ();
5640
5641	tree wmptr = CALL_EXPR_ARG (exp, 0);
5642	tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
5643	tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
5644	build_int_cst (TREE_TYPE (wmptr), 0));
5645	rtx wmark = expand_expr (exp: wmtree, NULL_RTX, mode: ptr_mode, modifier: EXPAND_MEMORY);
5646
5647	rtx wmarkr = force_reg (ptr_mode, wmark);
5648
5649	#if ! STACK_GROWS_DOWNWARD
5650	rtx base = stktop;
5651	rtx end = wmarkr;
5652	#else
5653	rtx base = wmarkr;
5654	rtx end = stktop;
5655	#endif
5656
5657	/* We're going to modify it, so make sure it's not e.g. the stack pointer. */
5658	base = copy_to_reg (base);
5659
5660	rtx_code_label *done = gen_label_rtx ();
5661	do_compare_rtx_and_jump (base, end, LT, STACK_UNSIGNED,
5662	ptr_mode, NULL_RTX, done, NULL,
5663	profile_probability::very_likely ());
5664
5665	if (optimize < 3)
5666	expand_call (exp, NULL_RTX, true);
5667	else
5668	{
5669	/* Ok, now we've determined we want to copy the block, so convert the
5670	addresses to Pmode, as needed to dereference them to access ptr_mode
5671	memory locations, so that we don't have to convert anything within the
5672	loop. */
5673	base = memory_address (ptr_mode, base);
5674	end = memory_address (ptr_mode, end);
5675
5676	rtx zero = force_operand (const0_rtx, NULL_RTX);
5677	int ulen = GET_MODE_SIZE (mode: ptr_mode);
5678
5679	/* ??? It would be nice to use setmem or similar patterns here,
5680	but they do not necessarily obey the stack growth direction,
5681	which has security implications. We also have to avoid calls
5682	(memset, bzero or any machine-specific ones), which are
5683	likely unsafe here (see TARGET_STRUB_MAY_USE_MEMSET). */
5684	#if ! STACK_GROWS_DOWNWARD
5685	rtx incr = plus_constant (Pmode, base, ulen);
5686	rtx dstm = gen_rtx_MEM (ptr_mode, base);
5687
5688	rtx_code_label *loop = gen_label_rtx ();
5689	emit_label (loop);
5690	emit_move_insn (dstm, zero);
5691	emit_move_insn (base, force_operand (incr, NULL_RTX));
5692	#else
5693	rtx decr = plus_constant (Pmode, end, -ulen);
5694	rtx dstm = gen_rtx_MEM (ptr_mode, end);
5695
5696	rtx_code_label *loop = gen_label_rtx ();
5697	emit_label (loop);
5698	emit_move_insn (end, force_operand (decr, NULL_RTX));
5699	emit_move_insn (dstm, zero);
5700	#endif
5701	do_compare_rtx_and_jump (base, end, LT, STACK_UNSIGNED,
5702	Pmode, NULL_RTX, NULL, loop,
5703	profile_probability::very_likely ());
5704	}
5705
5706	emit_label (done);
5707
5708	return const0_rtx;
5709	}
5710
5711	/* Expand EXP, a call to the alloca builtin. Return NULL_RTX if we
5712	failed and the caller should emit a normal call. */
5713
5714	static rtx
5715	expand_builtin_alloca (tree exp)
5716	{
5717	rtx op0;
5718	rtx result;
5719	unsigned int align;
5720	tree fndecl = get_callee_fndecl (exp);
5721	HOST_WIDE_INT max_size;
5722	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
5723	bool alloca_for_var = CALL_ALLOCA_FOR_VAR_P (exp);
5724	bool valid_arglist
5725	= (fcode == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
5726	? validate_arglist (callexpr: exp, INTEGER_TYPE, INTEGER_TYPE, INTEGER_TYPE,
5727	VOID_TYPE)
5728	: fcode == BUILT_IN_ALLOCA_WITH_ALIGN
5729	? validate_arglist (callexpr: exp, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE)
5730	: validate_arglist (callexpr: exp, INTEGER_TYPE, VOID_TYPE));
5731
5732	if (!valid_arglist)
5733	return NULL_RTX;
5734
5735	/* Compute the argument. */
5736	op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
5737
5738	/* Compute the alignment. */
5739	align = (fcode == BUILT_IN_ALLOCA
5740	? BIGGEST_ALIGNMENT
5741	: TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 1)));
5742
5743	/* Compute the maximum size. */
5744	max_size = (fcode == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
5745	? TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 2))
5746	: -1);
5747
5748	/* Allocate the desired space. If the allocation stems from the declaration
5749	of a variable-sized object, it cannot accumulate. */
5750	result
5751	= allocate_dynamic_stack_space (op0, 0, align, max_size, alloca_for_var);
5752	result = convert_memory_address (ptr_mode, result);
5753
5754	/* Dynamic allocations for variables are recorded during gimplification. */
5755	if (!alloca_for_var && (flag_callgraph_info & CALLGRAPH_INFO_DYNAMIC_ALLOC))
5756	record_dynamic_alloc (decl_or_exp: exp);
5757
5758	return result;
5759	}
5760
5761	/* Emit a call to __asan_allocas_unpoison call in EXP. Add to second argument
5762	of the call virtual_stack_dynamic_rtx - stack_pointer_rtx, which is the
5763	STACK_DYNAMIC_OFFSET value. See motivation for this in comment to
5764	handle_builtin_stack_restore function. */
5765
5766	static rtx
5767	expand_asan_emit_allocas_unpoison (tree exp)
5768	{
5769	tree arg0 = CALL_EXPR_ARG (exp, 0);
5770	tree arg1 = CALL_EXPR_ARG (exp, 1);
5771	rtx top = expand_expr (exp: arg0, NULL_RTX, mode: ptr_mode, modifier: EXPAND_NORMAL);
5772	rtx bot = expand_expr (exp: arg1, NULL_RTX, mode: ptr_mode, modifier: EXPAND_NORMAL);
5773	rtx off = expand_simple_binop (Pmode, MINUS, virtual_stack_dynamic_rtx,
5774	stack_pointer_rtx, NULL_RTX, 0,
5775	OPTAB_LIB_WIDEN);
5776	off = convert_modes (mode: ptr_mode, Pmode, x: off, unsignedp: 0);
5777	bot = expand_simple_binop (ptr_mode, PLUS, bot, off, NULL_RTX, 0,
5778	OPTAB_LIB_WIDEN);
5779	rtx ret = init_one_libfunc ("__asan_allocas_unpoison");
5780	ret = emit_library_call_value (fun: ret, NULL_RTX, fn_type: LCT_NORMAL, outmode: ptr_mode,
5781	arg1: top, arg1_mode: ptr_mode, arg2: bot, arg2_mode: ptr_mode);
5782	return ret;
5783	}
5784
5785	/* Expand a call to bswap builtin in EXP.
5786	Return NULL_RTX if a normal call should be emitted rather than expanding the
5787	function in-line. If convenient, the result should be placed in TARGET.
5788	SUBTARGET may be used as the target for computing one of EXP's operands. */
5789
5790	static rtx
5791	expand_builtin_bswap (machine_mode target_mode, tree exp, rtx target,
5792	rtx subtarget)
5793	{
5794	tree arg;
5795	rtx op0;
5796
5797	if (!validate_arglist (callexpr: exp, INTEGER_TYPE, VOID_TYPE))
5798	return NULL_RTX;
5799
5800	arg = CALL_EXPR_ARG (exp, 0);
5801	op0 = expand_expr (exp: arg,
5802	target: subtarget && GET_MODE (subtarget) == target_mode
5803	? subtarget : NULL_RTX,
5804	mode: target_mode, modifier: EXPAND_NORMAL);
5805	if (GET_MODE (op0) != target_mode)
5806	op0 = convert_to_mode (target_mode, op0, 1);
5807
5808	target = expand_unop (target_mode, bswap_optab, op0, target, 1);
5809
5810	gcc_assert (target);
5811
5812	return convert_to_mode (target_mode, target, 1);
5813	}
5814
5815	/* Expand a call to a unary builtin in EXP.
5816	Return NULL_RTX if a normal call should be emitted rather than expanding the
5817	function in-line. If convenient, the result should be placed in TARGET.
5818	SUBTARGET may be used as the target for computing one of EXP's operands. */
5819
5820	static rtx
5821	expand_builtin_unop (machine_mode target_mode, tree exp, rtx target,
5822	rtx subtarget, optab op_optab)
5823	{
5824	rtx op0;
5825
5826	if (!validate_arglist (callexpr: exp, INTEGER_TYPE, VOID_TYPE))
5827	return NULL_RTX;
5828
5829	/* Compute the argument. */
5830	op0 = expand_expr (CALL_EXPR_ARG (exp, 0),
5831	target: (subtarget
5832	&& (TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 0)))
5833	== GET_MODE (subtarget))) ? subtarget : NULL_RTX,
5834	VOIDmode, modifier: EXPAND_NORMAL);
5835	/* Compute op, into TARGET if possible.
5836	Set TARGET to wherever the result comes back. */
5837	target = expand_unop (TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 0))),
5838	op_optab, op0, target, op_optab != clrsb_optab);
5839	gcc_assert (target);
5840
5841	return convert_to_mode (target_mode, target, 0);
5842	}
5843
5844	/* Expand a call to __builtin_expect. We just return our argument
5845	as the builtin_expect semantic should've been already executed by
5846	tree branch prediction pass. */
5847
5848	static rtx
5849	expand_builtin_expect (tree exp, rtx target)
5850	{
5851	tree arg;
5852
5853	if (call_expr_nargs (exp) < 2)
5854	return const0_rtx;
5855	arg = CALL_EXPR_ARG (exp, 0);
5856
5857	target = expand_expr (exp: arg, target, VOIDmode, modifier: EXPAND_NORMAL);
5858	/* When guessing was done, the hints should be already stripped away. */
5859	gcc_assert (!flag_guess_branch_prob
5860	|| optimize == 0 || seen_error ());
5861	return target;
5862	}
5863
5864	/* Expand a call to __builtin_expect_with_probability. We just return our
5865	argument as the builtin_expect semantic should've been already executed by
5866	tree branch prediction pass. */
5867
5868	static rtx
5869	expand_builtin_expect_with_probability (tree exp, rtx target)
5870	{
5871	tree arg;
5872
5873	if (call_expr_nargs (exp) < 3)
5874	return const0_rtx;
5875	arg = CALL_EXPR_ARG (exp, 0);
5876
5877	target = expand_expr (exp: arg, target, VOIDmode, modifier: EXPAND_NORMAL);
5878	/* When guessing was done, the hints should be already stripped away. */
5879	gcc_assert (!flag_guess_branch_prob
5880	|| optimize == 0 || seen_error ());
5881	return target;
5882	}
5883
5884
5885	/* Expand a call to __builtin_assume_aligned. We just return our first
5886	argument as the builtin_assume_aligned semantic should've been already
5887	executed by CCP. */
5888
5889	static rtx
5890	expand_builtin_assume_aligned (tree exp, rtx target)
5891	{
5892	if (call_expr_nargs (exp) < 2)
5893	return const0_rtx;
5894	target = expand_expr (CALL_EXPR_ARG (exp, 0), target, VOIDmode,
5895	modifier: EXPAND_NORMAL);
5896	gcc_assert (!TREE_SIDE_EFFECTS (CALL_EXPR_ARG (exp, 1))
5897	&& (call_expr_nargs (exp) < 3
5898	|| !TREE_SIDE_EFFECTS (CALL_EXPR_ARG (exp, 2))));
5899	return target;
5900	}
5901
5902	void
5903	expand_builtin_trap (void)
5904	{
5905	if (targetm.have_trap ())
5906	{
5907	rtx_insn *insn = emit_insn (targetm.gen_trap ());
5908	/* For trap insns when not accumulating outgoing args force
5909	REG_ARGS_SIZE note to prevent crossjumping of calls with
5910	different args sizes. */
5911	if (!ACCUMULATE_OUTGOING_ARGS)
5912	add_args_size_note (insn, stack_pointer_delta);
5913	}
5914	else
5915	{
5916	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ABORT);
5917	tree call_expr = build_call_expr (fn, 0);
5918	expand_call (call_expr, NULL_RTX, false);
5919	}
5920
5921	emit_barrier ();
5922	}
5923
5924	/* Expand a call to __builtin_unreachable. We do nothing except emit
5925	a barrier saying that control flow will not pass here.
5926
5927	It is the responsibility of the program being compiled to ensure
5928	that control flow does never reach __builtin_unreachable. */
5929	static void
5930	expand_builtin_unreachable (void)
5931	{
5932	/* Use gimple_build_builtin_unreachable or builtin_decl_unreachable
5933	to avoid this. */
5934	gcc_checking_assert (!sanitize_flags_p (SANITIZE_UNREACHABLE));
5935	emit_barrier ();
5936	}
5937
5938	/* Expand EXP, a call to fabs, fabsf or fabsl.
5939	Return NULL_RTX if a normal call should be emitted rather than expanding
5940	the function inline. If convenient, the result should be placed
5941	in TARGET. SUBTARGET may be used as the target for computing
5942	the operand. */
5943
5944	static rtx
5945	expand_builtin_fabs (tree exp, rtx target, rtx subtarget)
5946	{
5947	machine_mode mode;
5948	tree arg;
5949	rtx op0;
5950
5951	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
5952	return NULL_RTX;
5953
5954	arg = CALL_EXPR_ARG (exp, 0);
5955	CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (exp: arg);
5956	mode = TYPE_MODE (TREE_TYPE (arg));
5957	op0 = expand_expr (exp: arg, target: subtarget, VOIDmode, modifier: EXPAND_NORMAL);
5958	return expand_abs (mode, op0, target, 0, safe_from_p (target, arg, 1));
5959	}
5960
5961	/* Expand EXP, a call to copysign, copysignf, or copysignl.
5962	Return NULL is a normal call should be emitted rather than expanding the
5963	function inline. If convenient, the result should be placed in TARGET.
5964	SUBTARGET may be used as the target for computing the operand. */
5965
5966	static rtx
5967	expand_builtin_copysign (tree exp, rtx target, rtx subtarget)
5968	{
5969	rtx op0, op1;
5970	tree arg;
5971
5972	if (!validate_arglist (callexpr: exp, REAL_TYPE, REAL_TYPE, VOID_TYPE))
5973	return NULL_RTX;
5974
5975	arg = CALL_EXPR_ARG (exp, 0);
5976	op0 = expand_expr (exp: arg, target: subtarget, VOIDmode, modifier: EXPAND_NORMAL);
5977
5978	arg = CALL_EXPR_ARG (exp, 1);
5979	op1 = expand_normal (exp: arg);
5980
5981	return expand_copysign (op0, op1, target);
5982	}
5983
5984	/* Emit a call to __builtin___clear_cache. */
5985
5986	void
5987	default_emit_call_builtin___clear_cache (rtx begin, rtx end)
5988	{
5989	rtx callee = gen_rtx_SYMBOL_REF (Pmode,
5990	BUILTIN_ASM_NAME_PTR
5991	(BUILT_IN_CLEAR_CACHE));
5992
5993	emit_library_call (fun: callee,
5994	fn_type: LCT_NORMAL, VOIDmode,
5995	convert_memory_address (ptr_mode, begin), arg1_mode: ptr_mode,
5996	convert_memory_address (ptr_mode, end), arg2_mode: ptr_mode);
5997	}
5998
5999	/* Emit a call to __builtin___clear_cache, unless the target specifies
6000	it as do-nothing. This function can be used by trampoline
6001	finalizers to duplicate the effects of expanding a call to the
6002	clear_cache builtin. */
6003
6004	void
6005	maybe_emit_call_builtin___clear_cache (rtx begin, rtx end)
6006	{
6007	gcc_assert ((GET_MODE (begin) == ptr_mode || GET_MODE (begin) == Pmode
6008	|| CONST_INT_P (begin))
6009	&& (GET_MODE (end) == ptr_mode || GET_MODE (end) == Pmode
6010	|| CONST_INT_P (end)));
6011
6012	if (targetm.have_clear_cache ())
6013	{
6014	/* We have a "clear_cache" insn, and it will handle everything. */
6015	class expand_operand ops[2];
6016
6017	create_address_operand (op: &ops[0], value: begin);
6018	create_address_operand (op: &ops[1], value: end);
6019
6020	if (maybe_expand_insn (icode: targetm.code_for_clear_cache, nops: 2, ops))
6021	return;
6022	}
6023	else
6024	{
6025	#ifndef CLEAR_INSN_CACHE
6026	/* There is no "clear_cache" insn, and __clear_cache() in libgcc
6027	does nothing. There is no need to call it. Do nothing. */
6028	return;
6029	#endif /* CLEAR_INSN_CACHE */
6030	}
6031
6032	targetm.calls.emit_call_builtin___clear_cache (begin, end);
6033	}
6034
6035	/* Expand a call to __builtin___clear_cache. */
6036
6037	static void
6038	expand_builtin___clear_cache (tree exp)
6039	{
6040	tree begin, end;
6041	rtx begin_rtx, end_rtx;
6042
6043	/* We must not expand to a library call. If we did, any
6044	fallback library function in libgcc that might contain a call to
6045	__builtin___clear_cache() would recurse infinitely. */
6046	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
6047	{
6048	error ("both arguments to %<__builtin___clear_cache%> must be pointers");
6049	return;
6050	}
6051
6052	begin = CALL_EXPR_ARG (exp, 0);
6053	begin_rtx = expand_expr (exp: begin, NULL_RTX, Pmode, modifier: EXPAND_NORMAL);
6054
6055	end = CALL_EXPR_ARG (exp, 1);
6056	end_rtx = expand_expr (exp: end, NULL_RTX, Pmode, modifier: EXPAND_NORMAL);
6057
6058	maybe_emit_call_builtin___clear_cache (begin: begin_rtx, end: end_rtx);
6059	}
6060
6061	/* Given a trampoline address, make sure it satisfies TRAMPOLINE_ALIGNMENT. */
6062
6063	static rtx
6064	round_trampoline_addr (rtx tramp)
6065	{
6066	rtx temp, addend, mask;
6067
6068	/* If we don't need too much alignment, we'll have been guaranteed
6069	proper alignment by get_trampoline_type. */
6070	if (TRAMPOLINE_ALIGNMENT <= STACK_BOUNDARY)
6071	return tramp;
6072
6073	/* Round address up to desired boundary. */
6074	temp = gen_reg_rtx (Pmode);
6075	addend = gen_int_mode (TRAMPOLINE_ALIGNMENT / BITS_PER_UNIT - 1, Pmode);
6076	mask = gen_int_mode (-TRAMPOLINE_ALIGNMENT / BITS_PER_UNIT, Pmode);
6077
6078	temp = expand_simple_binop (Pmode, PLUS, tramp, addend,
6079	temp, 0, OPTAB_LIB_WIDEN);
6080	tramp = expand_simple_binop (Pmode, AND, temp, mask,
6081	temp, 0, OPTAB_LIB_WIDEN);
6082
6083	return tramp;
6084	}
6085
6086	static rtx
6087	expand_builtin_init_trampoline (tree exp, bool onstack)
6088	{
6089	tree t_tramp, t_func, t_chain;
6090	rtx m_tramp, r_tramp, r_chain, tmp;
6091
6092	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE,
6093	POINTER_TYPE, VOID_TYPE))
6094	return NULL_RTX;
6095
6096	t_tramp = CALL_EXPR_ARG (exp, 0);
6097	t_func = CALL_EXPR_ARG (exp, 1);
6098	t_chain = CALL_EXPR_ARG (exp, 2);
6099
6100	r_tramp = expand_normal (exp: t_tramp);
6101	m_tramp = gen_rtx_MEM (BLKmode, r_tramp);
6102	MEM_NOTRAP_P (m_tramp) = 1;
6103
6104	/* If ONSTACK, the TRAMP argument should be the address of a field
6105	within the local function's FRAME decl. Either way, let's see if
6106	we can fill in the MEM_ATTRs for this memory. */
6107	if (TREE_CODE (t_tramp) == ADDR_EXPR)
6108	set_mem_attributes (m_tramp, TREE_OPERAND (t_tramp, 0), true);
6109
6110	/* Creator of a heap trampoline is responsible for making sure the
6111	address is aligned to at least STACK_BOUNDARY. Normally malloc
6112	will ensure this anyhow. */
6113	tmp = round_trampoline_addr (tramp: r_tramp);
6114	if (tmp != r_tramp)
6115	{
6116	m_tramp = change_address (m_tramp, BLKmode, tmp);
6117	set_mem_align (m_tramp, TRAMPOLINE_ALIGNMENT);
6118	set_mem_size (m_tramp, TRAMPOLINE_SIZE);
6119	}
6120
6121	/* The FUNC argument should be the address of the nested function.
6122	Extract the actual function decl to pass to the hook. */
6123	gcc_assert (TREE_CODE (t_func) == ADDR_EXPR);
6124	t_func = TREE_OPERAND (t_func, 0);
6125	gcc_assert (TREE_CODE (t_func) == FUNCTION_DECL);
6126
6127	r_chain = expand_normal (exp: t_chain);
6128
6129	/* Generate insns to initialize the trampoline. */
6130	targetm.calls.trampoline_init (m_tramp, t_func, r_chain);
6131
6132	if (onstack)
6133	{
6134	trampolines_created = 1;
6135
6136	if (targetm.calls.custom_function_descriptors != 0)
6137	warning_at (DECL_SOURCE_LOCATION (t_func), OPT_Wtrampolines,
6138	"trampoline generated for nested function %qD", t_func);
6139	}
6140
6141	return const0_rtx;
6142	}
6143
6144	static rtx
6145	expand_builtin_adjust_trampoline (tree exp)
6146	{
6147	rtx tramp;
6148
6149	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
6150	return NULL_RTX;
6151
6152	tramp = expand_normal (CALL_EXPR_ARG (exp, 0));
6153	tramp = round_trampoline_addr (tramp);
6154	if (targetm.calls.trampoline_adjust_address)
6155	tramp = targetm.calls.trampoline_adjust_address (tramp);
6156
6157	return tramp;
6158	}
6159
6160	/* Expand a call to the builtin descriptor initialization routine.
6161	A descriptor is made up of a couple of pointers to the static
6162	chain and the code entry in this order. */
6163
6164	static rtx
6165	expand_builtin_init_descriptor (tree exp)
6166	{
6167	tree t_descr, t_func, t_chain;
6168	rtx m_descr, r_descr, r_func, r_chain;
6169
6170	if (!validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, POINTER_TYPE,
6171	VOID_TYPE))
6172	return NULL_RTX;
6173
6174	t_descr = CALL_EXPR_ARG (exp, 0);
6175	t_func = CALL_EXPR_ARG (exp, 1);
6176	t_chain = CALL_EXPR_ARG (exp, 2);
6177
6178	r_descr = expand_normal (exp: t_descr);
6179	m_descr = gen_rtx_MEM (BLKmode, r_descr);
6180	MEM_NOTRAP_P (m_descr) = 1;
6181	set_mem_align (m_descr, GET_MODE_ALIGNMENT (ptr_mode));
6182
6183	r_func = expand_normal (exp: t_func);
6184	r_chain = expand_normal (exp: t_chain);
6185
6186	/* Generate insns to initialize the descriptor. */
6187	emit_move_insn (adjust_address_nv (m_descr, ptr_mode, 0), r_chain);
6188	emit_move_insn (adjust_address_nv (m_descr, ptr_mode,
6189	POINTER_SIZE / BITS_PER_UNIT), r_func);
6190
6191	return const0_rtx;
6192	}
6193
6194	/* Expand a call to the builtin descriptor adjustment routine. */
6195
6196	static rtx
6197	expand_builtin_adjust_descriptor (tree exp)
6198	{
6199	rtx tramp;
6200
6201	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
6202	return NULL_RTX;
6203
6204	tramp = expand_normal (CALL_EXPR_ARG (exp, 0));
6205
6206	/* Unalign the descriptor to allow runtime identification. */
6207	tramp = plus_constant (ptr_mode, tramp,
6208	targetm.calls.custom_function_descriptors);
6209
6210	return force_operand (tramp, NULL_RTX);
6211	}
6212
6213	/* Expand the call EXP to the built-in signbit, signbitf or signbitl
6214	function. The function first checks whether the back end provides
6215	an insn to implement signbit for the respective mode. If not, it
6216	checks whether the floating point format of the value is such that
6217	the sign bit can be extracted. If that is not the case, error out.
6218	EXP is the expression that is a call to the builtin function; if
6219	convenient, the result should be placed in TARGET. */
6220	static rtx
6221	expand_builtin_signbit (tree exp, rtx target)
6222	{
6223	const struct real_format *fmt;
6224	scalar_float_mode fmode;
6225	scalar_int_mode rmode, imode;
6226	tree arg;
6227	int word, bitpos;
6228	enum insn_code icode;
6229	rtx temp;
6230	location_t loc = EXPR_LOCATION (exp);
6231
6232	if (!validate_arglist (callexpr: exp, REAL_TYPE, VOID_TYPE))
6233	return NULL_RTX;
6234
6235	arg = CALL_EXPR_ARG (exp, 0);
6236	fmode = SCALAR_FLOAT_TYPE_MODE (TREE_TYPE (arg));
6237	rmode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
6238	fmt = REAL_MODE_FORMAT (fmode);
6239
6240	arg = builtin_save_expr (exp: arg);
6241
6242	/* Expand the argument yielding a RTX expression. */
6243	temp = expand_normal (exp: arg);
6244
6245	/* Check if the back end provides an insn that handles signbit for the
6246	argument's mode. */
6247	icode = optab_handler (op: signbit_optab, mode: fmode);
6248	if (icode != CODE_FOR_nothing)
6249	{
6250	rtx_insn *last = get_last_insn ();
6251	rtx this_target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
6252	if (maybe_emit_unop_insn (icode, this_target, temp, UNKNOWN))
6253	return this_target;
6254	delete_insns_since (last);
6255	}
6256
6257	/* For floating point formats without a sign bit, implement signbit
6258	as "ARG < 0.0". */
6259	bitpos = fmt->signbit_ro;
6260	if (bitpos < 0)
6261	{
6262	/* But we can't do this if the format supports signed zero. */
6263	gcc_assert (!fmt->has_signed_zero || !HONOR_SIGNED_ZEROS (fmode));
6264
6265	arg = fold_build2_loc (loc, LT_EXPR, TREE_TYPE (exp), arg,
6266	build_real (TREE_TYPE (arg), dconst0));
6267	return expand_expr (exp: arg, target, VOIDmode, modifier: EXPAND_NORMAL);
6268	}
6269
6270	if (GET_MODE_SIZE (mode: fmode) <= UNITS_PER_WORD)
6271	{
6272	imode = int_mode_for_mode (fmode).require ();
6273	temp = gen_lowpart (imode, temp);
6274	}
6275	else
6276	{
6277	imode = word_mode;
6278	/* Handle targets with different FP word orders. */
6279	if (FLOAT_WORDS_BIG_ENDIAN)
6280	word = (GET_MODE_BITSIZE (mode: fmode) - bitpos) / BITS_PER_WORD;
6281	else
6282	word = bitpos / BITS_PER_WORD;
6283	temp = operand_subword_force (temp, word, fmode);
6284	bitpos = bitpos % BITS_PER_WORD;
6285	}
6286
6287	/* Force the intermediate word_mode (or narrower) result into a
6288	register. This avoids attempting to create paradoxical SUBREGs
6289	of floating point modes below. */
6290	temp = force_reg (imode, temp);
6291
6292	/* If the bitpos is within the "result mode" lowpart, the operation
6293	can be implement with a single bitwise AND. Otherwise, we need
6294	a right shift and an AND. */
6295
6296	if (bitpos < GET_MODE_BITSIZE (mode: rmode))
6297	{
6298	wide_int mask = wi::set_bit_in_zero (bit: bitpos, precision: GET_MODE_PRECISION (mode: rmode));
6299
6300	if (GET_MODE_SIZE (mode: imode) > GET_MODE_SIZE (mode: rmode))
6301	temp = gen_lowpart (rmode, temp);
6302	temp = expand_binop (rmode, and_optab, temp,
6303	immed_wide_int_const (mask, rmode),
6304	NULL_RTX, 1, OPTAB_LIB_WIDEN);
6305	}
6306	else
6307	{
6308	/* Perform a logical right shift to place the signbit in the least
6309	significant bit, then truncate the result to the desired mode
6310	and mask just this bit. */
6311	temp = expand_shift (RSHIFT_EXPR, imode, temp, bitpos, NULL_RTX, 1);
6312	temp = gen_lowpart (rmode, temp);
6313	temp = expand_binop (rmode, and_optab, temp, const1_rtx,
6314	NULL_RTX, 1, OPTAB_LIB_WIDEN);
6315	}
6316
6317	return temp;
6318	}
6319
6320	/* Expand fork or exec calls. TARGET is the desired target of the
6321	call. EXP is the call. FN is the
6322	identificator of the actual function. IGNORE is nonzero if the
6323	value is to be ignored. */
6324
6325	static rtx
6326	expand_builtin_fork_or_exec (tree fn, tree exp, rtx target, int ignore)
6327	{
6328	tree id, decl;
6329	tree call;
6330
6331	/* If we are not profiling, just call the function. */
6332	if (!profile_arc_flag && !condition_coverage_flag)
6333	return NULL_RTX;
6334
6335	/* Otherwise call the wrapper. This should be equivalent for the rest of
6336	compiler, so the code does not diverge, and the wrapper may run the
6337	code necessary for keeping the profiling sane. */
6338
6339	switch (DECL_FUNCTION_CODE (decl: fn))
6340	{
6341	case BUILT_IN_FORK:
6342	id = get_identifier ("__gcov_fork");
6343	break;
6344
6345	case BUILT_IN_EXECL:
6346	id = get_identifier ("__gcov_execl");
6347	break;
6348
6349	case BUILT_IN_EXECV:
6350	id = get_identifier ("__gcov_execv");
6351	break;
6352
6353	case BUILT_IN_EXECLP:
6354	id = get_identifier ("__gcov_execlp");
6355	break;
6356
6357	case BUILT_IN_EXECLE:
6358	id = get_identifier ("__gcov_execle");
6359	break;
6360
6361	case BUILT_IN_EXECVP:
6362	id = get_identifier ("__gcov_execvp");
6363	break;
6364
6365	case BUILT_IN_EXECVE:
6366	id = get_identifier ("__gcov_execve");
6367	break;
6368
6369	default:
6370	gcc_unreachable ();
6371	}
6372
6373	decl = build_decl (DECL_SOURCE_LOCATION (fn),
6374	FUNCTION_DECL, id, TREE_TYPE (fn));
6375	DECL_EXTERNAL (decl) = 1;
6376	TREE_PUBLIC (decl) = 1;
6377	DECL_ARTIFICIAL (decl) = 1;
6378	TREE_NOTHROW (decl) = 1;
6379	DECL_VISIBILITY (decl) = VISIBILITY_DEFAULT;
6380	DECL_VISIBILITY_SPECIFIED (decl) = 1;
6381	call = rewrite_call_expr (EXPR_LOCATION (exp), exp, 0, decl, 0);
6382	return expand_call (call, target, ignore);
6383	}
6384
6385
6386
6387	/* Reconstitute a mode for a __sync intrinsic operation. Since the type of
6388	the pointer in these functions is void*, the tree optimizers may remove
6389	casts. The mode computed in expand_builtin isn't reliable either, due
6390	to __sync_bool_compare_and_swap.
6391
6392	FCODE_DIFF should be fcode - base, where base is the FOO_1 code for the
6393	group of builtins. This gives us log2 of the mode size. */
6394
6395	static inline machine_mode
6396	get_builtin_sync_mode (int fcode_diff)
6397	{
6398	/* The size is not negotiable, so ask not to get BLKmode in return
6399	if the target indicates that a smaller size would be better. */
6400	return int_mode_for_size (BITS_PER_UNIT << fcode_diff, limit: 0).require ();
6401	}
6402
6403	/* Expand the memory expression LOC and return the appropriate memory operand
6404	for the builtin_sync operations. */
6405
6406	static rtx
6407	get_builtin_sync_mem (tree loc, machine_mode mode)
6408	{
6409	rtx addr, mem;
6410	int addr_space = TYPE_ADDR_SPACE (POINTER_TYPE_P (TREE_TYPE (loc))
6411	? TREE_TYPE (TREE_TYPE (loc))
6412	: TREE_TYPE (loc));
6413	scalar_int_mode addr_mode = targetm.addr_space.address_mode (addr_space);
6414
6415	addr = expand_expr (exp: loc, NULL_RTX, mode: addr_mode, modifier: EXPAND_SUM);
6416	addr = convert_memory_address (addr_mode, addr);
6417
6418	/* Note that we explicitly do not want any alias information for this
6419	memory, so that we kill all other live memories. Otherwise we don't
6420	satisfy the full barrier semantics of the intrinsic. */
6421	mem = gen_rtx_MEM (mode, addr);
6422
6423	set_mem_addr_space (mem, addr_space);
6424
6425	mem = validize_mem (mem);
6426
6427	/* The alignment needs to be at least according to that of the mode. */
6428	set_mem_align (mem, MAX (GET_MODE_ALIGNMENT (mode),
6429	get_pointer_alignment (loc)));
6430	set_mem_alias_set (mem, ALIAS_SET_MEMORY_BARRIER);
6431	MEM_VOLATILE_P (mem) = 1;
6432
6433	return mem;
6434	}
6435
6436	/* Make sure an argument is in the right mode.
6437	EXP is the tree argument.
6438	MODE is the mode it should be in. */
6439
6440	static rtx
6441	expand_expr_force_mode (tree exp, machine_mode mode)
6442	{
6443	rtx val;
6444	machine_mode old_mode;
6445
6446	if (TREE_CODE (exp) == SSA_NAME
6447	&& TYPE_MODE (TREE_TYPE (exp)) != mode)
6448	{
6449	/* Undo argument promotion if possible, as combine might not
6450	be able to do it later due to MEM_VOLATILE_P uses in the
6451	patterns. */
6452	gimple *g = get_gimple_for_ssa_name (exp);
6453	if (g && gimple_assign_cast_p (s: g))
6454	{
6455	tree rhs = gimple_assign_rhs1 (gs: g);
6456	tree_code code = gimple_assign_rhs_code (gs: g);
6457	if (CONVERT_EXPR_CODE_P (code)
6458	&& TYPE_MODE (TREE_TYPE (rhs)) == mode
6459	&& INTEGRAL_TYPE_P (TREE_TYPE (exp))
6460	&& INTEGRAL_TYPE_P (TREE_TYPE (rhs))
6461	&& (TYPE_PRECISION (TREE_TYPE (exp))
6462	> TYPE_PRECISION (TREE_TYPE (rhs))))
6463	exp = rhs;
6464	}
6465	}
6466
6467	val = expand_expr (exp, NULL_RTX, mode, modifier: EXPAND_NORMAL);
6468	/* If VAL is promoted to a wider mode, convert it back to MODE. Take care
6469	of CONST_INTs, where we know the old_mode only from the call argument. */
6470
6471	old_mode = GET_MODE (val);
6472	if (old_mode == VOIDmode)
6473	old_mode = TYPE_MODE (TREE_TYPE (exp));
6474	val = convert_modes (mode, oldmode: old_mode, x: val, unsignedp: 1);
6475	return val;
6476	}
6477
6478
6479	/* Expand the __sync_xxx_and_fetch and __sync_fetch_and_xxx intrinsics.
6480	EXP is the CALL_EXPR. CODE is the rtx code
6481	that corresponds to the arithmetic or logical operation from the name;
6482	an exception here is that NOT actually means NAND. TARGET is an optional
6483	place for us to store the results; AFTER is true if this is the
6484	fetch_and_xxx form. */
6485
6486	static rtx
6487	expand_builtin_sync_operation (machine_mode mode, tree exp,
6488	enum rtx_code code, bool after,
6489	rtx target)
6490	{
6491	rtx val, mem;
6492	location_t loc = EXPR_LOCATION (exp);
6493
6494	if (code == NOT && warn_sync_nand)
6495	{
6496	tree fndecl = get_callee_fndecl (exp);
6497	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
6498
6499	static bool warned_f_a_n, warned_n_a_f;
6500
6501	switch (fcode)
6502	{
6503	case BUILT_IN_SYNC_FETCH_AND_NAND_1:
6504	case BUILT_IN_SYNC_FETCH_AND_NAND_2:
6505	case BUILT_IN_SYNC_FETCH_AND_NAND_4:
6506	case BUILT_IN_SYNC_FETCH_AND_NAND_8:
6507	case BUILT_IN_SYNC_FETCH_AND_NAND_16:
6508	if (warned_f_a_n)
6509	break;
6510
6511	fndecl = builtin_decl_implicit (fncode: BUILT_IN_SYNC_FETCH_AND_NAND_N);
6512	inform (loc, "%qD changed semantics in GCC 4.4", fndecl);
6513	warned_f_a_n = true;
6514	break;
6515
6516	case BUILT_IN_SYNC_NAND_AND_FETCH_1:
6517	case BUILT_IN_SYNC_NAND_AND_FETCH_2:
6518	case BUILT_IN_SYNC_NAND_AND_FETCH_4:
6519	case BUILT_IN_SYNC_NAND_AND_FETCH_8:
6520	case BUILT_IN_SYNC_NAND_AND_FETCH_16:
6521	if (warned_n_a_f)
6522	break;
6523
6524	fndecl = builtin_decl_implicit (fncode: BUILT_IN_SYNC_NAND_AND_FETCH_N);
6525	inform (loc, "%qD changed semantics in GCC 4.4", fndecl);
6526	warned_n_a_f = true;
6527	break;
6528
6529	default:
6530	gcc_unreachable ();
6531	}
6532	}
6533
6534	/* Expand the operands. */
6535	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6536	val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
6537
6538	return expand_atomic_fetch_op (target, mem, val, code, MEMMODEL_SYNC_SEQ_CST,
6539	after);
6540	}
6541
6542	/* Expand the __sync_val_compare_and_swap and __sync_bool_compare_and_swap
6543	intrinsics. EXP is the CALL_EXPR. IS_BOOL is
6544	true if this is the boolean form. TARGET is a place for us to store the
6545	results; this is NOT optional if IS_BOOL is true. */
6546
6547	static rtx
6548	expand_builtin_compare_and_swap (machine_mode mode, tree exp,
6549	bool is_bool, rtx target)
6550	{
6551	rtx old_val, new_val, mem;
6552	rtx *pbool, *poval;
6553
6554	/* Expand the operands. */
6555	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6556	old_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
6557	new_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
6558
6559	pbool = poval = NULL;
6560	if (target != const0_rtx)
6561	{
6562	if (is_bool)
6563	pbool = &target;
6564	else
6565	poval = &target;
6566	}
6567	if (!expand_atomic_compare_and_swap (pbool, poval, mem, old_val, new_val,
6568	false, MEMMODEL_SYNC_SEQ_CST,
6569	MEMMODEL_SYNC_SEQ_CST))
6570	return NULL_RTX;
6571
6572	return target;
6573	}
6574
6575	/* Expand the __sync_lock_test_and_set intrinsic. Note that the most
6576	general form is actually an atomic exchange, and some targets only
6577	support a reduced form with the second argument being a constant 1.
6578	EXP is the CALL_EXPR; TARGET is an optional place for us to store
6579	the results. */
6580
6581	static rtx
6582	expand_builtin_sync_lock_test_and_set (machine_mode mode, tree exp,
6583	rtx target)
6584	{
6585	rtx val, mem;
6586
6587	/* Expand the operands. */
6588	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6589	val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
6590
6591	return expand_sync_lock_test_and_set (target, mem, val);
6592	}
6593
6594	/* Expand the __sync_lock_release intrinsic. EXP is the CALL_EXPR. */
6595
6596	static void
6597	expand_builtin_sync_lock_release (machine_mode mode, tree exp)
6598	{
6599	rtx mem;
6600
6601	/* Expand the operands. */
6602	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6603
6604	expand_atomic_store (mem, const0_rtx, MEMMODEL_SYNC_RELEASE, true);
6605	}
6606
6607	/* Given an integer representing an ``enum memmodel'', verify its
6608	correctness and return the memory model enum. */
6609
6610	static enum memmodel
6611	get_memmodel (tree exp)
6612	{
6613	/* If the parameter is not a constant, it's a run time value so we'll just
6614	convert it to MEMMODEL_SEQ_CST to avoid annoying runtime checking. */
6615	if (TREE_CODE (exp) != INTEGER_CST)
6616	return MEMMODEL_SEQ_CST;
6617
6618	rtx op = expand_normal (exp);
6619
6620	unsigned HOST_WIDE_INT val = INTVAL (op);
6621	if (targetm.memmodel_check)
6622	val = targetm.memmodel_check (val);
6623	else if (val & ~MEMMODEL_MASK)
6624	return MEMMODEL_SEQ_CST;
6625
6626	/* Should never see a user explicit SYNC memodel model, so >= LAST works. */
6627	if (memmodel_base (val) >= MEMMODEL_LAST)
6628	return MEMMODEL_SEQ_CST;
6629
6630	/* Workaround for Bugzilla 59448. GCC doesn't track consume properly, so
6631	be conservative and promote consume to acquire. */
6632	if (val == MEMMODEL_CONSUME)
6633	val = MEMMODEL_ACQUIRE;
6634
6635	return (enum memmodel) val;
6636	}
6637
6638	/* Expand the __atomic_exchange intrinsic:
6639	TYPE __atomic_exchange (TYPE *object, TYPE desired, enum memmodel)
6640	EXP is the CALL_EXPR.
6641	TARGET is an optional place for us to store the results. */
6642
6643	static rtx
6644	expand_builtin_atomic_exchange (machine_mode mode, tree exp, rtx target)
6645	{
6646	rtx val, mem;
6647	enum memmodel model;
6648
6649	model = get_memmodel (CALL_EXPR_ARG (exp, 2));
6650
6651	if (!flag_inline_atomics)
6652	return NULL_RTX;
6653
6654	/* Expand the operands. */
6655	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6656	val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
6657
6658	return expand_atomic_exchange (target, mem, val, model);
6659	}
6660
6661	/* Expand the __atomic_compare_exchange intrinsic:
6662	bool __atomic_compare_exchange (TYPE *object, TYPE *expect,
6663	TYPE desired, BOOL weak,
6664	enum memmodel success,
6665	enum memmodel failure)
6666	EXP is the CALL_EXPR.
6667	TARGET is an optional place for us to store the results. */
6668
6669	static rtx
6670	expand_builtin_atomic_compare_exchange (machine_mode mode, tree exp,
6671	rtx target)
6672	{
6673	rtx expect, desired, mem, oldval;
6674	rtx_code_label *label;
6675	tree weak;
6676	bool is_weak;
6677
6678	memmodel success = get_memmodel (CALL_EXPR_ARG (exp, 4));
6679	memmodel failure = get_memmodel (CALL_EXPR_ARG (exp, 5));
6680
6681	if (failure > success)
6682	success = MEMMODEL_SEQ_CST;
6683
6684	if (is_mm_release (model: failure) || is_mm_acq_rel (model: failure))
6685	{
6686	failure = MEMMODEL_SEQ_CST;
6687	success = MEMMODEL_SEQ_CST;
6688	}
6689
6690
6691	if (!flag_inline_atomics)
6692	return NULL_RTX;
6693
6694	/* Expand the operands. */
6695	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6696
6697	expect = expand_normal (CALL_EXPR_ARG (exp, 1));
6698	expect = convert_memory_address (Pmode, expect);
6699	expect = gen_rtx_MEM (mode, expect);
6700	desired = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
6701
6702	weak = CALL_EXPR_ARG (exp, 3);
6703	is_weak = false;
6704	if (tree_fits_shwi_p (weak) && tree_to_shwi (weak) != 0)
6705	is_weak = true;
6706
6707	if (target == const0_rtx)
6708	target = NULL;
6709
6710	/* Lest the rtl backend create a race condition with an imporoper store
6711	to memory, always create a new pseudo for OLDVAL. */
6712	oldval = NULL;
6713
6714	if (!expand_atomic_compare_and_swap (&target, &oldval, mem, expect, desired,
6715	is_weak, success, failure))
6716	return NULL_RTX;
6717
6718	/* Conditionally store back to EXPECT, lest we create a race condition
6719	with an improper store to memory. */
6720	/* ??? With a rearrangement of atomics at the gimple level, we can handle
6721	the normal case where EXPECT is totally private, i.e. a register. At
6722	which point the store can be unconditional. */
6723	label = gen_label_rtx ();
6724	emit_cmp_and_jump_insns (target, const0_rtx, NE, NULL,
6725	GET_MODE (target), 1, label);
6726	emit_move_insn (expect, oldval);
6727	emit_label (label);
6728
6729	return target;
6730	}
6731
6732	/* Helper function for expand_ifn_atomic_compare_exchange - expand
6733	internal ATOMIC_COMPARE_EXCHANGE call into __atomic_compare_exchange_N
6734	call. The weak parameter must be dropped to match the expected parameter
6735	list and the expected argument changed from value to pointer to memory
6736	slot. */
6737
6738	static void
6739	expand_ifn_atomic_compare_exchange_into_call (gcall *call, machine_mode mode)
6740	{
6741	unsigned int z;
6742	vec<tree, va_gc> *vec;
6743
6744	vec_alloc (v&: vec, nelems: 5);
6745	vec->quick_push (obj: gimple_call_arg (gs: call, index: 0));
6746	tree expected = gimple_call_arg (gs: call, index: 1);
6747	rtx x = assign_stack_temp_for_type (mode, GET_MODE_SIZE (mode),
6748	TREE_TYPE (expected));
6749	rtx expd = expand_expr (exp: expected, target: x, mode, modifier: EXPAND_NORMAL);
6750	if (expd != x)
6751	emit_move_insn (x, expd);
6752	tree v = make_tree (TREE_TYPE (expected), x);
6753	vec->quick_push (obj: build1 (ADDR_EXPR,
6754	build_pointer_type (TREE_TYPE (expected)), v));
6755	vec->quick_push (obj: gimple_call_arg (gs: call, index: 2));
6756	/* Skip the boolean weak parameter. */
6757	for (z = 4; z < 6; z++)
6758	vec->quick_push (obj: gimple_call_arg (gs: call, index: z));
6759	/* At present we only have BUILT_IN_ATOMIC_COMPARE_EXCHANGE_{1,2,4,8,16}. */
6760	unsigned int bytes_log2 = exact_log2 (x: GET_MODE_SIZE (mode).to_constant ());
6761	gcc_assert (bytes_log2 < 5);
6762	built_in_function fncode
6763	= (built_in_function) ((int) BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1
6764	+ bytes_log2);
6765	tree fndecl = builtin_decl_explicit (fncode);
6766	tree fn = build1 (ADDR_EXPR, build_pointer_type (TREE_TYPE (fndecl)),
6767	fndecl);
6768	tree exp = build_call_vec (boolean_type_node, fn, vec);
6769	tree lhs = gimple_call_lhs (gs: call);
6770	rtx boolret = expand_call (exp, NULL_RTX, lhs == NULL_TREE);
6771	if (lhs)
6772	{
6773	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
6774	if (GET_MODE (boolret) != mode)
6775	boolret = convert_modes (mode, GET_MODE (boolret), x: boolret, unsignedp: 1);
6776	x = force_reg (mode, x);
6777	write_complex_part (target, boolret, true, true);
6778	write_complex_part (target, x, false, false);
6779	}
6780	}
6781
6782	/* Expand IFN_ATOMIC_COMPARE_EXCHANGE internal function. */
6783
6784	void
6785	expand_ifn_atomic_compare_exchange (gcall *call)
6786	{
6787	int size = tree_to_shwi (gimple_call_arg (gs: call, index: 3)) & 255;
6788	gcc_assert (size == 1 || size == 2 || size == 4 || size == 8 || size == 16);
6789	machine_mode mode = int_mode_for_size (BITS_PER_UNIT * size, limit: 0).require ();
6790
6791	memmodel success = get_memmodel (exp: gimple_call_arg (gs: call, index: 4));
6792	memmodel failure = get_memmodel (exp: gimple_call_arg (gs: call, index: 5));
6793
6794	if (failure > success)
6795	success = MEMMODEL_SEQ_CST;
6796
6797	if (is_mm_release (model: failure) || is_mm_acq_rel (model: failure))
6798	{
6799	failure = MEMMODEL_SEQ_CST;
6800	success = MEMMODEL_SEQ_CST;
6801	}
6802
6803	if (!flag_inline_atomics)
6804	{
6805	expand_ifn_atomic_compare_exchange_into_call (call, mode);
6806	return;
6807	}
6808
6809	/* Expand the operands. */
6810	rtx mem = get_builtin_sync_mem (loc: gimple_call_arg (gs: call, index: 0), mode);
6811
6812	rtx expect = expand_expr_force_mode (exp: gimple_call_arg (gs: call, index: 1), mode);
6813	rtx desired = expand_expr_force_mode (exp: gimple_call_arg (gs: call, index: 2), mode);
6814
6815	bool is_weak = (tree_to_shwi (gimple_call_arg (gs: call, index: 3)) & 256) != 0;
6816
6817	rtx boolret = NULL;
6818	rtx oldval = NULL;
6819
6820	if (!expand_atomic_compare_and_swap (&boolret, &oldval, mem, expect, desired,
6821	is_weak, success, failure))
6822	{
6823	expand_ifn_atomic_compare_exchange_into_call (call, mode);
6824	return;
6825	}
6826
6827	tree lhs = gimple_call_lhs (gs: call);
6828	if (lhs)
6829	{
6830	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
6831	if (GET_MODE (boolret) != mode)
6832	boolret = convert_modes (mode, GET_MODE (boolret), x: boolret, unsignedp: 1);
6833	write_complex_part (target, boolret, true, true);
6834	write_complex_part (target, oldval, false, false);
6835	}
6836	}
6837
6838	/* Expand the __atomic_load intrinsic:
6839	TYPE __atomic_load (TYPE *object, enum memmodel)
6840	EXP is the CALL_EXPR.
6841	TARGET is an optional place for us to store the results. */
6842
6843	static rtx
6844	expand_builtin_atomic_load (machine_mode mode, tree exp, rtx target)
6845	{
6846	memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 1));
6847	if (is_mm_release (model) || is_mm_acq_rel (model))
6848	model = MEMMODEL_SEQ_CST;
6849
6850	if (!flag_inline_atomics)
6851	return NULL_RTX;
6852
6853	/* Expand the operand. */
6854	rtx mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6855
6856	return expand_atomic_load (target, mem, model);
6857	}
6858
6859
6860	/* Expand the __atomic_store intrinsic:
6861	void __atomic_store (TYPE *object, TYPE desired, enum memmodel)
6862	EXP is the CALL_EXPR.
6863	TARGET is an optional place for us to store the results. */
6864
6865	static rtx
6866	expand_builtin_atomic_store (machine_mode mode, tree exp)
6867	{
6868	memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 2));
6869	if (!(is_mm_relaxed (model) || is_mm_seq_cst (model)
6870	|| is_mm_release (model)))
6871	model = MEMMODEL_SEQ_CST;
6872
6873	if (!flag_inline_atomics)
6874	return NULL_RTX;
6875
6876	/* Expand the operands. */
6877	rtx mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6878	rtx val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
6879
6880	return expand_atomic_store (mem, val, model, false);
6881	}
6882
6883	/* Expand the __atomic_fetch_XXX intrinsic:
6884	TYPE __atomic_fetch_XXX (TYPE *object, TYPE val, enum memmodel)
6885	EXP is the CALL_EXPR.
6886	TARGET is an optional place for us to store the results.
6887	CODE is the operation, PLUS, MINUS, ADD, XOR, or IOR.
6888	FETCH_AFTER is true if returning the result of the operation.
6889	FETCH_AFTER is false if returning the value before the operation.
6890	IGNORE is true if the result is not used.
6891	EXT_CALL is the correct builtin for an external call if this cannot be
6892	resolved to an instruction sequence. */
6893
6894	static rtx
6895	expand_builtin_atomic_fetch_op (machine_mode mode, tree exp, rtx target,
6896	enum rtx_code code, bool fetch_after,
6897	bool ignore, enum built_in_function ext_call)
6898	{
6899	rtx val, mem, ret;
6900	enum memmodel model;
6901	tree fndecl;
6902	tree addr;
6903
6904	model = get_memmodel (CALL_EXPR_ARG (exp, 2));
6905
6906	/* Expand the operands. */
6907	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
6908	val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
6909
6910	/* Only try generating instructions if inlining is turned on. */
6911	if (flag_inline_atomics)
6912	{
6913	ret = expand_atomic_fetch_op (target, mem, val, code, model, fetch_after);
6914	if (ret)
6915	return ret;
6916	}
6917
6918	/* Return if a different routine isn't needed for the library call. */
6919	if (ext_call == BUILT_IN_NONE)
6920	return NULL_RTX;
6921
6922	/* Change the call to the specified function. */
6923	fndecl = get_callee_fndecl (exp);
6924	addr = CALL_EXPR_FN (exp);
6925	STRIP_NOPS (addr);
6926
6927	gcc_assert (TREE_OPERAND (addr, 0) == fndecl);
6928	TREE_OPERAND (addr, 0) = builtin_decl_explicit (fncode: ext_call);
6929
6930	/* If we will emit code after the call, the call cannot be a tail call.
6931	If it is emitted as a tail call, a barrier is emitted after it, and
6932	then all trailing code is removed. */
6933	if (!ignore)
6934	CALL_EXPR_TAILCALL (exp) = 0;
6935
6936	/* Expand the call here so we can emit trailing code. */
6937	ret = expand_call (exp, target, ignore);
6938
6939	/* Replace the original function just in case it matters. */
6940	TREE_OPERAND (addr, 0) = fndecl;
6941
6942	/* Then issue the arithmetic correction to return the right result. */
6943	if (!ignore)
6944	{
6945	if (code == NOT)
6946	{
6947	ret = expand_simple_binop (mode, AND, ret, val, NULL_RTX, true,
6948	OPTAB_LIB_WIDEN);
6949	ret = expand_simple_unop (mode, NOT, ret, target, true);
6950	}
6951	else
6952	ret = expand_simple_binop (mode, code, ret, val, target, true,
6953	OPTAB_LIB_WIDEN);
6954	}
6955	return ret;
6956	}
6957
6958	/* Expand IFN_ATOMIC_BIT_TEST_AND_* internal function. */
6959
6960	void
6961	expand_ifn_atomic_bit_test_and (gcall *call)
6962	{
6963	tree ptr = gimple_call_arg (gs: call, index: 0);
6964	tree bit = gimple_call_arg (gs: call, index: 1);
6965	tree flag = gimple_call_arg (gs: call, index: 2);
6966	tree lhs = gimple_call_lhs (gs: call);
6967	enum memmodel model = MEMMODEL_SYNC_SEQ_CST;
6968	machine_mode mode = TYPE_MODE (TREE_TYPE (flag));
6969	enum rtx_code code;
6970	optab optab;
6971	class expand_operand ops[5];
6972
6973	gcc_assert (flag_inline_atomics);
6974
6975	if (gimple_call_num_args (gs: call) == 5)
6976	model = get_memmodel (exp: gimple_call_arg (gs: call, index: 3));
6977
6978	rtx mem = get_builtin_sync_mem (loc: ptr, mode);
6979	rtx val = expand_expr_force_mode (exp: bit, mode);
6980
6981	switch (gimple_call_internal_fn (gs: call))
6982	{
6983	case IFN_ATOMIC_BIT_TEST_AND_SET:
6984	code = IOR;
6985	optab = atomic_bit_test_and_set_optab;
6986	break;
6987	case IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT:
6988	code = XOR;
6989	optab = atomic_bit_test_and_complement_optab;
6990	break;
6991	case IFN_ATOMIC_BIT_TEST_AND_RESET:
6992	code = AND;
6993	optab = atomic_bit_test_and_reset_optab;
6994	break;
6995	default:
6996	gcc_unreachable ();
6997	}
6998
6999	if (lhs == NULL_TREE)
7000	{
7001	rtx val2 = expand_simple_binop (mode, ASHIFT, const1_rtx,
7002	val, NULL_RTX, true, OPTAB_DIRECT);
7003	if (code == AND)
7004	val2 = expand_simple_unop (mode, NOT, val2, NULL_RTX, true);
7005	if (expand_atomic_fetch_op (const0_rtx, mem, val2, code, model, false))
7006	return;
7007	}
7008
7009	rtx target;
7010	if (lhs)
7011	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
7012	else
7013	target = gen_reg_rtx (mode);
7014	enum insn_code icode = direct_optab_handler (op: optab, mode);
7015	gcc_assert (icode != CODE_FOR_nothing);
7016	create_output_operand (op: &ops[0], x: target, mode);
7017	create_fixed_operand (op: &ops[1], x: mem);
7018	create_convert_operand_to (op: &ops[2], value: val, mode, unsigned_p: true);
7019	create_integer_operand (&ops[3], model);
7020	create_integer_operand (&ops[4], integer_onep (flag));
7021	if (maybe_expand_insn (icode, nops: 5, ops))
7022	return;
7023
7024	rtx bitval = val;
7025	val = expand_simple_binop (mode, ASHIFT, const1_rtx,
7026	val, NULL_RTX, true, OPTAB_DIRECT);
7027	rtx maskval = val;
7028	if (code == AND)
7029	val = expand_simple_unop (mode, NOT, val, NULL_RTX, true);
7030	rtx result = expand_atomic_fetch_op (gen_reg_rtx (mode), mem, val,
7031	code, model, false);
7032	if (!result)
7033	{
7034	bool is_atomic = gimple_call_num_args (gs: call) == 5;
7035	tree tcall = gimple_call_arg (gs: call, index: 3 + is_atomic);
7036	tree fndecl = gimple_call_addr_fndecl (fn: tcall);
7037	tree type = TREE_TYPE (TREE_TYPE (fndecl));
7038	tree exp = build_call_nary (type, tcall, 2 + is_atomic, ptr,
7039	make_tree (type, val),
7040	is_atomic
7041	? gimple_call_arg (gs: call, index: 3)
7042	: integer_zero_node);
7043	result = expand_builtin (exp, gen_reg_rtx (mode), NULL_RTX,
7044	mode, !lhs);
7045	}
7046	if (!lhs)
7047	return;
7048	if (integer_onep (flag))
7049	{
7050	result = expand_simple_binop (mode, ASHIFTRT, result, bitval,
7051	NULL_RTX, true, OPTAB_DIRECT);
7052	result = expand_simple_binop (mode, AND, result, const1_rtx, target,
7053	true, OPTAB_DIRECT);
7054	}
7055	else
7056	result = expand_simple_binop (mode, AND, result, maskval, target, true,
7057	OPTAB_DIRECT);
7058	if (result != target)
7059	emit_move_insn (target, result);
7060	}
7061
7062	/* Expand IFN_ATOMIC_*_FETCH_CMP_0 internal function. */
7063
7064	void
7065	expand_ifn_atomic_op_fetch_cmp_0 (gcall *call)
7066	{
7067	tree cmp = gimple_call_arg (gs: call, index: 0);
7068	tree ptr = gimple_call_arg (gs: call, index: 1);
7069	tree arg = gimple_call_arg (gs: call, index: 2);
7070	tree lhs = gimple_call_lhs (gs: call);
7071	enum memmodel model = MEMMODEL_SYNC_SEQ_CST;
7072	machine_mode mode = TYPE_MODE (TREE_TYPE (cmp));
7073	optab optab;
7074	rtx_code code;
7075	class expand_operand ops[5];
7076
7077	gcc_assert (flag_inline_atomics);
7078
7079	if (gimple_call_num_args (gs: call) == 5)
7080	model = get_memmodel (exp: gimple_call_arg (gs: call, index: 3));
7081
7082	rtx mem = get_builtin_sync_mem (loc: ptr, mode);
7083	rtx op = expand_expr_force_mode (exp: arg, mode);
7084
7085	switch (gimple_call_internal_fn (gs: call))
7086	{
7087	case IFN_ATOMIC_ADD_FETCH_CMP_0:
7088	code = PLUS;
7089	optab = atomic_add_fetch_cmp_0_optab;
7090	break;
7091	case IFN_ATOMIC_SUB_FETCH_CMP_0:
7092	code = MINUS;
7093	optab = atomic_sub_fetch_cmp_0_optab;
7094	break;
7095	case IFN_ATOMIC_AND_FETCH_CMP_0:
7096	code = AND;
7097	optab = atomic_and_fetch_cmp_0_optab;
7098	break;
7099	case IFN_ATOMIC_OR_FETCH_CMP_0:
7100	code = IOR;
7101	optab = atomic_or_fetch_cmp_0_optab;
7102	break;
7103	case IFN_ATOMIC_XOR_FETCH_CMP_0:
7104	code = XOR;
7105	optab = atomic_xor_fetch_cmp_0_optab;
7106	break;
7107	default:
7108	gcc_unreachable ();
7109	}
7110
7111	enum rtx_code comp = UNKNOWN;
7112	switch (tree_to_uhwi (cmp))
7113	{
7114	case ATOMIC_OP_FETCH_CMP_0_EQ: comp = EQ; break;
7115	case ATOMIC_OP_FETCH_CMP_0_NE: comp = NE; break;
7116	case ATOMIC_OP_FETCH_CMP_0_GT: comp = GT; break;
7117	case ATOMIC_OP_FETCH_CMP_0_GE: comp = GE; break;
7118	case ATOMIC_OP_FETCH_CMP_0_LT: comp = LT; break;
7119	case ATOMIC_OP_FETCH_CMP_0_LE: comp = LE; break;
7120	default: gcc_unreachable ();
7121	}
7122
7123	rtx target;
7124	if (lhs == NULL_TREE)
7125	target = gen_reg_rtx (TYPE_MODE (boolean_type_node));
7126	else
7127	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
7128	enum insn_code icode = direct_optab_handler (op: optab, mode);
7129	gcc_assert (icode != CODE_FOR_nothing);
7130	create_output_operand (op: &ops[0], x: target, TYPE_MODE (boolean_type_node));
7131	create_fixed_operand (op: &ops[1], x: mem);
7132	create_convert_operand_to (op: &ops[2], value: op, mode, unsigned_p: true);
7133	create_integer_operand (&ops[3], model);
7134	create_integer_operand (&ops[4], comp);
7135	if (maybe_expand_insn (icode, nops: 5, ops))
7136	return;
7137
7138	rtx result = expand_atomic_fetch_op (gen_reg_rtx (mode), mem, op,
7139	code, model, true);
7140	if (!result)
7141	{
7142	bool is_atomic = gimple_call_num_args (gs: call) == 5;
7143	tree tcall = gimple_call_arg (gs: call, index: 3 + is_atomic);
7144	tree fndecl = gimple_call_addr_fndecl (fn: tcall);
7145	tree type = TREE_TYPE (TREE_TYPE (fndecl));
7146	tree exp = build_call_nary (type, tcall,
7147	2 + is_atomic, ptr, arg,
7148	is_atomic
7149	? gimple_call_arg (gs: call, index: 3)
7150	: integer_zero_node);
7151	result = expand_builtin (exp, gen_reg_rtx (mode), NULL_RTX,
7152	mode, !lhs);
7153	}
7154
7155	if (lhs)
7156	{
7157	result = emit_store_flag_force (target, comp, result, const0_rtx, mode,
7158	0, 1);
7159	if (result != target)
7160	emit_move_insn (target, result);
7161	}
7162	}
7163
7164	/* Expand an atomic clear operation.
7165	void _atomic_clear (BOOL *obj, enum memmodel)
7166	EXP is the call expression. */
7167
7168	static rtx
7169	expand_builtin_atomic_clear (tree exp)
7170	{
7171	machine_mode mode = int_mode_for_size (BOOL_TYPE_SIZE, limit: 0).require ();
7172	rtx mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
7173	memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 1));
7174
7175	if (is_mm_consume (model) || is_mm_acquire (model) || is_mm_acq_rel (model))
7176	model = MEMMODEL_SEQ_CST;
7177
7178	/* Try issuing an __atomic_store, and allow fallback to __sync_lock_release.
7179	Failing that, a store is issued by __atomic_store. The only way this can
7180	fail is if the bool type is larger than a word size. Unlikely, but
7181	handle it anyway for completeness. Assume a single threaded model since
7182	there is no atomic support in this case, and no barriers are required. */
7183	rtx ret = expand_atomic_store (mem, const0_rtx, model, true);
7184	if (!ret)
7185	emit_move_insn (mem, const0_rtx);
7186	return const0_rtx;
7187	}
7188
7189	/* Expand an atomic test_and_set operation.
7190	bool _atomic_test_and_set (BOOL *obj, enum memmodel)
7191	EXP is the call expression. */
7192
7193	static rtx
7194	expand_builtin_atomic_test_and_set (tree exp, rtx target)
7195	{
7196	rtx mem;
7197	enum memmodel model;
7198	machine_mode mode;
7199
7200	mode = int_mode_for_size (BOOL_TYPE_SIZE, limit: 0).require ();
7201	mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
7202	model = get_memmodel (CALL_EXPR_ARG (exp, 1));
7203
7204	return expand_atomic_test_and_set (target, mem, model);
7205	}
7206
7207
7208	/* Return true if (optional) argument ARG1 of size ARG0 is always lock free on
7209	this architecture. If ARG1 is NULL, use typical alignment for size ARG0. */
7210
7211	static tree
7212	fold_builtin_atomic_always_lock_free (tree arg0, tree arg1)
7213	{
7214	int size;
7215	machine_mode mode;
7216	unsigned int mode_align, type_align;
7217
7218	if (TREE_CODE (arg0) != INTEGER_CST)
7219	return NULL_TREE;
7220
7221	/* We need a corresponding integer mode for the access to be lock-free. */
7222	size = INTVAL (expand_normal (arg0)) * BITS_PER_UNIT;
7223	if (!int_mode_for_size (size, limit: 0).exists (mode: &mode))
7224	return boolean_false_node;
7225
7226	mode_align = GET_MODE_ALIGNMENT (mode);
7227
7228	if (TREE_CODE (arg1) == INTEGER_CST)
7229	{
7230	unsigned HOST_WIDE_INT val = UINTVAL (expand_normal (arg1));
7231
7232	/* Either this argument is null, or it's a fake pointer encoding
7233	the alignment of the object. */
7234	val = least_bit_hwi (x: val);
7235	val *= BITS_PER_UNIT;
7236
7237	if (val == 0 || mode_align < val)
7238	type_align = mode_align;
7239	else
7240	type_align = val;
7241	}
7242	else
7243	{
7244	tree ttype = TREE_TYPE (arg1);
7245
7246	/* This function is usually invoked and folded immediately by the front
7247	end before anything else has a chance to look at it. The pointer
7248	parameter at this point is usually cast to a void *, so check for that
7249	and look past the cast. */
7250	if (CONVERT_EXPR_P (arg1)
7251	&& POINTER_TYPE_P (ttype)
7252	&& VOID_TYPE_P (TREE_TYPE (ttype))
7253	&& POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (arg1, 0))))
7254	arg1 = TREE_OPERAND (arg1, 0);
7255
7256	ttype = TREE_TYPE (arg1);
7257	gcc_assert (POINTER_TYPE_P (ttype));
7258
7259	/* Get the underlying type of the object. */
7260	ttype = TREE_TYPE (ttype);
7261	type_align = TYPE_ALIGN (ttype);
7262	}
7263
7264	/* If the object has smaller alignment, the lock free routines cannot
7265	be used. */
7266	if (type_align < mode_align)
7267	return boolean_false_node;
7268
7269	/* Check if a compare_and_swap pattern exists for the mode which represents
7270	the required size. The pattern is not allowed to fail, so the existence
7271	of the pattern indicates support is present. Also require that an
7272	atomic load exists for the required size. */
7273	if (can_compare_and_swap_p (mode, true) && can_atomic_load_p (mode))
7274	return boolean_true_node;
7275	else
7276	return boolean_false_node;
7277	}
7278
7279	/* Return true if the parameters to call EXP represent an object which will
7280	always generate lock free instructions. The first argument represents the
7281	size of the object, and the second parameter is a pointer to the object
7282	itself. If NULL is passed for the object, then the result is based on
7283	typical alignment for an object of the specified size. Otherwise return
7284	false. */
7285
7286	static rtx
7287	expand_builtin_atomic_always_lock_free (tree exp)
7288	{
7289	tree size;
7290	tree arg0 = CALL_EXPR_ARG (exp, 0);
7291	tree arg1 = CALL_EXPR_ARG (exp, 1);
7292
7293	if (TREE_CODE (arg0) != INTEGER_CST)
7294	{
7295	error ("non-constant argument 1 to %qs", "__atomic_always_lock_free");
7296	return const0_rtx;
7297	}
7298
7299	size = fold_builtin_atomic_always_lock_free (arg0, arg1);
7300	if (size == boolean_true_node)
7301	return const1_rtx;
7302	return const0_rtx;
7303	}
7304
7305	/* Return a one or zero if it can be determined that object ARG1 of size ARG
7306	is lock free on this architecture. */
7307
7308	static tree
7309	fold_builtin_atomic_is_lock_free (tree arg0, tree arg1)
7310	{
7311	if (!flag_inline_atomics)
7312	return NULL_TREE;
7313
7314	/* If it isn't always lock free, don't generate a result. */
7315	if (fold_builtin_atomic_always_lock_free (arg0, arg1) == boolean_true_node)
7316	return boolean_true_node;
7317
7318	return NULL_TREE;
7319	}
7320
7321	/* Return true if the parameters to call EXP represent an object which will
7322	always generate lock free instructions. The first argument represents the
7323	size of the object, and the second parameter is a pointer to the object
7324	itself. If NULL is passed for the object, then the result is based on
7325	typical alignment for an object of the specified size. Otherwise return
7326	NULL*/
7327
7328	static rtx
7329	expand_builtin_atomic_is_lock_free (tree exp)
7330	{
7331	tree size;
7332	tree arg0 = CALL_EXPR_ARG (exp, 0);
7333	tree arg1 = CALL_EXPR_ARG (exp, 1);
7334
7335	if (!INTEGRAL_TYPE_P (TREE_TYPE (arg0)))
7336	{
7337	error ("non-integer argument 1 to %qs", "__atomic_is_lock_free");
7338	return NULL_RTX;
7339	}
7340
7341	if (!flag_inline_atomics)
7342	return NULL_RTX;
7343
7344	/* If the value is known at compile time, return the RTX for it. */
7345	size = fold_builtin_atomic_is_lock_free (arg0, arg1);
7346	if (size == boolean_true_node)
7347	return const1_rtx;
7348
7349	return NULL_RTX;
7350	}
7351
7352	/* Expand the __atomic_thread_fence intrinsic:
7353	void __atomic_thread_fence (enum memmodel)
7354	EXP is the CALL_EXPR. */
7355
7356	static void
7357	expand_builtin_atomic_thread_fence (tree exp)
7358	{
7359	enum memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 0));
7360	expand_mem_thread_fence (model);
7361	}
7362
7363	/* Expand the __atomic_signal_fence intrinsic:
7364	void __atomic_signal_fence (enum memmodel)
7365	EXP is the CALL_EXPR. */
7366
7367	static void
7368	expand_builtin_atomic_signal_fence (tree exp)
7369	{
7370	enum memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 0));
7371	expand_mem_signal_fence (model);
7372	}
7373
7374	/* Expand the __sync_synchronize intrinsic. */
7375
7376	static void
7377	expand_builtin_sync_synchronize (void)
7378	{
7379	expand_mem_thread_fence (MEMMODEL_SYNC_SEQ_CST);
7380	}
7381
7382	static rtx
7383	expand_builtin_thread_pointer (tree exp, rtx target)
7384	{
7385	enum insn_code icode;
7386	if (!validate_arglist (callexpr: exp, VOID_TYPE))
7387	return const0_rtx;
7388	icode = direct_optab_handler (op: get_thread_pointer_optab, Pmode);
7389	if (icode != CODE_FOR_nothing)
7390	{
7391	class expand_operand op;
7392	/* If the target is not sutitable then create a new target. */
7393	if (target == NULL_RTX
7394	|| !REG_P (target)
7395	|| GET_MODE (target) != Pmode)
7396	target = gen_reg_rtx (Pmode);
7397	create_output_operand (op: &op, x: target, Pmode);
7398	expand_insn (icode, nops: 1, ops: &op);
7399	return target;
7400	}
7401	error ("%<__builtin_thread_pointer%> is not supported on this target");
7402	return const0_rtx;
7403	}
7404
7405	static void
7406	expand_builtin_set_thread_pointer (tree exp)
7407	{
7408	enum insn_code icode;
7409	if (!validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
7410	return;
7411	icode = direct_optab_handler (op: set_thread_pointer_optab, Pmode);
7412	if (icode != CODE_FOR_nothing)
7413	{
7414	class expand_operand op;
7415	rtx val = expand_expr (CALL_EXPR_ARG (exp, 0), NULL_RTX,
7416	Pmode, modifier: EXPAND_NORMAL);
7417	create_input_operand (op: &op, value: val, Pmode);
7418	expand_insn (icode, nops: 1, ops: &op);
7419	return;
7420	}
7421	error ("%<__builtin_set_thread_pointer%> is not supported on this target");
7422	}
7423
7424
7425	/* Emit code to restore the current value of stack. */
7426
7427	static void
7428	expand_stack_restore (tree var)
7429	{
7430	rtx_insn *prev;
7431	rtx sa = expand_normal (exp: var);
7432
7433	sa = convert_memory_address (Pmode, sa);
7434
7435	prev = get_last_insn ();
7436	emit_stack_restore (SAVE_BLOCK, sa);
7437
7438	record_new_stack_level ();
7439
7440	fixup_args_size_notes (prev, get_last_insn (), 0);
7441	}
7442
7443	/* Emit code to save the current value of stack. */
7444
7445	static rtx
7446	expand_stack_save (void)
7447	{
7448	rtx ret = NULL_RTX;
7449
7450	emit_stack_save (SAVE_BLOCK, &ret);
7451	return ret;
7452	}
7453
7454	/* Emit code to get the openacc gang, worker or vector id or size. */
7455
7456	static rtx
7457	expand_builtin_goacc_parlevel_id_size (tree exp, rtx target, int ignore)
7458	{
7459	const char *name;
7460	rtx fallback_retval;
7461	rtx_insn *(*gen_fn) (rtx, rtx);
7462	switch (DECL_FUNCTION_CODE (decl: get_callee_fndecl (exp)))
7463	{
7464	case BUILT_IN_GOACC_PARLEVEL_ID:
7465	name = "__builtin_goacc_parlevel_id";
7466	fallback_retval = const0_rtx;
7467	gen_fn = targetm.gen_oacc_dim_pos;
7468	break;
7469	case BUILT_IN_GOACC_PARLEVEL_SIZE:
7470	name = "__builtin_goacc_parlevel_size";
7471	fallback_retval = const1_rtx;
7472	gen_fn = targetm.gen_oacc_dim_size;
7473	break;
7474	default:
7475	gcc_unreachable ();
7476	}
7477
7478	if (oacc_get_fn_attrib (fn: current_function_decl) == NULL_TREE)
7479	{
7480	error ("%qs only supported in OpenACC code", name);
7481	return const0_rtx;
7482	}
7483
7484	tree arg = CALL_EXPR_ARG (exp, 0);
7485	if (TREE_CODE (arg) != INTEGER_CST)
7486	{
7487	error ("non-constant argument 0 to %qs", name);
7488	return const0_rtx;
7489	}
7490
7491	int dim = TREE_INT_CST_LOW (arg);
7492	switch (dim)
7493	{
7494	case GOMP_DIM_GANG:
7495	case GOMP_DIM_WORKER:
7496	case GOMP_DIM_VECTOR:
7497	break;
7498	default:
7499	error ("illegal argument 0 to %qs", name);
7500	return const0_rtx;
7501	}
7502
7503	if (ignore)
7504	return target;
7505
7506	if (target == NULL_RTX)
7507	target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
7508
7509	if (!targetm.have_oacc_dim_size ())
7510	{
7511	emit_move_insn (target, fallback_retval);
7512	return target;
7513	}
7514
7515	rtx reg = MEM_P (target) ? gen_reg_rtx (GET_MODE (target)) : target;
7516	emit_insn (gen_fn (reg, GEN_INT (dim)));
7517	if (reg != target)
7518	emit_move_insn (target, reg);
7519
7520	return target;
7521	}
7522
7523	/* Expand a string compare operation using a sequence of char comparison
7524	to get rid of the calling overhead, with result going to TARGET if
7525	that's convenient.
7526
7527	VAR_STR is the variable string source;
7528	CONST_STR is the constant string source;
7529	LENGTH is the number of chars to compare;
7530	CONST_STR_N indicates which source string is the constant string;
7531	IS_MEMCMP indicates whether it's a memcmp or strcmp.
7532
7533	to: (assume const_str_n is 2, i.e., arg2 is a constant string)
7534
7535	target = (int) (unsigned char) var_str[0]
7536	- (int) (unsigned char) const_str[0];
7537	if (target != 0)
7538	goto ne_label;
7539	...
7540	target = (int) (unsigned char) var_str[length - 2]
7541	- (int) (unsigned char) const_str[length - 2];
7542	if (target != 0)
7543	goto ne_label;
7544	target = (int) (unsigned char) var_str[length - 1]
7545	- (int) (unsigned char) const_str[length - 1];
7546	ne_label:
7547	*/
7548
7549	static rtx
7550	inline_string_cmp (rtx target, tree var_str, const char *const_str,
7551	unsigned HOST_WIDE_INT length,
7552	int const_str_n, machine_mode mode)
7553	{
7554	HOST_WIDE_INT offset = 0;
7555	rtx var_rtx_array
7556	= get_memory_rtx (exp: var_str, len: build_int_cst (unsigned_type_node,length));
7557	rtx var_rtx = NULL_RTX;
7558	rtx const_rtx = NULL_RTX;
7559	rtx result = target ? target : gen_reg_rtx (mode);
7560	rtx_code_label *ne_label = gen_label_rtx ();
7561	tree unit_type_node = unsigned_char_type_node;
7562	scalar_int_mode unit_mode
7563	= as_a <scalar_int_mode> TYPE_MODE (unit_type_node);
7564
7565	start_sequence ();
7566
7567	for (unsigned HOST_WIDE_INT i = 0; i < length; i++)
7568	{
7569	var_rtx
7570	= adjust_address (var_rtx_array, TYPE_MODE (unit_type_node), offset);
7571	const_rtx = c_readstr (str: const_str + offset, mode: unit_mode);
7572	rtx op0 = (const_str_n == 1) ? const_rtx : var_rtx;
7573	rtx op1 = (const_str_n == 1) ? var_rtx : const_rtx;
7574
7575	op0 = convert_modes (mode, oldmode: unit_mode, x: op0, unsignedp: 1);
7576	op1 = convert_modes (mode, oldmode: unit_mode, x: op1, unsignedp: 1);
7577	rtx diff = expand_simple_binop (mode, MINUS, op0, op1,
7578	result, 1, OPTAB_WIDEN);
7579
7580	/* Force the difference into result register. We cannot reassign
7581	result here ("result = diff") or we may end up returning
7582	uninitialized result when expand_simple_binop allocates a new
7583	pseudo-register for returning. */
7584	if (diff != result)
7585	emit_move_insn (result, diff);
7586
7587	if (i < length - 1)
7588	emit_cmp_and_jump_insns (result, CONST0_RTX (mode), NE, NULL_RTX,
7589	mode, true, ne_label);
7590	offset += GET_MODE_SIZE (mode: unit_mode);
7591	}
7592
7593	emit_label (ne_label);
7594	rtx_insn *insns = get_insns ();
7595	end_sequence ();
7596	emit_insn (insns);
7597
7598	return result;
7599	}
7600
7601	/* Inline expansion of a call to str(n)cmp and memcmp, with result going
7602	to TARGET if that's convenient.
7603	If the call is not been inlined, return NULL_RTX. */
7604
7605	static rtx
7606	inline_expand_builtin_bytecmp (tree exp, rtx target)
7607	{
7608	tree fndecl = get_callee_fndecl (exp);
7609	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
7610	bool is_ncmp = (fcode == BUILT_IN_STRNCMP || fcode == BUILT_IN_MEMCMP);
7611
7612	/* Do NOT apply this inlining expansion when optimizing for size or
7613	optimization level below 2 or if unused *cmp hasn't been DCEd. */
7614	if (optimize < 2 || optimize_insn_for_size_p () || target == const0_rtx)
7615	return NULL_RTX;
7616
7617	gcc_checking_assert (fcode == BUILT_IN_STRCMP
7618	|| fcode == BUILT_IN_STRNCMP
7619	|| fcode == BUILT_IN_MEMCMP);
7620
7621	/* On a target where the type of the call (int) has same or narrower presicion
7622	than unsigned char, give up the inlining expansion. */
7623	if (TYPE_PRECISION (unsigned_char_type_node)
7624	>= TYPE_PRECISION (TREE_TYPE (exp)))
7625	return NULL_RTX;
7626
7627	tree arg1 = CALL_EXPR_ARG (exp, 0);
7628	tree arg2 = CALL_EXPR_ARG (exp, 1);
7629	tree len3_tree = is_ncmp ? CALL_EXPR_ARG (exp, 2) : NULL_TREE;
7630
7631	unsigned HOST_WIDE_INT len1 = 0;
7632	unsigned HOST_WIDE_INT len2 = 0;
7633	unsigned HOST_WIDE_INT len3 = 0;
7634
7635	/* Get the object representation of the initializers of ARG1 and ARG2
7636	as strings, provided they refer to constant objects, with their byte
7637	sizes in LEN1 and LEN2, respectively. */
7638	const char *bytes1 = getbyterep (arg1, &len1);
7639	const char *bytes2 = getbyterep (arg2, &len2);
7640
7641	/* Fail if neither argument refers to an initialized constant. */
7642	if (!bytes1 && !bytes2)
7643	return NULL_RTX;
7644
7645	if (is_ncmp)
7646	{
7647	/* Fail if the memcmp/strncmp bound is not a constant. */
7648	if (!tree_fits_uhwi_p (len3_tree))
7649	return NULL_RTX;
7650
7651	len3 = tree_to_uhwi (len3_tree);
7652
7653	if (fcode == BUILT_IN_MEMCMP)
7654	{
7655	/* Fail if the memcmp bound is greater than the size of either
7656	of the two constant objects. */
7657	if ((bytes1 && len1 < len3)
7658	|| (bytes2 && len2 < len3))
7659	return NULL_RTX;
7660	}
7661	}
7662
7663	if (fcode != BUILT_IN_MEMCMP)
7664	{
7665	/* For string functions (i.e., strcmp and strncmp) reduce LEN1
7666	and LEN2 to the length of the nul-terminated string stored
7667	in each. */
7668	if (bytes1 != NULL)
7669	len1 = strnlen (string: bytes1, maxlen: len1) + 1;
7670	if (bytes2 != NULL)
7671	len2 = strnlen (string: bytes2, maxlen: len2) + 1;
7672	}
7673
7674	/* See inline_string_cmp. */
7675	int const_str_n;
7676	if (!len1)
7677	const_str_n = 2;
7678	else if (!len2)
7679	const_str_n = 1;
7680	else if (len2 > len1)
7681	const_str_n = 1;
7682	else
7683	const_str_n = 2;
7684
7685	/* For strncmp only, compute the new bound as the smallest of
7686	the lengths of the two strings (plus 1) and the bound provided
7687	to the function. */
7688	unsigned HOST_WIDE_INT bound = (const_str_n == 1) ? len1 : len2;
7689	if (is_ncmp && len3 < bound)
7690	bound = len3;
7691
7692	/* If the bound of the comparison is larger than the threshold,
7693	do nothing. */
7694	if (bound > (unsigned HOST_WIDE_INT) param_builtin_string_cmp_inline_length)
7695	return NULL_RTX;
7696
7697	machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
7698
7699	/* Now, start inline expansion the call. */
7700	return inline_string_cmp (target, var_str: (const_str_n == 1) ? arg2 : arg1,
7701	const_str: (const_str_n == 1) ? bytes1 : bytes2, length: bound,
7702	const_str_n, mode);
7703	}
7704
7705	/* Expand a call to __builtin_speculation_safe_value_<N>. MODE
7706	represents the size of the first argument to that call, or VOIDmode
7707	if the argument is a pointer. IGNORE will be true if the result
7708	isn't used. */
7709	static rtx
7710	expand_speculation_safe_value (machine_mode mode, tree exp, rtx target,
7711	bool ignore)
7712	{
7713	rtx val, failsafe;
7714	unsigned nargs = call_expr_nargs (exp);
7715
7716	tree arg0 = CALL_EXPR_ARG (exp, 0);
7717
7718	if (mode == VOIDmode)
7719	{
7720	mode = TYPE_MODE (TREE_TYPE (arg0));
7721	gcc_assert (GET_MODE_CLASS (mode) == MODE_INT);
7722	}
7723
7724	val = expand_expr (exp: arg0, NULL_RTX, mode, modifier: EXPAND_NORMAL);
7725
7726	/* An optional second argument can be used as a failsafe value on
7727	some machines. If it isn't present, then the failsafe value is
7728	assumed to be 0. */
7729	if (nargs > 1)
7730	{
7731	tree arg1 = CALL_EXPR_ARG (exp, 1);
7732	failsafe = expand_expr (exp: arg1, NULL_RTX, mode, modifier: EXPAND_NORMAL);
7733	}
7734	else
7735	failsafe = const0_rtx;
7736
7737	/* If the result isn't used, the behavior is undefined. It would be
7738	nice to emit a warning here, but path splitting means this might
7739	happen with legitimate code. So simply drop the builtin
7740	expansion in that case; we've handled any side-effects above. */
7741	if (ignore)
7742	return const0_rtx;
7743
7744	/* If we don't have a suitable target, create one to hold the result. */
7745	if (target == NULL || GET_MODE (target) != mode)
7746	target = gen_reg_rtx (mode);
7747
7748	if (GET_MODE (val) != mode && GET_MODE (val) != VOIDmode)
7749	val = convert_modes (mode, VOIDmode, x: val, unsignedp: false);
7750
7751	return targetm.speculation_safe_value (mode, target, val, failsafe);
7752	}
7753
7754	/* Expand an expression EXP that calls a built-in function,
7755	with result going to TARGET if that's convenient
7756	(and in mode MODE if that's convenient).
7757	SUBTARGET may be used as the target for computing one of EXP's operands.
7758	IGNORE is nonzero if the value is to be ignored. */
7759
7760	rtx
7761	expand_builtin (tree exp, rtx target, rtx subtarget, machine_mode mode,
7762	int ignore)
7763	{
7764	tree fndecl = get_callee_fndecl (exp);
7765	machine_mode target_mode = TYPE_MODE (TREE_TYPE (exp));
7766	int flags;
7767
7768	if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
7769	return targetm.expand_builtin (exp, target, subtarget, mode, ignore);
7770
7771	/* When ASan is enabled, we don't want to expand some memory/string
7772	builtins and rely on libsanitizer's hooks. This allows us to avoid
7773	redundant checks and be sure, that possible overflow will be detected
7774	by ASan. */
7775
7776	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
7777	if (param_asan_kernel_mem_intrinsic_prefix
7778	&& sanitize_flags_p (flag: SANITIZE_KERNEL_ADDRESS
7779	| SANITIZE_KERNEL_HWADDRESS))
7780	switch (fcode)
7781	{
7782	rtx save_decl_rtl, ret;
7783	case BUILT_IN_MEMCPY:
7784	case BUILT_IN_MEMMOVE:
7785	case BUILT_IN_MEMSET:
7786	save_decl_rtl = DECL_RTL (fndecl);
7787	DECL_RTL (fndecl) = asan_memfn_rtl (fndecl);
7788	ret = expand_call (exp, target, ignore);
7789	DECL_RTL (fndecl) = save_decl_rtl;
7790	return ret;
7791	default:
7792	break;
7793	}
7794	if (sanitize_flags_p (flag: SANITIZE_ADDRESS | SANITIZE_HWADDRESS)
7795	&& asan_intercepted_p (fcode))
7796	return expand_call (exp, target, ignore);
7797
7798	/* When not optimizing, generate calls to library functions for a certain
7799	set of builtins. */
7800	if (!optimize
7801	&& !called_as_built_in (node: fndecl)
7802	&& fcode != BUILT_IN_FORK
7803	&& fcode != BUILT_IN_EXECL
7804	&& fcode != BUILT_IN_EXECV
7805	&& fcode != BUILT_IN_EXECLP
7806	&& fcode != BUILT_IN_EXECLE
7807	&& fcode != BUILT_IN_EXECVP
7808	&& fcode != BUILT_IN_EXECVE
7809	&& fcode != BUILT_IN_CLEAR_CACHE
7810	&& !ALLOCA_FUNCTION_CODE_P (fcode)
7811	&& fcode != BUILT_IN_FREE
7812	&& (fcode != BUILT_IN_MEMSET
7813	|| !(flag_inline_stringops & ILSOP_MEMSET))
7814	&& (fcode != BUILT_IN_MEMCPY
7815	|| !(flag_inline_stringops & ILSOP_MEMCPY))
7816	&& (fcode != BUILT_IN_MEMMOVE
7817	|| !(flag_inline_stringops & ILSOP_MEMMOVE))
7818	&& (fcode != BUILT_IN_MEMCMP
7819	|| !(flag_inline_stringops & ILSOP_MEMCMP)))
7820	return expand_call (exp, target, ignore);
7821
7822	/* The built-in function expanders test for target == const0_rtx
7823	to determine whether the function's result will be ignored. */
7824	if (ignore)
7825	target = const0_rtx;
7826
7827	/* If the result of a pure or const built-in function is ignored, and
7828	none of its arguments are volatile, we can avoid expanding the
7829	built-in call and just evaluate the arguments for side-effects. */
7830	if (target == const0_rtx
7831	&& ((flags = flags_from_decl_or_type (fndecl)) & (ECF_CONST | ECF_PURE))
7832	&& !(flags & ECF_LOOPING_CONST_OR_PURE))
7833	{
7834	bool volatilep = false;
7835	tree arg;
7836	call_expr_arg_iterator iter;
7837
7838	FOR_EACH_CALL_EXPR_ARG (arg, iter, exp)
7839	if (TREE_THIS_VOLATILE (arg))
7840	{
7841	volatilep = true;
7842	break;
7843	}
7844
7845	if (! volatilep)
7846	{
7847	FOR_EACH_CALL_EXPR_ARG (arg, iter, exp)
7848	expand_expr (exp: arg, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
7849	return const0_rtx;
7850	}
7851	}
7852
7853	switch (fcode)
7854	{
7855	CASE_FLT_FN (BUILT_IN_FABS):
7856	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
7857	case BUILT_IN_FABSD32:
7858	case BUILT_IN_FABSD64:
7859	case BUILT_IN_FABSD128:
7860	target = expand_builtin_fabs (exp, target, subtarget);
7861	if (target)
7862	return target;
7863	break;
7864
7865	CASE_FLT_FN (BUILT_IN_COPYSIGN):
7866	CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
7867	target = expand_builtin_copysign (exp, target, subtarget);
7868	if (target)
7869	return target;
7870	break;
7871
7872	/* Just do a normal library call if we were unable to fold
7873	the values. */
7874	CASE_FLT_FN (BUILT_IN_CABS):
7875	CASE_FLT_FN_FLOATN_NX (BUILT_IN_CABS):
7876	break;
7877
7878	CASE_FLT_FN (BUILT_IN_FMA):
7879	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
7880	target = expand_builtin_mathfn_ternary (exp, target, subtarget);
7881	if (target)
7882	return target;
7883	break;
7884
7885	CASE_FLT_FN (BUILT_IN_ILOGB):
7886	if (! flag_unsafe_math_optimizations)
7887	break;
7888	gcc_fallthrough ();
7889	CASE_FLT_FN (BUILT_IN_ISINF):
7890	CASE_FLT_FN (BUILT_IN_FINITE):
7891	case BUILT_IN_ISFINITE:
7892	case BUILT_IN_ISNORMAL:
7893	target = expand_builtin_interclass_mathfn (exp, target);
7894	if (target)
7895	return target;
7896	break;
7897
7898	case BUILT_IN_ISSIGNALING:
7899	target = expand_builtin_issignaling (exp, target);
7900	if (target)
7901	return target;
7902	break;
7903
7904	CASE_FLT_FN (BUILT_IN_ICEIL):
7905	CASE_FLT_FN (BUILT_IN_LCEIL):
7906	CASE_FLT_FN (BUILT_IN_LLCEIL):
7907	CASE_FLT_FN (BUILT_IN_LFLOOR):
7908	CASE_FLT_FN (BUILT_IN_IFLOOR):
7909	CASE_FLT_FN (BUILT_IN_LLFLOOR):
7910	target = expand_builtin_int_roundingfn (exp, target);
7911	if (target)
7912	return target;
7913	break;
7914
7915	CASE_FLT_FN (BUILT_IN_IRINT):
7916	CASE_FLT_FN (BUILT_IN_LRINT):
7917	CASE_FLT_FN (BUILT_IN_LLRINT):
7918	CASE_FLT_FN (BUILT_IN_IROUND):
7919	CASE_FLT_FN (BUILT_IN_LROUND):
7920	CASE_FLT_FN (BUILT_IN_LLROUND):
7921	target = expand_builtin_int_roundingfn_2 (exp, target);
7922	if (target)
7923	return target;
7924	break;
7925
7926	CASE_FLT_FN (BUILT_IN_POWI):
7927	target = expand_builtin_powi (exp, target);
7928	if (target)
7929	return target;
7930	break;
7931
7932	CASE_FLT_FN (BUILT_IN_CEXPI):
7933	target = expand_builtin_cexpi (exp, target);
7934	gcc_assert (target);
7935	return target;
7936
7937	CASE_FLT_FN (BUILT_IN_SIN):
7938	CASE_FLT_FN (BUILT_IN_COS):
7939	if (! flag_unsafe_math_optimizations)
7940	break;
7941	target = expand_builtin_mathfn_3 (exp, target, subtarget);
7942	if (target)
7943	return target;
7944	break;
7945
7946	CASE_FLT_FN (BUILT_IN_SINCOS):
7947	if (! flag_unsafe_math_optimizations)
7948	break;
7949	target = expand_builtin_sincos (exp);
7950	if (target)
7951	return target;
7952	break;
7953
7954	case BUILT_IN_FEGETROUND:
7955	target = expand_builtin_fegetround (exp, target, target_mode);
7956	if (target)
7957	return target;
7958	break;
7959
7960	case BUILT_IN_FECLEAREXCEPT:
7961	target = expand_builtin_feclear_feraise_except (exp, target, target_mode,
7962	op_optab: feclearexcept_optab);
7963	if (target)
7964	return target;
7965	break;
7966
7967	case BUILT_IN_FERAISEEXCEPT:
7968	target = expand_builtin_feclear_feraise_except (exp, target, target_mode,
7969	op_optab: feraiseexcept_optab);
7970	if (target)
7971	return target;
7972	break;
7973
7974	case BUILT_IN_APPLY_ARGS:
7975	return expand_builtin_apply_args ();
7976
7977	/* __builtin_apply (FUNCTION, ARGUMENTS, ARGSIZE) invokes
7978	FUNCTION with a copy of the parameters described by
7979	ARGUMENTS, and ARGSIZE. It returns a block of memory
7980	allocated on the stack into which is stored all the registers
7981	that might possibly be used for returning the result of a
7982	function. ARGUMENTS is the value returned by
7983	__builtin_apply_args. ARGSIZE is the number of bytes of
7984	arguments that must be copied. ??? How should this value be
7985	computed? We'll also need a safe worst case value for varargs
7986	functions. */
7987	case BUILT_IN_APPLY:
7988	if (!validate_arglist (callexpr: exp, POINTER_TYPE,
7989	POINTER_TYPE, INTEGER_TYPE, VOID_TYPE)
7990	&& !validate_arglist (callexpr: exp, REFERENCE_TYPE,
7991	POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
7992	return const0_rtx;
7993	else
7994	{
7995	rtx ops[3];
7996
7997	ops[0] = expand_normal (CALL_EXPR_ARG (exp, 0));
7998	ops[1] = expand_normal (CALL_EXPR_ARG (exp, 1));
7999	ops[2] = expand_normal (CALL_EXPR_ARG (exp, 2));
8000
8001	return expand_builtin_apply (function: ops[0], arguments: ops[1], argsize: ops[2]);
8002	}
8003
8004	/* __builtin_return (RESULT) causes the function to return the
8005	value described by RESULT. RESULT is address of the block of
8006	memory returned by __builtin_apply. */
8007	case BUILT_IN_RETURN:
8008	if (validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
8009	expand_builtin_return (result: expand_normal (CALL_EXPR_ARG (exp, 0)));
8010	return const0_rtx;
8011
8012	case BUILT_IN_SAVEREGS:
8013	return expand_builtin_saveregs ();
8014
8015	case BUILT_IN_VA_ARG_PACK:
8016	/* All valid uses of __builtin_va_arg_pack () are removed during
8017	inlining. */
8018	error ("invalid use of %<__builtin_va_arg_pack ()%>");
8019	return const0_rtx;
8020
8021	case BUILT_IN_VA_ARG_PACK_LEN:
8022	/* All valid uses of __builtin_va_arg_pack_len () are removed during
8023	inlining. */
8024	error ("invalid use of %<__builtin_va_arg_pack_len ()%>");
8025	return const0_rtx;
8026
8027	/* Return the address of the first anonymous stack arg. */
8028	case BUILT_IN_NEXT_ARG:
8029	if (fold_builtin_next_arg (exp, false))
8030	return const0_rtx;
8031	return expand_builtin_next_arg ();
8032
8033	case BUILT_IN_CLEAR_CACHE:
8034	expand_builtin___clear_cache (exp);
8035	return const0_rtx;
8036
8037	case BUILT_IN_CLASSIFY_TYPE:
8038	return expand_builtin_classify_type (exp);
8039
8040	case BUILT_IN_CONSTANT_P:
8041	return const0_rtx;
8042
8043	case BUILT_IN_FRAME_ADDRESS:
8044	case BUILT_IN_RETURN_ADDRESS:
8045	return expand_builtin_frame_address (fndecl, exp);
8046
8047	case BUILT_IN_STACK_ADDRESS:
8048	return expand_builtin_stack_address ();
8049
8050	case BUILT_IN___STRUB_ENTER:
8051	target = expand_builtin_strub_enter (exp);
8052	if (target)
8053	return target;
8054	break;
8055
8056	case BUILT_IN___STRUB_UPDATE:
8057	target = expand_builtin_strub_update (exp);
8058	if (target)
8059	return target;
8060	break;
8061
8062	case BUILT_IN___STRUB_LEAVE:
8063	target = expand_builtin_strub_leave (exp);
8064	if (target)
8065	return target;
8066	break;
8067
8068	/* Returns the address of the area where the structure is returned.
8069	0 otherwise. */
8070	case BUILT_IN_AGGREGATE_INCOMING_ADDRESS:
8071	if (call_expr_nargs (exp) != 0
8072	|| ! AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl)))
8073	|| !MEM_P (DECL_RTL (DECL_RESULT (current_function_decl))))
8074	return const0_rtx;
8075	else
8076	return XEXP (DECL_RTL (DECL_RESULT (current_function_decl)), 0);
8077
8078	CASE_BUILT_IN_ALLOCA:
8079	target = expand_builtin_alloca (exp);
8080	if (target)
8081	return target;
8082	break;
8083
8084	case BUILT_IN_ASAN_ALLOCAS_UNPOISON:
8085	return expand_asan_emit_allocas_unpoison (exp);
8086
8087	case BUILT_IN_STACK_SAVE:
8088	return expand_stack_save ();
8089
8090	case BUILT_IN_STACK_RESTORE:
8091	expand_stack_restore (CALL_EXPR_ARG (exp, 0));
8092	return const0_rtx;
8093
8094	case BUILT_IN_BSWAP16:
8095	case BUILT_IN_BSWAP32:
8096	case BUILT_IN_BSWAP64:
8097	case BUILT_IN_BSWAP128:
8098	target = expand_builtin_bswap (target_mode, exp, target, subtarget);
8099	if (target)
8100	return target;
8101	break;
8102
8103	CASE_INT_FN (BUILT_IN_FFS):
8104	target = expand_builtin_unop (target_mode, exp, target,
8105	subtarget, op_optab: ffs_optab);
8106	if (target)
8107	return target;
8108	break;
8109
8110	CASE_INT_FN (BUILT_IN_CLZ):
8111	target = expand_builtin_unop (target_mode, exp, target,
8112	subtarget, op_optab: clz_optab);
8113	if (target)
8114	return target;
8115	break;
8116
8117	CASE_INT_FN (BUILT_IN_CTZ):
8118	target = expand_builtin_unop (target_mode, exp, target,
8119	subtarget, op_optab: ctz_optab);
8120	if (target)
8121	return target;
8122	break;
8123
8124	CASE_INT_FN (BUILT_IN_CLRSB):
8125	target = expand_builtin_unop (target_mode, exp, target,
8126	subtarget, op_optab: clrsb_optab);
8127	if (target)
8128	return target;
8129	break;
8130
8131	CASE_INT_FN (BUILT_IN_POPCOUNT):
8132	target = expand_builtin_unop (target_mode, exp, target,
8133	subtarget, op_optab: popcount_optab);
8134	if (target)
8135	return target;
8136	break;
8137
8138	CASE_INT_FN (BUILT_IN_PARITY):
8139	target = expand_builtin_unop (target_mode, exp, target,
8140	subtarget, op_optab: parity_optab);
8141	if (target)
8142	return target;
8143	break;
8144
8145	case BUILT_IN_STRLEN:
8146	target = expand_builtin_strlen (exp, target, target_mode);
8147	if (target)
8148	return target;
8149	break;
8150
8151	case BUILT_IN_STRNLEN:
8152	target = expand_builtin_strnlen (exp, target, target_mode);
8153	if (target)
8154	return target;
8155	break;
8156
8157	case BUILT_IN_STRCPY:
8158	target = expand_builtin_strcpy (exp, target);
8159	if (target)
8160	return target;
8161	break;
8162
8163	case BUILT_IN_STRNCPY:
8164	target = expand_builtin_strncpy (exp, target);
8165	if (target)
8166	return target;
8167	break;
8168
8169	case BUILT_IN_STPCPY:
8170	target = expand_builtin_stpcpy (exp, target, mode);
8171	if (target)
8172	return target;
8173	break;
8174
8175	case BUILT_IN_MEMCPY:
8176	target = expand_builtin_memcpy (exp, target);
8177	if (target)
8178	return target;
8179	break;
8180
8181	case BUILT_IN_MEMMOVE:
8182	target = expand_builtin_memmove (exp, target);
8183	if (target)
8184	return target;
8185	break;
8186
8187	case BUILT_IN_MEMPCPY:
8188	target = expand_builtin_mempcpy (exp, target);
8189	if (target)
8190	return target;
8191	break;
8192
8193	case BUILT_IN_MEMSET:
8194	target = expand_builtin_memset (exp, target, mode);
8195	if (target)
8196	return target;
8197	break;
8198
8199	case BUILT_IN_BZERO:
8200	target = expand_builtin_bzero (exp);
8201	if (target)
8202	return target;
8203	break;
8204
8205	/* Expand it as BUILT_IN_MEMCMP_EQ first. If not successful, change it
8206	back to a BUILT_IN_STRCMP. Remember to delete the 3rd parameter
8207	when changing it to a strcmp call. */
8208	case BUILT_IN_STRCMP_EQ:
8209	target = expand_builtin_memcmp (exp, target, result_eq: true);
8210	if (target)
8211	return target;
8212
8213	/* Change this call back to a BUILT_IN_STRCMP. */
8214	TREE_OPERAND (exp, 1)
8215	= build_fold_addr_expr (builtin_decl_explicit (BUILT_IN_STRCMP));
8216
8217	/* Delete the last parameter. */
8218	unsigned int i;
8219	vec<tree, va_gc> *arg_vec;
8220	vec_alloc (v&: arg_vec, nelems: 2);
8221	for (i = 0; i < 2; i++)
8222	arg_vec->quick_push (CALL_EXPR_ARG (exp, i));
8223	exp = build_call_vec (TREE_TYPE (exp), CALL_EXPR_FN (exp), arg_vec);
8224	/* FALLTHROUGH */
8225
8226	case BUILT_IN_STRCMP:
8227	target = expand_builtin_strcmp (exp, target);
8228	if (target)
8229	return target;
8230	break;
8231
8232	/* Expand it as BUILT_IN_MEMCMP_EQ first. If not successful, change it
8233	back to a BUILT_IN_STRNCMP. */
8234	case BUILT_IN_STRNCMP_EQ:
8235	target = expand_builtin_memcmp (exp, target, result_eq: true);
8236	if (target)
8237	return target;
8238
8239	/* Change it back to a BUILT_IN_STRNCMP. */
8240	TREE_OPERAND (exp, 1)
8241	= build_fold_addr_expr (builtin_decl_explicit (BUILT_IN_STRNCMP));
8242	/* FALLTHROUGH */
8243
8244	case BUILT_IN_STRNCMP:
8245	target = expand_builtin_strncmp (exp, target, mode);
8246	if (target)
8247	return target;
8248	break;
8249
8250	case BUILT_IN_BCMP:
8251	case BUILT_IN_MEMCMP:
8252	case BUILT_IN_MEMCMP_EQ:
8253	target = expand_builtin_memcmp (exp, target, result_eq: fcode == BUILT_IN_MEMCMP_EQ);
8254	if (target)
8255	return target;
8256	if (fcode == BUILT_IN_MEMCMP_EQ)
8257	{
8258	tree newdecl = builtin_decl_explicit (fncode: BUILT_IN_MEMCMP);
8259	TREE_OPERAND (exp, 1) = build_fold_addr_expr (newdecl);
8260	}
8261	break;
8262
8263	case BUILT_IN_SETJMP:
8264	/* This should have been lowered to the builtins below. */
8265	gcc_unreachable ();
8266
8267	case BUILT_IN_SETJMP_SETUP:
8268	/* __builtin_setjmp_setup is passed a pointer to an array of five words
8269	and the receiver label. */
8270	if (validate_arglist (callexpr: exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
8271	{
8272	rtx buf_addr = expand_expr (CALL_EXPR_ARG (exp, 0), target: subtarget,
8273	VOIDmode, modifier: EXPAND_NORMAL);
8274	tree label = TREE_OPERAND (CALL_EXPR_ARG (exp, 1), 0);
8275	rtx_insn *label_r = label_rtx (label);
8276
8277	expand_builtin_setjmp_setup (buf_addr, receiver_label: label_r);
8278	return const0_rtx;
8279	}
8280	break;
8281
8282	case BUILT_IN_SETJMP_RECEIVER:
8283	/* __builtin_setjmp_receiver is passed the receiver label. */
8284	if (validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
8285	{
8286	tree label = TREE_OPERAND (CALL_EXPR_ARG (exp, 0), 0);
8287	rtx_insn *label_r = label_rtx (label);
8288
8289	expand_builtin_setjmp_receiver (receiver_label: label_r);
8290	nonlocal_goto_handler_labels
8291	= gen_rtx_INSN_LIST (VOIDmode, label_r,
8292	nonlocal_goto_handler_labels);
8293	/* ??? Do not let expand_label treat us as such since we would
8294	not want to be both on the list of non-local labels and on
8295	the list of forced labels. */
8296	FORCED_LABEL (label) = 0;
8297	return const0_rtx;
8298	}
8299	break;
8300
8301	/* __builtin_longjmp is passed a pointer to an array of five words.
8302	It's similar to the C library longjmp function but works with
8303	__builtin_setjmp above. */
8304	case BUILT_IN_LONGJMP:
8305	if (validate_arglist (callexpr: exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
8306	{
8307	rtx buf_addr = expand_expr (CALL_EXPR_ARG (exp, 0), target: subtarget,
8308	VOIDmode, modifier: EXPAND_NORMAL);
8309	rtx value = expand_normal (CALL_EXPR_ARG (exp, 1));
8310
8311	if (value != const1_rtx)
8312	{
8313	error ("%<__builtin_longjmp%> second argument must be 1");
8314	return const0_rtx;
8315	}
8316
8317	expand_builtin_longjmp (buf_addr, value);
8318	return const0_rtx;
8319	}
8320	break;
8321
8322	case BUILT_IN_NONLOCAL_GOTO:
8323	target = expand_builtin_nonlocal_goto (exp);
8324	if (target)
8325	return target;
8326	break;
8327
8328	/* This updates the setjmp buffer that is its argument with the value
8329	of the current stack pointer. */
8330	case BUILT_IN_UPDATE_SETJMP_BUF:
8331	if (validate_arglist (callexpr: exp, POINTER_TYPE, VOID_TYPE))
8332	{
8333	rtx buf_addr
8334	= expand_normal (CALL_EXPR_ARG (exp, 0));
8335
8336	expand_builtin_update_setjmp_buf (buf_addr);
8337	return const0_rtx;
8338	}
8339	break;
8340
8341	case BUILT_IN_TRAP:
8342	case BUILT_IN_UNREACHABLE_TRAP:
8343	expand_builtin_trap ();
8344	return const0_rtx;
8345
8346	case BUILT_IN_UNREACHABLE:
8347	expand_builtin_unreachable ();
8348	return const0_rtx;
8349
8350	CASE_FLT_FN (BUILT_IN_SIGNBIT):
8351	case BUILT_IN_SIGNBITD32:
8352	case BUILT_IN_SIGNBITD64:
8353	case BUILT_IN_SIGNBITD128:
8354	target = expand_builtin_signbit (exp, target);
8355	if (target)
8356	return target;
8357	break;
8358
8359	/* Various hooks for the DWARF 2 __throw routine. */
8360	case BUILT_IN_UNWIND_INIT:
8361	expand_builtin_unwind_init ();
8362	return const0_rtx;
8363	case BUILT_IN_DWARF_CFA:
8364	return virtual_cfa_rtx;
8365	#ifdef DWARF2_UNWIND_INFO
8366	case BUILT_IN_DWARF_SP_COLUMN:
8367	return expand_builtin_dwarf_sp_column ();
8368	case BUILT_IN_INIT_DWARF_REG_SIZES:
8369	expand_builtin_init_dwarf_reg_sizes (CALL_EXPR_ARG (exp, 0));
8370	return const0_rtx;
8371	#endif
8372	case BUILT_IN_FROB_RETURN_ADDR:
8373	return expand_builtin_frob_return_addr (CALL_EXPR_ARG (exp, 0));
8374	case BUILT_IN_EXTRACT_RETURN_ADDR:
8375	return expand_builtin_extract_return_addr (CALL_EXPR_ARG (exp, 0));
8376	case BUILT_IN_EH_RETURN:
8377	expand_builtin_eh_return (CALL_EXPR_ARG (exp, 0),
8378	CALL_EXPR_ARG (exp, 1));
8379	return const0_rtx;
8380	case BUILT_IN_EH_RETURN_DATA_REGNO:
8381	return expand_builtin_eh_return_data_regno (exp);
8382	case BUILT_IN_EXTEND_POINTER:
8383	return expand_builtin_extend_pointer (CALL_EXPR_ARG (exp, 0));
8384	case BUILT_IN_EH_POINTER:
8385	return expand_builtin_eh_pointer (exp);
8386	case BUILT_IN_EH_FILTER:
8387	return expand_builtin_eh_filter (exp);
8388	case BUILT_IN_EH_COPY_VALUES:
8389	return expand_builtin_eh_copy_values (exp);
8390
8391	case BUILT_IN_VA_START:
8392	return expand_builtin_va_start (exp);
8393	case BUILT_IN_VA_END:
8394	return expand_builtin_va_end (exp);
8395	case BUILT_IN_VA_COPY:
8396	return expand_builtin_va_copy (exp);
8397	case BUILT_IN_EXPECT:
8398	return expand_builtin_expect (exp, target);
8399	case BUILT_IN_EXPECT_WITH_PROBABILITY:
8400	return expand_builtin_expect_with_probability (exp, target);
8401	case BUILT_IN_ASSUME_ALIGNED:
8402	return expand_builtin_assume_aligned (exp, target);
8403	case BUILT_IN_PREFETCH:
8404	expand_builtin_prefetch (exp);
8405	return const0_rtx;
8406
8407	case BUILT_IN_INIT_TRAMPOLINE:
8408	return expand_builtin_init_trampoline (exp, onstack: true);
8409	case BUILT_IN_INIT_HEAP_TRAMPOLINE:
8410	return expand_builtin_init_trampoline (exp, onstack: false);
8411	case BUILT_IN_ADJUST_TRAMPOLINE:
8412	return expand_builtin_adjust_trampoline (exp);
8413
8414	case BUILT_IN_INIT_DESCRIPTOR:
8415	return expand_builtin_init_descriptor (exp);
8416	case BUILT_IN_ADJUST_DESCRIPTOR:
8417	return expand_builtin_adjust_descriptor (exp);
8418
8419	case BUILT_IN_GCC_NESTED_PTR_CREATED:
8420	case BUILT_IN_GCC_NESTED_PTR_DELETED:
8421	break; /* At present, no expansion, just call the function. */
8422
8423	case BUILT_IN_FORK:
8424	case BUILT_IN_EXECL:
8425	case BUILT_IN_EXECV:
8426	case BUILT_IN_EXECLP:
8427	case BUILT_IN_EXECLE:
8428	case BUILT_IN_EXECVP:
8429	case BUILT_IN_EXECVE:
8430	target = expand_builtin_fork_or_exec (fn: fndecl, exp, target, ignore);
8431	if (target)
8432	return target;
8433	break;
8434
8435	case BUILT_IN_SYNC_FETCH_AND_ADD_1:
8436	case BUILT_IN_SYNC_FETCH_AND_ADD_2:
8437	case BUILT_IN_SYNC_FETCH_AND_ADD_4:
8438	case BUILT_IN_SYNC_FETCH_AND_ADD_8:
8439	case BUILT_IN_SYNC_FETCH_AND_ADD_16:
8440	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_FETCH_AND_ADD_1);
8441	target = expand_builtin_sync_operation (mode, exp, code: PLUS, after: false, target);
8442	if (target)
8443	return target;
8444	break;
8445
8446	case BUILT_IN_SYNC_FETCH_AND_SUB_1:
8447	case BUILT_IN_SYNC_FETCH_AND_SUB_2:
8448	case BUILT_IN_SYNC_FETCH_AND_SUB_4:
8449	case BUILT_IN_SYNC_FETCH_AND_SUB_8:
8450	case BUILT_IN_SYNC_FETCH_AND_SUB_16:
8451	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_FETCH_AND_SUB_1);
8452	target = expand_builtin_sync_operation (mode, exp, code: MINUS, after: false, target);
8453	if (target)
8454	return target;
8455	break;
8456
8457	case BUILT_IN_SYNC_FETCH_AND_OR_1:
8458	case BUILT_IN_SYNC_FETCH_AND_OR_2:
8459	case BUILT_IN_SYNC_FETCH_AND_OR_4:
8460	case BUILT_IN_SYNC_FETCH_AND_OR_8:
8461	case BUILT_IN_SYNC_FETCH_AND_OR_16:
8462	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_FETCH_AND_OR_1);
8463	target = expand_builtin_sync_operation (mode, exp, code: IOR, after: false, target);
8464	if (target)
8465	return target;
8466	break;
8467
8468	case BUILT_IN_SYNC_FETCH_AND_AND_1:
8469	case BUILT_IN_SYNC_FETCH_AND_AND_2:
8470	case BUILT_IN_SYNC_FETCH_AND_AND_4:
8471	case BUILT_IN_SYNC_FETCH_AND_AND_8:
8472	case BUILT_IN_SYNC_FETCH_AND_AND_16:
8473	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_FETCH_AND_AND_1);
8474	target = expand_builtin_sync_operation (mode, exp, code: AND, after: false, target);
8475	if (target)
8476	return target;
8477	break;
8478
8479	case BUILT_IN_SYNC_FETCH_AND_XOR_1:
8480	case BUILT_IN_SYNC_FETCH_AND_XOR_2:
8481	case BUILT_IN_SYNC_FETCH_AND_XOR_4:
8482	case BUILT_IN_SYNC_FETCH_AND_XOR_8:
8483	case BUILT_IN_SYNC_FETCH_AND_XOR_16:
8484	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_FETCH_AND_XOR_1);
8485	target = expand_builtin_sync_operation (mode, exp, code: XOR, after: false, target);
8486	if (target)
8487	return target;
8488	break;
8489
8490	case BUILT_IN_SYNC_FETCH_AND_NAND_1:
8491	case BUILT_IN_SYNC_FETCH_AND_NAND_2:
8492	case BUILT_IN_SYNC_FETCH_AND_NAND_4:
8493	case BUILT_IN_SYNC_FETCH_AND_NAND_8:
8494	case BUILT_IN_SYNC_FETCH_AND_NAND_16:
8495	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_FETCH_AND_NAND_1);
8496	target = expand_builtin_sync_operation (mode, exp, code: NOT, after: false, target);
8497	if (target)
8498	return target;
8499	break;
8500
8501	case BUILT_IN_SYNC_ADD_AND_FETCH_1:
8502	case BUILT_IN_SYNC_ADD_AND_FETCH_2:
8503	case BUILT_IN_SYNC_ADD_AND_FETCH_4:
8504	case BUILT_IN_SYNC_ADD_AND_FETCH_8:
8505	case BUILT_IN_SYNC_ADD_AND_FETCH_16:
8506	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_ADD_AND_FETCH_1);
8507	target = expand_builtin_sync_operation (mode, exp, code: PLUS, after: true, target);
8508	if (target)
8509	return target;
8510	break;
8511
8512	case BUILT_IN_SYNC_SUB_AND_FETCH_1:
8513	case BUILT_IN_SYNC_SUB_AND_FETCH_2:
8514	case BUILT_IN_SYNC_SUB_AND_FETCH_4:
8515	case BUILT_IN_SYNC_SUB_AND_FETCH_8:
8516	case BUILT_IN_SYNC_SUB_AND_FETCH_16:
8517	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_SUB_AND_FETCH_1);
8518	target = expand_builtin_sync_operation (mode, exp, code: MINUS, after: true, target);
8519	if (target)
8520	return target;
8521	break;
8522
8523	case BUILT_IN_SYNC_OR_AND_FETCH_1:
8524	case BUILT_IN_SYNC_OR_AND_FETCH_2:
8525	case BUILT_IN_SYNC_OR_AND_FETCH_4:
8526	case BUILT_IN_SYNC_OR_AND_FETCH_8:
8527	case BUILT_IN_SYNC_OR_AND_FETCH_16:
8528	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_OR_AND_FETCH_1);
8529	target = expand_builtin_sync_operation (mode, exp, code: IOR, after: true, target);
8530	if (target)
8531	return target;
8532	break;
8533
8534	case BUILT_IN_SYNC_AND_AND_FETCH_1:
8535	case BUILT_IN_SYNC_AND_AND_FETCH_2:
8536	case BUILT_IN_SYNC_AND_AND_FETCH_4:
8537	case BUILT_IN_SYNC_AND_AND_FETCH_8:
8538	case BUILT_IN_SYNC_AND_AND_FETCH_16:
8539	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_AND_AND_FETCH_1);
8540	target = expand_builtin_sync_operation (mode, exp, code: AND, after: true, target);
8541	if (target)
8542	return target;
8543	break;
8544
8545	case BUILT_IN_SYNC_XOR_AND_FETCH_1:
8546	case BUILT_IN_SYNC_XOR_AND_FETCH_2:
8547	case BUILT_IN_SYNC_XOR_AND_FETCH_4:
8548	case BUILT_IN_SYNC_XOR_AND_FETCH_8:
8549	case BUILT_IN_SYNC_XOR_AND_FETCH_16:
8550	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_XOR_AND_FETCH_1);
8551	target = expand_builtin_sync_operation (mode, exp, code: XOR, after: true, target);
8552	if (target)
8553	return target;
8554	break;
8555
8556	case BUILT_IN_SYNC_NAND_AND_FETCH_1:
8557	case BUILT_IN_SYNC_NAND_AND_FETCH_2:
8558	case BUILT_IN_SYNC_NAND_AND_FETCH_4:
8559	case BUILT_IN_SYNC_NAND_AND_FETCH_8:
8560	case BUILT_IN_SYNC_NAND_AND_FETCH_16:
8561	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_NAND_AND_FETCH_1);
8562	target = expand_builtin_sync_operation (mode, exp, code: NOT, after: true, target);
8563	if (target)
8564	return target;
8565	break;
8566
8567	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1:
8568	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_2:
8569	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_4:
8570	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_8:
8571	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_16:
8572	if (mode == VOIDmode)
8573	mode = TYPE_MODE (boolean_type_node);
8574	if (!target || !register_operand (target, mode))
8575	target = gen_reg_rtx (mode);
8576
8577	mode = get_builtin_sync_mode
8578	(fcode_diff: fcode - BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1);
8579	target = expand_builtin_compare_and_swap (mode, exp, is_bool: true, target);
8580	if (target)
8581	return target;
8582	break;
8583
8584	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1:
8585	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_2:
8586	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_4:
8587	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_8:
8588	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_16:
8589	mode = get_builtin_sync_mode
8590	(fcode_diff: fcode - BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1);
8591	target = expand_builtin_compare_and_swap (mode, exp, is_bool: false, target);
8592	if (target)
8593	return target;
8594	break;
8595
8596	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_1:
8597	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_2:
8598	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_4:
8599	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_8:
8600	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_16:
8601	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_LOCK_TEST_AND_SET_1);
8602	target = expand_builtin_sync_lock_test_and_set (mode, exp, target);
8603	if (target)
8604	return target;
8605	break;
8606
8607	case BUILT_IN_SYNC_LOCK_RELEASE_1:
8608	case BUILT_IN_SYNC_LOCK_RELEASE_2:
8609	case BUILT_IN_SYNC_LOCK_RELEASE_4:
8610	case BUILT_IN_SYNC_LOCK_RELEASE_8:
8611	case BUILT_IN_SYNC_LOCK_RELEASE_16:
8612	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SYNC_LOCK_RELEASE_1);
8613	expand_builtin_sync_lock_release (mode, exp);
8614	return const0_rtx;
8615
8616	case BUILT_IN_SYNC_SYNCHRONIZE:
8617	expand_builtin_sync_synchronize ();
8618	return const0_rtx;
8619
8620	case BUILT_IN_ATOMIC_EXCHANGE_1:
8621	case BUILT_IN_ATOMIC_EXCHANGE_2:
8622	case BUILT_IN_ATOMIC_EXCHANGE_4:
8623	case BUILT_IN_ATOMIC_EXCHANGE_8:
8624	case BUILT_IN_ATOMIC_EXCHANGE_16:
8625	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_EXCHANGE_1);
8626	target = expand_builtin_atomic_exchange (mode, exp, target);
8627	if (target)
8628	return target;
8629	break;
8630
8631	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
8632	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
8633	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
8634	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
8635	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
8636	{
8637	unsigned int nargs, z;
8638	vec<tree, va_gc> *vec;
8639
8640	mode =
8641	get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1);
8642	target = expand_builtin_atomic_compare_exchange (mode, exp, target);
8643	if (target)
8644	return target;
8645
8646	/* If this is turned into an external library call, the weak parameter
8647	must be dropped to match the expected parameter list. */
8648	nargs = call_expr_nargs (exp);
8649	vec_alloc (v&: vec, nelems: nargs - 1);
8650	for (z = 0; z < 3; z++)
8651	vec->quick_push (CALL_EXPR_ARG (exp, z));
8652	/* Skip the boolean weak parameter. */
8653	for (z = 4; z < 6; z++)
8654	vec->quick_push (CALL_EXPR_ARG (exp, z));
8655	exp = build_call_vec (TREE_TYPE (exp), CALL_EXPR_FN (exp), vec);
8656	break;
8657	}
8658
8659	case BUILT_IN_ATOMIC_LOAD_1:
8660	case BUILT_IN_ATOMIC_LOAD_2:
8661	case BUILT_IN_ATOMIC_LOAD_4:
8662	case BUILT_IN_ATOMIC_LOAD_8:
8663	case BUILT_IN_ATOMIC_LOAD_16:
8664	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_LOAD_1);
8665	target = expand_builtin_atomic_load (mode, exp, target);
8666	if (target)
8667	return target;
8668	break;
8669
8670	case BUILT_IN_ATOMIC_STORE_1:
8671	case BUILT_IN_ATOMIC_STORE_2:
8672	case BUILT_IN_ATOMIC_STORE_4:
8673	case BUILT_IN_ATOMIC_STORE_8:
8674	case BUILT_IN_ATOMIC_STORE_16:
8675	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_STORE_1);
8676	target = expand_builtin_atomic_store (mode, exp);
8677	if (target)
8678	return const0_rtx;
8679	break;
8680
8681	case BUILT_IN_ATOMIC_ADD_FETCH_1:
8682	case BUILT_IN_ATOMIC_ADD_FETCH_2:
8683	case BUILT_IN_ATOMIC_ADD_FETCH_4:
8684	case BUILT_IN_ATOMIC_ADD_FETCH_8:
8685	case BUILT_IN_ATOMIC_ADD_FETCH_16:
8686	{
8687	enum built_in_function lib;
8688	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_ADD_FETCH_1);
8689	lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_ADD_1 +
8690	(fcode - BUILT_IN_ATOMIC_ADD_FETCH_1));
8691	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: PLUS, fetch_after: true,
8692	ignore, ext_call: lib);
8693	if (target)
8694	return target;
8695	break;
8696	}
8697	case BUILT_IN_ATOMIC_SUB_FETCH_1:
8698	case BUILT_IN_ATOMIC_SUB_FETCH_2:
8699	case BUILT_IN_ATOMIC_SUB_FETCH_4:
8700	case BUILT_IN_ATOMIC_SUB_FETCH_8:
8701	case BUILT_IN_ATOMIC_SUB_FETCH_16:
8702	{
8703	enum built_in_function lib;
8704	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_SUB_FETCH_1);
8705	lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_SUB_1 +
8706	(fcode - BUILT_IN_ATOMIC_SUB_FETCH_1));
8707	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: MINUS, fetch_after: true,
8708	ignore, ext_call: lib);
8709	if (target)
8710	return target;
8711	break;
8712	}
8713	case BUILT_IN_ATOMIC_AND_FETCH_1:
8714	case BUILT_IN_ATOMIC_AND_FETCH_2:
8715	case BUILT_IN_ATOMIC_AND_FETCH_4:
8716	case BUILT_IN_ATOMIC_AND_FETCH_8:
8717	case BUILT_IN_ATOMIC_AND_FETCH_16:
8718	{
8719	enum built_in_function lib;
8720	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_AND_FETCH_1);
8721	lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_AND_1 +
8722	(fcode - BUILT_IN_ATOMIC_AND_FETCH_1));
8723	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: AND, fetch_after: true,
8724	ignore, ext_call: lib);
8725	if (target)
8726	return target;
8727	break;
8728	}
8729	case BUILT_IN_ATOMIC_NAND_FETCH_1:
8730	case BUILT_IN_ATOMIC_NAND_FETCH_2:
8731	case BUILT_IN_ATOMIC_NAND_FETCH_4:
8732	case BUILT_IN_ATOMIC_NAND_FETCH_8:
8733	case BUILT_IN_ATOMIC_NAND_FETCH_16:
8734	{
8735	enum built_in_function lib;
8736	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_NAND_FETCH_1);
8737	lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_NAND_1 +
8738	(fcode - BUILT_IN_ATOMIC_NAND_FETCH_1));
8739	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: NOT, fetch_after: true,
8740	ignore, ext_call: lib);
8741	if (target)
8742	return target;
8743	break;
8744	}
8745	case BUILT_IN_ATOMIC_XOR_FETCH_1:
8746	case BUILT_IN_ATOMIC_XOR_FETCH_2:
8747	case BUILT_IN_ATOMIC_XOR_FETCH_4:
8748	case BUILT_IN_ATOMIC_XOR_FETCH_8:
8749	case BUILT_IN_ATOMIC_XOR_FETCH_16:
8750	{
8751	enum built_in_function lib;
8752	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_XOR_FETCH_1);
8753	lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_XOR_1 +
8754	(fcode - BUILT_IN_ATOMIC_XOR_FETCH_1));
8755	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: XOR, fetch_after: true,
8756	ignore, ext_call: lib);
8757	if (target)
8758	return target;
8759	break;
8760	}
8761	case BUILT_IN_ATOMIC_OR_FETCH_1:
8762	case BUILT_IN_ATOMIC_OR_FETCH_2:
8763	case BUILT_IN_ATOMIC_OR_FETCH_4:
8764	case BUILT_IN_ATOMIC_OR_FETCH_8:
8765	case BUILT_IN_ATOMIC_OR_FETCH_16:
8766	{
8767	enum built_in_function lib;
8768	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_OR_FETCH_1);
8769	lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_OR_1 +
8770	(fcode - BUILT_IN_ATOMIC_OR_FETCH_1));
8771	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: IOR, fetch_after: true,
8772	ignore, ext_call: lib);
8773	if (target)
8774	return target;
8775	break;
8776	}
8777	case BUILT_IN_ATOMIC_FETCH_ADD_1:
8778	case BUILT_IN_ATOMIC_FETCH_ADD_2:
8779	case BUILT_IN_ATOMIC_FETCH_ADD_4:
8780	case BUILT_IN_ATOMIC_FETCH_ADD_8:
8781	case BUILT_IN_ATOMIC_FETCH_ADD_16:
8782	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_FETCH_ADD_1);
8783	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: PLUS, fetch_after: false,
8784	ignore, ext_call: BUILT_IN_NONE);
8785	if (target)
8786	return target;
8787	break;
8788
8789	case BUILT_IN_ATOMIC_FETCH_SUB_1:
8790	case BUILT_IN_ATOMIC_FETCH_SUB_2:
8791	case BUILT_IN_ATOMIC_FETCH_SUB_4:
8792	case BUILT_IN_ATOMIC_FETCH_SUB_8:
8793	case BUILT_IN_ATOMIC_FETCH_SUB_16:
8794	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_FETCH_SUB_1);
8795	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: MINUS, fetch_after: false,
8796	ignore, ext_call: BUILT_IN_NONE);
8797	if (target)
8798	return target;
8799	break;
8800
8801	case BUILT_IN_ATOMIC_FETCH_AND_1:
8802	case BUILT_IN_ATOMIC_FETCH_AND_2:
8803	case BUILT_IN_ATOMIC_FETCH_AND_4:
8804	case BUILT_IN_ATOMIC_FETCH_AND_8:
8805	case BUILT_IN_ATOMIC_FETCH_AND_16:
8806	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_FETCH_AND_1);
8807	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: AND, fetch_after: false,
8808	ignore, ext_call: BUILT_IN_NONE);
8809	if (target)
8810	return target;
8811	break;
8812
8813	case BUILT_IN_ATOMIC_FETCH_NAND_1:
8814	case BUILT_IN_ATOMIC_FETCH_NAND_2:
8815	case BUILT_IN_ATOMIC_FETCH_NAND_4:
8816	case BUILT_IN_ATOMIC_FETCH_NAND_8:
8817	case BUILT_IN_ATOMIC_FETCH_NAND_16:
8818	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_FETCH_NAND_1);
8819	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: NOT, fetch_after: false,
8820	ignore, ext_call: BUILT_IN_NONE);
8821	if (target)
8822	return target;
8823	break;
8824
8825	case BUILT_IN_ATOMIC_FETCH_XOR_1:
8826	case BUILT_IN_ATOMIC_FETCH_XOR_2:
8827	case BUILT_IN_ATOMIC_FETCH_XOR_4:
8828	case BUILT_IN_ATOMIC_FETCH_XOR_8:
8829	case BUILT_IN_ATOMIC_FETCH_XOR_16:
8830	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_FETCH_XOR_1);
8831	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: XOR, fetch_after: false,
8832	ignore, ext_call: BUILT_IN_NONE);
8833	if (target)
8834	return target;
8835	break;
8836
8837	case BUILT_IN_ATOMIC_FETCH_OR_1:
8838	case BUILT_IN_ATOMIC_FETCH_OR_2:
8839	case BUILT_IN_ATOMIC_FETCH_OR_4:
8840	case BUILT_IN_ATOMIC_FETCH_OR_8:
8841	case BUILT_IN_ATOMIC_FETCH_OR_16:
8842	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_ATOMIC_FETCH_OR_1);
8843	target = expand_builtin_atomic_fetch_op (mode, exp, target, code: IOR, fetch_after: false,
8844	ignore, ext_call: BUILT_IN_NONE);
8845	if (target)
8846	return target;
8847	break;
8848
8849	case BUILT_IN_ATOMIC_TEST_AND_SET:
8850	target = expand_builtin_atomic_test_and_set (exp, target);
8851	if (target)
8852	return target;
8853	break;
8854
8855	case BUILT_IN_ATOMIC_CLEAR:
8856	return expand_builtin_atomic_clear (exp);
8857
8858	case BUILT_IN_ATOMIC_ALWAYS_LOCK_FREE:
8859	return expand_builtin_atomic_always_lock_free (exp);
8860
8861	case BUILT_IN_ATOMIC_IS_LOCK_FREE:
8862	target = expand_builtin_atomic_is_lock_free (exp);
8863	if (target)
8864	return target;
8865	break;
8866
8867	case BUILT_IN_ATOMIC_THREAD_FENCE:
8868	expand_builtin_atomic_thread_fence (exp);
8869	return const0_rtx;
8870
8871	case BUILT_IN_ATOMIC_SIGNAL_FENCE:
8872	expand_builtin_atomic_signal_fence (exp);
8873	return const0_rtx;
8874
8875	case BUILT_IN_OBJECT_SIZE:
8876	case BUILT_IN_DYNAMIC_OBJECT_SIZE:
8877	return expand_builtin_object_size (exp);
8878
8879	case BUILT_IN_MEMCPY_CHK:
8880	case BUILT_IN_MEMPCPY_CHK:
8881	case BUILT_IN_MEMMOVE_CHK:
8882	case BUILT_IN_MEMSET_CHK:
8883	target = expand_builtin_memory_chk (exp, target, mode, fcode);
8884	if (target)
8885	return target;
8886	break;
8887
8888	case BUILT_IN_STRCPY_CHK:
8889	case BUILT_IN_STPCPY_CHK:
8890	case BUILT_IN_STRNCPY_CHK:
8891	case BUILT_IN_STPNCPY_CHK:
8892	case BUILT_IN_STRCAT_CHK:
8893	case BUILT_IN_STRNCAT_CHK:
8894	case BUILT_IN_SNPRINTF_CHK:
8895	case BUILT_IN_VSNPRINTF_CHK:
8896	maybe_emit_chk_warning (exp, fcode);
8897	break;
8898
8899	case BUILT_IN_SPRINTF_CHK:
8900	case BUILT_IN_VSPRINTF_CHK:
8901	maybe_emit_sprintf_chk_warning (exp, fcode);
8902	break;
8903
8904	case BUILT_IN_THREAD_POINTER:
8905	return expand_builtin_thread_pointer (exp, target);
8906
8907	case BUILT_IN_SET_THREAD_POINTER:
8908	expand_builtin_set_thread_pointer (exp);
8909	return const0_rtx;
8910
8911	case BUILT_IN_ACC_ON_DEVICE:
8912	/* Do library call, if we failed to expand the builtin when
8913	folding. */
8914	break;
8915
8916	case BUILT_IN_GOACC_PARLEVEL_ID:
8917	case BUILT_IN_GOACC_PARLEVEL_SIZE:
8918	return expand_builtin_goacc_parlevel_id_size (exp, target, ignore);
8919
8920	case BUILT_IN_SPECULATION_SAFE_VALUE_PTR:
8921	return expand_speculation_safe_value (VOIDmode, exp, target, ignore);
8922
8923	case BUILT_IN_SPECULATION_SAFE_VALUE_1:
8924	case BUILT_IN_SPECULATION_SAFE_VALUE_2:
8925	case BUILT_IN_SPECULATION_SAFE_VALUE_4:
8926	case BUILT_IN_SPECULATION_SAFE_VALUE_8:
8927	case BUILT_IN_SPECULATION_SAFE_VALUE_16:
8928	mode = get_builtin_sync_mode (fcode_diff: fcode - BUILT_IN_SPECULATION_SAFE_VALUE_1);
8929	return expand_speculation_safe_value (mode, exp, target, ignore);
8930
8931	default: /* just do library call, if unknown builtin */
8932	break;
8933	}
8934
8935	/* The switch statement above can drop through to cause the function
8936	to be called normally. */
8937	return expand_call (exp, target, ignore);
8938	}
8939
8940	/* Determine whether a tree node represents a call to a built-in
8941	function. If the tree T is a call to a built-in function with
8942	the right number of arguments of the appropriate types, return
8943	the DECL_FUNCTION_CODE of the call, e.g. BUILT_IN_SQRT.
8944	Otherwise the return value is END_BUILTINS. */
8945
8946	enum built_in_function
8947	builtin_mathfn_code (const_tree t)
8948	{
8949	const_tree fndecl, arg, parmlist;
8950	const_tree argtype, parmtype;
8951	const_call_expr_arg_iterator iter;
8952
8953	if (TREE_CODE (t) != CALL_EXPR)
8954	return END_BUILTINS;
8955
8956	fndecl = get_callee_fndecl (t);
8957	if (fndecl == NULL_TREE || !fndecl_built_in_p (node: fndecl, klass: BUILT_IN_NORMAL))
8958	return END_BUILTINS;
8959
8960	parmlist = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
8961	init_const_call_expr_arg_iterator (exp: t, iter: &iter);
8962	for (; parmlist; parmlist = TREE_CHAIN (parmlist))
8963	{
8964	/* If a function doesn't take a variable number of arguments,
8965	the last element in the list will have type `void'. */
8966	parmtype = TREE_VALUE (parmlist);
8967	if (VOID_TYPE_P (parmtype))
8968	{
8969	if (more_const_call_expr_args_p (iter: &iter))
8970	return END_BUILTINS;
8971	return DECL_FUNCTION_CODE (decl: fndecl);
8972	}
8973
8974	if (! more_const_call_expr_args_p (iter: &iter))
8975	return END_BUILTINS;
8976
8977	arg = next_const_call_expr_arg (iter: &iter);
8978	argtype = TREE_TYPE (arg);
8979
8980	if (SCALAR_FLOAT_TYPE_P (parmtype))
8981	{
8982	if (! SCALAR_FLOAT_TYPE_P (argtype))
8983	return END_BUILTINS;
8984	}
8985	else if (COMPLEX_FLOAT_TYPE_P (parmtype))
8986	{
8987	if (! COMPLEX_FLOAT_TYPE_P (argtype))
8988	return END_BUILTINS;
8989	}
8990	else if (POINTER_TYPE_P (parmtype))
8991	{
8992	if (! POINTER_TYPE_P (argtype))
8993	return END_BUILTINS;
8994	}
8995	else if (INTEGRAL_TYPE_P (parmtype))
8996	{
8997	if (! INTEGRAL_TYPE_P (argtype))
8998	return END_BUILTINS;
8999	}
9000	else
9001	return END_BUILTINS;
9002	}
9003
9004	/* Variable-length argument list. */
9005	return DECL_FUNCTION_CODE (decl: fndecl);
9006	}
9007
9008	/* Fold a call to __builtin_constant_p, if we know its argument ARG will
9009	evaluate to a constant. */
9010
9011	static tree
9012	fold_builtin_constant_p (tree arg)
9013	{
9014	/* We return 1 for a numeric type that's known to be a constant
9015	value at compile-time or for an aggregate type that's a
9016	literal constant. */
9017	STRIP_NOPS (arg);
9018
9019	/* If we know this is a constant, emit the constant of one. */
9020	if (CONSTANT_CLASS_P (arg)
9021	|| (TREE_CODE (arg) == CONSTRUCTOR
9022	&& TREE_CONSTANT (arg)))
9023	return integer_one_node;
9024	if (TREE_CODE (arg) == ADDR_EXPR)
9025	{
9026	tree op = TREE_OPERAND (arg, 0);
9027	if (TREE_CODE (op) == STRING_CST
9028	|| (TREE_CODE (op) == ARRAY_REF
9029	&& integer_zerop (TREE_OPERAND (op, 1))
9030	&& TREE_CODE (TREE_OPERAND (op, 0)) == STRING_CST))
9031	return integer_one_node;
9032	}
9033
9034	/* If this expression has side effects, show we don't know it to be a
9035	constant. Likewise if it's a pointer or aggregate type since in
9036	those case we only want literals, since those are only optimized
9037	when generating RTL, not later.
9038	And finally, if we are compiling an initializer, not code, we
9039	need to return a definite result now; there's not going to be any
9040	more optimization done. */
9041	if (TREE_SIDE_EFFECTS (arg)
9042	|| AGGREGATE_TYPE_P (TREE_TYPE (arg))
9043	|| POINTER_TYPE_P (TREE_TYPE (arg))
9044	|| cfun == 0
9045	|| folding_initializer
9046	|| force_folding_builtin_constant_p)
9047	return integer_zero_node;
9048
9049	return NULL_TREE;
9050	}
9051
9052	/* Create builtin_expect or builtin_expect_with_probability
9053	with PRED and EXPECTED as its arguments and return it as a truthvalue.
9054	Fortran FE can also produce builtin_expect with PREDICTOR as third argument.
9055	builtin_expect_with_probability instead uses third argument as PROBABILITY
9056	value. */
9057
9058	static tree
9059	build_builtin_expect_predicate (location_t loc, tree pred, tree expected,
9060	tree predictor, tree probability)
9061	{
9062	tree fn, arg_types, pred_type, expected_type, call_expr, ret_type;
9063
9064	fn = builtin_decl_explicit (fncode: probability == NULL_TREE ? BUILT_IN_EXPECT
9065	: BUILT_IN_EXPECT_WITH_PROBABILITY);
9066	arg_types = TYPE_ARG_TYPES (TREE_TYPE (fn));
9067	ret_type = TREE_TYPE (TREE_TYPE (fn));
9068	pred_type = TREE_VALUE (arg_types);
9069	expected_type = TREE_VALUE (TREE_CHAIN (arg_types));
9070
9071	pred = fold_convert_loc (loc, pred_type, pred);
9072	expected = fold_convert_loc (loc, expected_type, expected);
9073
9074	if (probability)
9075	call_expr = build_call_expr_loc (loc, fn, 3, pred, expected, probability);
9076	else
9077	call_expr = build_call_expr_loc (loc, fn, predictor ? 3 : 2, pred, expected,
9078	predictor);
9079
9080	return build2 (NE_EXPR, TREE_TYPE (pred), call_expr,
9081	build_int_cst (ret_type, 0));
9082	}
9083
9084	/* Fold a call to builtin_expect with arguments ARG0, ARG1, ARG2, ARG3. Return
9085	NULL_TREE if no simplification is possible. */
9086
9087	tree
9088	fold_builtin_expect (location_t loc, tree arg0, tree arg1, tree arg2,
9089	tree arg3)
9090	{
9091	tree inner, fndecl, inner_arg0;
9092	enum tree_code code;
9093
9094	/* Distribute the expected value over short-circuiting operators.
9095	See through the cast from truthvalue_type_node to long. */
9096	inner_arg0 = arg0;
9097	while (CONVERT_EXPR_P (inner_arg0)
9098	&& INTEGRAL_TYPE_P (TREE_TYPE (inner_arg0))
9099	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (inner_arg0, 0))))
9100	inner_arg0 = TREE_OPERAND (inner_arg0, 0);
9101
9102	/* If this is a builtin_expect within a builtin_expect keep the
9103	inner one. See through a comparison against a constant. It
9104	might have been added to create a thruthvalue. */
9105	inner = inner_arg0;
9106
9107	if (COMPARISON_CLASS_P (inner)
9108	&& TREE_CODE (TREE_OPERAND (inner, 1)) == INTEGER_CST)
9109	inner = TREE_OPERAND (inner, 0);
9110
9111	if (TREE_CODE (inner) == CALL_EXPR
9112	&& (fndecl = get_callee_fndecl (inner))
9113	&& fndecl_built_in_p (node: fndecl, name1: BUILT_IN_EXPECT,
9114	names: BUILT_IN_EXPECT_WITH_PROBABILITY))
9115	return arg0;
9116
9117	inner = inner_arg0;
9118	code = TREE_CODE (inner);
9119	if (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR)
9120	{
9121	tree op0 = TREE_OPERAND (inner, 0);
9122	tree op1 = TREE_OPERAND (inner, 1);
9123	arg1 = save_expr (arg1);
9124
9125	op0 = build_builtin_expect_predicate (loc, pred: op0, expected: arg1, predictor: arg2, probability: arg3);
9126	op1 = build_builtin_expect_predicate (loc, pred: op1, expected: arg1, predictor: arg2, probability: arg3);
9127	inner = build2 (code, TREE_TYPE (inner), op0, op1);
9128
9129	return fold_convert_loc (loc, TREE_TYPE (arg0), inner);
9130	}
9131
9132	/* If the argument isn't invariant then there's nothing else we can do. */
9133	if (!TREE_CONSTANT (inner_arg0))
9134	return NULL_TREE;
9135
9136	/* If we expect that a comparison against the argument will fold to
9137	a constant return the constant. In practice, this means a true
9138	constant or the address of a non-weak symbol. */
9139	inner = inner_arg0;
9140	STRIP_NOPS (inner);
9141	if (TREE_CODE (inner) == ADDR_EXPR)
9142	{
9143	do
9144	{
9145	inner = TREE_OPERAND (inner, 0);
9146	}
9147	while (TREE_CODE (inner) == COMPONENT_REF
9148	|| TREE_CODE (inner) == ARRAY_REF);
9149	if (VAR_OR_FUNCTION_DECL_P (inner) && DECL_WEAK (inner))
9150	return NULL_TREE;
9151	}
9152
9153	/* Otherwise, ARG0 already has the proper type for the return value. */
9154	return arg0;
9155	}
9156
9157	/* Fold a call to __builtin_classify_type with argument ARG. */
9158
9159	static tree
9160	fold_builtin_classify_type (tree arg)
9161	{
9162	if (arg == 0)
9163	return build_int_cst (integer_type_node, no_type_class);
9164
9165	return build_int_cst (integer_type_node, type_to_class (TREE_TYPE (arg)));
9166	}
9167
9168	/* Fold a call EXPR (which may be null) to __builtin_strlen with argument
9169	ARG. */
9170
9171	static tree
9172	fold_builtin_strlen (location_t loc, tree expr, tree type, tree arg)
9173	{
9174	if (!validate_arg (arg, code: POINTER_TYPE))
9175	return NULL_TREE;
9176	else
9177	{
9178	c_strlen_data lendata = { };
9179	tree len = c_strlen (arg, only_value: 0, data: &lendata);
9180
9181	if (len)
9182	return fold_convert_loc (loc, type, len);
9183
9184	/* TODO: Move this to gimple-ssa-warn-access once the pass runs
9185	also early enough to detect invalid reads in multimensional
9186	arrays and struct members. */
9187	if (!lendata.decl)
9188	c_strlen (arg, only_value: 1, data: &lendata);
9189
9190	if (lendata.decl)
9191	{
9192	if (EXPR_HAS_LOCATION (arg))
9193	loc = EXPR_LOCATION (arg);
9194	else if (loc == UNKNOWN_LOCATION)
9195	loc = input_location;
9196	warn_string_no_nul (loc, expr, "strlen", arg, lendata.decl);
9197	}
9198
9199	return NULL_TREE;
9200	}
9201	}
9202
9203	/* Fold a call to __builtin_inf or __builtin_huge_val. */
9204
9205	static tree
9206	fold_builtin_inf (location_t loc, tree type, int warn)
9207	{
9208	/* __builtin_inff is intended to be usable to define INFINITY on all
9209	targets. If an infinity is not available, INFINITY expands "to a
9210	positive constant of type float that overflows at translation
9211	time", footnote "In this case, using INFINITY will violate the
9212	constraint in 6.4.4 and thus require a diagnostic." (C99 7.12#4).
9213	Thus we pedwarn to ensure this constraint violation is
9214	diagnosed. */
9215	if (!MODE_HAS_INFINITIES (TYPE_MODE (type)) && warn)
9216	pedwarn (loc, 0, "target format does not support infinity");
9217
9218	return build_real (type, dconstinf);
9219	}
9220
9221	/* Fold function call to builtin sincos, sincosf, or sincosl. Return
9222	NULL_TREE if no simplification can be made. */
9223
9224	static tree
9225	fold_builtin_sincos (location_t loc,
9226	tree arg0, tree arg1, tree arg2)
9227	{
9228	tree type;
9229	tree fndecl, call = NULL_TREE;
9230
9231	if (!validate_arg (arg0, code: REAL_TYPE)
9232	|| !validate_arg (arg1, code: POINTER_TYPE)
9233	|| !validate_arg (arg2, code: POINTER_TYPE))
9234	return NULL_TREE;
9235
9236	type = TREE_TYPE (arg0);
9237
9238	/* Calculate the result when the argument is a constant. */
9239	built_in_function fn = mathfn_built_in_2 (type, fn: CFN_BUILT_IN_CEXPI);
9240	if (fn == END_BUILTINS)
9241	return NULL_TREE;
9242
9243	/* Canonicalize sincos to cexpi. */
9244	if (TREE_CODE (arg0) == REAL_CST)
9245	{
9246	tree complex_type = build_complex_type (type);
9247	call = fold_const_call (as_combined_fn (fn), complex_type, arg0);
9248	}
9249	if (!call)
9250	{
9251	if (!targetm.libc_has_function (function_c99_math_complex, type)
9252	|| !builtin_decl_implicit_p (fncode: fn))
9253	return NULL_TREE;
9254	fndecl = builtin_decl_explicit (fncode: fn);
9255	call = build_call_expr_loc (loc, fndecl, 1, arg0);
9256	call = builtin_save_expr (exp: call);
9257	}
9258
9259	tree ptype = build_pointer_type (type);
9260	arg1 = fold_convert (ptype, arg1);
9261	arg2 = fold_convert (ptype, arg2);
9262	return build2 (COMPOUND_EXPR, void_type_node,
9263	build2 (MODIFY_EXPR, void_type_node,
9264	build_fold_indirect_ref_loc (loc, arg1),
9265	fold_build1_loc (loc, IMAGPART_EXPR, type, call)),
9266	build2 (MODIFY_EXPR, void_type_node,
9267	build_fold_indirect_ref_loc (loc, arg2),
9268	fold_build1_loc (loc, REALPART_EXPR, type, call)));
9269	}
9270
9271	/* Fold function call to builtin memcmp with arguments ARG1 and ARG2.
9272	Return NULL_TREE if no simplification can be made. */
9273
9274	static tree
9275	fold_builtin_memcmp (location_t loc, tree arg1, tree arg2, tree len)
9276	{
9277	if (!validate_arg (arg1, code: POINTER_TYPE)
9278	|| !validate_arg (arg2, code: POINTER_TYPE)
9279	|| !validate_arg (len, code: INTEGER_TYPE))
9280	return NULL_TREE;
9281
9282	/* If the LEN parameter is zero, return zero. */
9283	if (integer_zerop (len))
9284	return omit_two_operands_loc (loc, integer_type_node, integer_zero_node,
9285	arg1, arg2);
9286
9287	/* If ARG1 and ARG2 are the same (and not volatile), return zero. */
9288	if (operand_equal_p (arg1, arg2, flags: 0))
9289	return omit_one_operand_loc (loc, integer_type_node, integer_zero_node, len);
9290
9291	/* If len parameter is one, return an expression corresponding to
9292	(*(const unsigned char*)arg1 - (const unsigned char*)arg2). */
9293	if (tree_fits_uhwi_p (len) && tree_to_uhwi (len) == 1)
9294	{
9295	tree cst_uchar_node = build_type_variant (unsigned_char_type_node, 1, 0);
9296	tree cst_uchar_ptr_node
9297	= build_pointer_type_for_mode (cst_uchar_node, ptr_mode, true);
9298
9299	tree ind1
9300	= fold_convert_loc (loc, integer_type_node,
9301	build1 (INDIRECT_REF, cst_uchar_node,
9302	fold_convert_loc (loc,
9303	cst_uchar_ptr_node,
9304	arg1)));
9305	tree ind2
9306	= fold_convert_loc (loc, integer_type_node,
9307	build1 (INDIRECT_REF, cst_uchar_node,
9308	fold_convert_loc (loc,
9309	cst_uchar_ptr_node,
9310	arg2)));
9311	return fold_build2_loc (loc, MINUS_EXPR, integer_type_node, ind1, ind2);
9312	}
9313
9314	return NULL_TREE;
9315	}
9316
9317	/* Fold a call to builtin isascii with argument ARG. */
9318
9319	static tree
9320	fold_builtin_isascii (location_t loc, tree arg)
9321	{
9322	if (!validate_arg (arg, code: INTEGER_TYPE))
9323	return NULL_TREE;
9324	else
9325	{
9326	/* Transform isascii(c) -> ((c & ~0x7f) == 0). */
9327	arg = fold_build2 (BIT_AND_EXPR, integer_type_node, arg,
9328	build_int_cst (integer_type_node,
9329	~ HOST_WIDE_INT_UC (0x7f)));
9330	return fold_build2_loc (loc, EQ_EXPR, integer_type_node,
9331	arg, integer_zero_node);
9332	}
9333	}
9334
9335	/* Fold a call to builtin toascii with argument ARG. */
9336
9337	static tree
9338	fold_builtin_toascii (location_t loc, tree arg)
9339	{
9340	if (!validate_arg (arg, code: INTEGER_TYPE))
9341	return NULL_TREE;
9342
9343	/* Transform toascii(c) -> (c & 0x7f). */
9344	return fold_build2_loc (loc, BIT_AND_EXPR, integer_type_node, arg,
9345	build_int_cst (integer_type_node, 0x7f));
9346	}
9347
9348	/* Fold a call to builtin isdigit with argument ARG. */
9349
9350	static tree
9351	fold_builtin_isdigit (location_t loc, tree arg)
9352	{
9353	if (!validate_arg (arg, code: INTEGER_TYPE))
9354	return NULL_TREE;
9355	else
9356	{
9357	/* Transform isdigit(c) -> (unsigned)(c) - '0' <= 9. */
9358	/* According to the C standard, isdigit is unaffected by locale.
9359	However, it definitely is affected by the target character set. */
9360	unsigned HOST_WIDE_INT target_digit0
9361	= lang_hooks.to_target_charset ('0');
9362
9363	if (target_digit0 == 0)
9364	return NULL_TREE;
9365
9366	arg = fold_convert_loc (loc, unsigned_type_node, arg);
9367	arg = fold_build2 (MINUS_EXPR, unsigned_type_node, arg,
9368	build_int_cst (unsigned_type_node, target_digit0));
9369	return fold_build2_loc (loc, LE_EXPR, integer_type_node, arg,
9370	build_int_cst (unsigned_type_node, 9));
9371	}
9372	}
9373
9374	/* Fold a call to fabs, fabsf or fabsl with argument ARG. */
9375
9376	static tree
9377	fold_builtin_fabs (location_t loc, tree arg, tree type)
9378	{
9379	if (!validate_arg (arg, code: REAL_TYPE))
9380	return NULL_TREE;
9381
9382	arg = fold_convert_loc (loc, type, arg);
9383	return fold_build1_loc (loc, ABS_EXPR, type, arg);
9384	}
9385
9386	/* Fold a call to abs, labs, llabs or imaxabs with argument ARG. */
9387
9388	static tree
9389	fold_builtin_abs (location_t loc, tree arg, tree type)
9390	{
9391	if (!validate_arg (arg, code: INTEGER_TYPE))
9392	return NULL_TREE;
9393
9394	arg = fold_convert_loc (loc, type, arg);
9395	return fold_build1_loc (loc, ABS_EXPR, type, arg);
9396	}
9397
9398	/* Fold a call to builtin carg(a+bi) -> atan2(b,a). */
9399
9400	static tree
9401	fold_builtin_carg (location_t loc, tree arg, tree type)
9402	{
9403	if (validate_arg (arg, code: COMPLEX_TYPE)
9404	&& SCALAR_FLOAT_TYPE_P (TREE_TYPE (TREE_TYPE (arg))))
9405	{
9406	tree atan2_fn = mathfn_built_in (type, fn: BUILT_IN_ATAN2);
9407
9408	if (atan2_fn)
9409	{
9410	tree new_arg = builtin_save_expr (exp: arg);
9411	tree r_arg = fold_build1_loc (loc, REALPART_EXPR, type, new_arg);
9412	tree i_arg = fold_build1_loc (loc, IMAGPART_EXPR, type, new_arg);
9413	return build_call_expr_loc (loc, atan2_fn, 2, i_arg, r_arg);
9414	}
9415	}
9416
9417	return NULL_TREE;
9418	}
9419
9420	/* Fold a call to builtin frexp, we can assume the base is 2. */
9421
9422	static tree
9423	fold_builtin_frexp (location_t loc, tree arg0, tree arg1, tree rettype)
9424	{
9425	if (! validate_arg (arg0, code: REAL_TYPE) || ! validate_arg (arg1, code: POINTER_TYPE))
9426	return NULL_TREE;
9427
9428	STRIP_NOPS (arg0);
9429
9430	if (!(TREE_CODE (arg0) == REAL_CST && ! TREE_OVERFLOW (arg0)))
9431	return NULL_TREE;
9432
9433	arg1 = build_fold_indirect_ref_loc (loc, arg1);
9434
9435	/* Proceed if a valid pointer type was passed in. */
9436	if (TYPE_MAIN_VARIANT (TREE_TYPE (arg1)) == integer_type_node)
9437	{
9438	const REAL_VALUE_TYPE *const value = TREE_REAL_CST_PTR (arg0);
9439	tree frac, exp, res;
9440
9441	switch (value->cl)
9442	{
9443	case rvc_zero:
9444	/* For +-0, return (*exp = 0, +-0). */
9445	exp = integer_zero_node;
9446	frac = arg0;
9447	break;
9448	case rvc_nan:
9449	case rvc_inf:
9450	/* For +-NaN or +-Inf, *exp is unspecified, return arg0. */
9451	return omit_one_operand_loc (loc, rettype, arg0, arg1);
9452	case rvc_normal:
9453	{
9454	/* Since the frexp function always expects base 2, and in
9455	GCC normalized significands are already in the range
9456	[0.5, 1.0), we have exactly what frexp wants. */
9457	REAL_VALUE_TYPE frac_rvt = *value;
9458	SET_REAL_EXP (&frac_rvt, 0);
9459	frac = build_real (rettype, frac_rvt);
9460	exp = build_int_cst (integer_type_node, REAL_EXP (value));
9461	}
9462	break;
9463	default:
9464	gcc_unreachable ();
9465	}
9466
9467	/* Create the COMPOUND_EXPR (*arg1 = trunc, frac). */
9468	arg1 = fold_build2_loc (loc, MODIFY_EXPR, rettype, arg1, exp);
9469	TREE_SIDE_EFFECTS (arg1) = 1;
9470	res = fold_build2_loc (loc, COMPOUND_EXPR, rettype, arg1, frac);
9471	suppress_warning (res, OPT_Wunused_value);
9472	return res;
9473	}
9474
9475	return NULL_TREE;
9476	}
9477
9478	/* Fold a call to builtin modf. */
9479
9480	static tree
9481	fold_builtin_modf (location_t loc, tree arg0, tree arg1, tree rettype)
9482	{
9483	if (! validate_arg (arg0, code: REAL_TYPE) || ! validate_arg (arg1, code: POINTER_TYPE))
9484	return NULL_TREE;
9485
9486	STRIP_NOPS (arg0);
9487
9488	if (!(TREE_CODE (arg0) == REAL_CST && ! TREE_OVERFLOW (arg0)))
9489	return NULL_TREE;
9490
9491	arg1 = build_fold_indirect_ref_loc (loc, arg1);
9492
9493	/* Proceed if a valid pointer type was passed in. */
9494	if (TYPE_MAIN_VARIANT (TREE_TYPE (arg1)) == TYPE_MAIN_VARIANT (rettype))
9495	{
9496	const REAL_VALUE_TYPE *const value = TREE_REAL_CST_PTR (arg0);
9497	REAL_VALUE_TYPE trunc, frac;
9498	tree res;
9499
9500	switch (value->cl)
9501	{
9502	case rvc_nan:
9503	case rvc_zero:
9504	/* For +-NaN or +-0, return (*arg1 = arg0, arg0). */
9505	trunc = frac = *value;
9506	break;
9507	case rvc_inf:
9508	/* For +-Inf, return (*arg1 = arg0, +-0). */
9509	frac = dconst0;
9510	frac.sign = value->sign;
9511	trunc = *value;
9512	break;
9513	case rvc_normal:
9514	/* Return (*arg1 = trunc(arg0), arg0-trunc(arg0)). */
9515	real_trunc (&trunc, VOIDmode, value);
9516	real_arithmetic (&frac, MINUS_EXPR, value, &trunc);
9517	/* If the original number was negative and already
9518	integral, then the fractional part is -0.0. */
9519	if (value->sign && frac.cl == rvc_zero)
9520	frac.sign = value->sign;
9521	break;
9522	}
9523
9524	/* Create the COMPOUND_EXPR (*arg1 = trunc, frac). */
9525	arg1 = fold_build2_loc (loc, MODIFY_EXPR, rettype, arg1,
9526	build_real (rettype, trunc));
9527	TREE_SIDE_EFFECTS (arg1) = 1;
9528	res = fold_build2_loc (loc, COMPOUND_EXPR, rettype, arg1,
9529	build_real (rettype, frac));
9530	suppress_warning (res, OPT_Wunused_value);
9531	return res;
9532	}
9533
9534	return NULL_TREE;
9535	}
9536
9537	/* Given a location LOC, an interclass builtin function decl FNDECL
9538	and its single argument ARG, return an folded expression computing
9539	the same, or NULL_TREE if we either couldn't or didn't want to fold
9540	(the latter happen if there's an RTL instruction available). */
9541
9542	static tree
9543	fold_builtin_interclass_mathfn (location_t loc, tree fndecl, tree arg)
9544	{
9545	machine_mode mode;
9546
9547	if (!validate_arg (arg, code: REAL_TYPE))
9548	return NULL_TREE;
9549
9550	if (interclass_mathfn_icode (arg, fndecl) != CODE_FOR_nothing)
9551	return NULL_TREE;
9552
9553	mode = TYPE_MODE (TREE_TYPE (arg));
9554
9555	bool is_ibm_extended = MODE_COMPOSITE_P (mode);
9556
9557	/* If there is no optab, try generic code. */
9558	switch (DECL_FUNCTION_CODE (decl: fndecl))
9559	{
9560	tree result;
9561
9562	CASE_FLT_FN (BUILT_IN_ISINF):
9563	{
9564	/* isinf(x) -> isgreater(fabs(x),DBL_MAX). */
9565	tree const isgr_fn = builtin_decl_explicit (fncode: BUILT_IN_ISGREATER);
9566	tree type = TREE_TYPE (arg);
9567	REAL_VALUE_TYPE r;
9568	char buf[128];
9569
9570	if (is_ibm_extended)
9571	{
9572	/* NaN and Inf are encoded in the high-order double value
9573	only. The low-order value is not significant. */
9574	type = double_type_node;
9575	mode = DFmode;
9576	arg = fold_build1_loc (loc, NOP_EXPR, type, arg);
9577	}
9578	get_max_float (REAL_MODE_FORMAT (mode), buf, sizeof (buf), false);
9579	real_from_string (&r, buf);
9580	result = build_call_expr (isgr_fn, 2,
9581	fold_build1_loc (loc, ABS_EXPR, type, arg),
9582	build_real (type, r));
9583	return result;
9584	}
9585	CASE_FLT_FN (BUILT_IN_FINITE):
9586	case BUILT_IN_ISFINITE:
9587	{
9588	/* isfinite(x) -> islessequal(fabs(x),DBL_MAX). */
9589	tree const isle_fn = builtin_decl_explicit (fncode: BUILT_IN_ISLESSEQUAL);
9590	tree type = TREE_TYPE (arg);
9591	REAL_VALUE_TYPE r;
9592	char buf[128];
9593
9594	if (is_ibm_extended)
9595	{
9596	/* NaN and Inf are encoded in the high-order double value
9597	only. The low-order value is not significant. */
9598	type = double_type_node;
9599	mode = DFmode;
9600	arg = fold_build1_loc (loc, NOP_EXPR, type, arg);
9601	}
9602	get_max_float (REAL_MODE_FORMAT (mode), buf, sizeof (buf), false);
9603	real_from_string (&r, buf);
9604	result = build_call_expr (isle_fn, 2,
9605	fold_build1_loc (loc, ABS_EXPR, type, arg),
9606	build_real (type, r));
9607	/*result = fold_build2_loc (loc, UNGT_EXPR,
9608	TREE_TYPE (TREE_TYPE (fndecl)),
9609	fold_build1_loc (loc, ABS_EXPR, type, arg),
9610	build_real (type, r));
9611	result = fold_build1_loc (loc, TRUTH_NOT_EXPR,
9612	TREE_TYPE (TREE_TYPE (fndecl)),
9613	result);*/
9614	return result;
9615	}
9616	case BUILT_IN_ISNORMAL:
9617	{
9618	/* isnormal(x) -> isgreaterequal(fabs(x),DBL_MIN) &
9619	islessequal(fabs(x),DBL_MAX). */
9620	tree const isle_fn = builtin_decl_explicit (fncode: BUILT_IN_ISLESSEQUAL);
9621	tree type = TREE_TYPE (arg);
9622	tree orig_arg, max_exp, min_exp;
9623	machine_mode orig_mode = mode;
9624	REAL_VALUE_TYPE rmax, rmin;
9625	char buf[128];
9626
9627	orig_arg = arg = builtin_save_expr (exp: arg);
9628	if (is_ibm_extended)
9629	{
9630	/* Use double to test the normal range of IBM extended
9631	precision. Emin for IBM extended precision is
9632	different to emin for IEEE double, being 53 higher
9633	since the low double exponent is at least 53 lower
9634	than the high double exponent. */
9635	type = double_type_node;
9636	mode = DFmode;
9637	arg = fold_build1_loc (loc, NOP_EXPR, type, arg);
9638	}
9639	arg = fold_build1_loc (loc, ABS_EXPR, type, arg);
9640
9641	get_max_float (REAL_MODE_FORMAT (mode), buf, sizeof (buf), false);
9642	real_from_string (&rmax, buf);
9643	sprintf (s: buf, format: "0x1p%d", REAL_MODE_FORMAT (orig_mode)->emin - 1);
9644	real_from_string (&rmin, buf);
9645	max_exp = build_real (type, rmax);
9646	min_exp = build_real (type, rmin);
9647
9648	max_exp = build_call_expr (isle_fn, 2, arg, max_exp);
9649	if (is_ibm_extended)
9650	{
9651	/* Testing the high end of the range is done just using
9652	the high double, using the same test as isfinite().
9653	For the subnormal end of the range we first test the
9654	high double, then if its magnitude is equal to the
9655	limit of 0x1p-969, we test whether the low double is
9656	non-zero and opposite sign to the high double. */
9657	tree const islt_fn = builtin_decl_explicit (fncode: BUILT_IN_ISLESS);
9658	tree const isgt_fn = builtin_decl_explicit (fncode: BUILT_IN_ISGREATER);
9659	tree gt_min = build_call_expr (isgt_fn, 2, arg, min_exp);
9660	tree eq_min = fold_build2 (EQ_EXPR, integer_type_node,
9661	arg, min_exp);
9662	tree as_complex = build1 (VIEW_CONVERT_EXPR,
9663	complex_double_type_node, orig_arg);
9664	tree hi_dbl = build1 (REALPART_EXPR, type, as_complex);
9665	tree lo_dbl = build1 (IMAGPART_EXPR, type, as_complex);
9666	tree zero = build_real (type, dconst0);
9667	tree hilt = build_call_expr (islt_fn, 2, hi_dbl, zero);
9668	tree lolt = build_call_expr (islt_fn, 2, lo_dbl, zero);
9669	tree logt = build_call_expr (isgt_fn, 2, lo_dbl, zero);
9670	tree ok_lo = fold_build1 (TRUTH_NOT_EXPR, integer_type_node,
9671	fold_build3 (COND_EXPR,
9672	integer_type_node,
9673	hilt, logt, lolt));
9674	eq_min = fold_build2 (TRUTH_ANDIF_EXPR, integer_type_node,
9675	eq_min, ok_lo);
9676	min_exp = fold_build2 (TRUTH_ORIF_EXPR, integer_type_node,
9677	gt_min, eq_min);
9678	}
9679	else
9680	{
9681	tree const isge_fn
9682	= builtin_decl_explicit (fncode: BUILT_IN_ISGREATEREQUAL);
9683	min_exp = build_call_expr (isge_fn, 2, arg, min_exp);
9684	}
9685	result = fold_build2 (BIT_AND_EXPR, integer_type_node,
9686	max_exp, min_exp);
9687	return result;
9688	}
9689	default:
9690	break;
9691	}
9692
9693	return NULL_TREE;
9694	}
9695
9696	/* Fold a call to __builtin_isnan(), __builtin_isinf, __builtin_finite.
9697	ARG is the argument for the call. */
9698
9699	static tree
9700	fold_builtin_classify (location_t loc, tree fndecl, tree arg, int builtin_index)
9701	{
9702	tree type = TREE_TYPE (TREE_TYPE (fndecl));
9703
9704	if (!validate_arg (arg, code: REAL_TYPE))
9705	return NULL_TREE;
9706
9707	switch (builtin_index)
9708	{
9709	case BUILT_IN_ISINF:
9710	if (tree_expr_infinite_p (arg))
9711	return omit_one_operand_loc (loc, type, integer_one_node, arg);
9712	if (!tree_expr_maybe_infinite_p (arg))
9713	return omit_one_operand_loc (loc, type, integer_zero_node, arg);
9714	return NULL_TREE;
9715
9716	case BUILT_IN_ISINF_SIGN:
9717	{
9718	/* isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 */
9719	/* In a boolean context, GCC will fold the inner COND_EXPR to
9720	1. So e.g. "if (isinf_sign(x))" would be folded to just
9721	"if (isinf(x) ? 1 : 0)" which becomes "if (isinf(x))". */
9722	tree signbit_fn = builtin_decl_explicit (fncode: BUILT_IN_SIGNBIT);
9723	tree isinf_fn = builtin_decl_explicit (fncode: BUILT_IN_ISINF);
9724	tree tmp = NULL_TREE;
9725
9726	arg = builtin_save_expr (exp: arg);
9727
9728	if (signbit_fn && isinf_fn)
9729	{
9730	tree signbit_call = build_call_expr_loc (loc, signbit_fn, 1, arg);
9731	tree isinf_call = build_call_expr_loc (loc, isinf_fn, 1, arg);
9732
9733	signbit_call = fold_build2_loc (loc, NE_EXPR, integer_type_node,
9734	signbit_call, integer_zero_node);
9735	isinf_call = fold_build2_loc (loc, NE_EXPR, integer_type_node,
9736	isinf_call, integer_zero_node);
9737
9738	tmp = fold_build3_loc (loc, COND_EXPR, integer_type_node, signbit_call,
9739	integer_minus_one_node, integer_one_node);
9740	tmp = fold_build3_loc (loc, COND_EXPR, integer_type_node,
9741	isinf_call, tmp,
9742	integer_zero_node);
9743	}
9744
9745	return tmp;
9746	}
9747
9748	case BUILT_IN_ISFINITE:
9749	if (tree_expr_finite_p (arg))
9750	return omit_one_operand_loc (loc, type, integer_one_node, arg);
9751	if (tree_expr_nan_p (arg) || tree_expr_infinite_p (arg))
9752	return omit_one_operand_loc (loc, type, integer_zero_node, arg);
9753	return NULL_TREE;
9754
9755	case BUILT_IN_ISNAN:
9756	if (tree_expr_nan_p (arg))
9757	return omit_one_operand_loc (loc, type, integer_one_node, arg);
9758	if (!tree_expr_maybe_nan_p (arg))
9759	return omit_one_operand_loc (loc, type, integer_zero_node, arg);
9760
9761	{
9762	bool is_ibm_extended = MODE_COMPOSITE_P (TYPE_MODE (TREE_TYPE (arg)));
9763	if (is_ibm_extended)
9764	{
9765	/* NaN and Inf are encoded in the high-order double value
9766	only. The low-order value is not significant. */
9767	arg = fold_build1_loc (loc, NOP_EXPR, double_type_node, arg);
9768	}
9769	}
9770	arg = builtin_save_expr (exp: arg);
9771	return fold_build2_loc (loc, UNORDERED_EXPR, type, arg, arg);
9772
9773	case BUILT_IN_ISSIGNALING:
9774	/* Folding to true for REAL_CST is done in fold_const_call_ss.
9775	Don't use tree_expr_signaling_nan_p (arg) -> integer_one_node
9776	and !tree_expr_maybe_signaling_nan_p (arg) -> integer_zero_node
9777	here, so there is some possibility of __builtin_issignaling working
9778	without -fsignaling-nans. Especially when -fno-signaling-nans is
9779	the default. */
9780	if (!tree_expr_maybe_nan_p (arg))
9781	return omit_one_operand_loc (loc, type, integer_zero_node, arg);
9782	return NULL_TREE;
9783
9784	default:
9785	gcc_unreachable ();
9786	}
9787	}
9788
9789	/* Fold a call to __builtin_fpclassify(int, int, int, int, int, ...).
9790	This builtin will generate code to return the appropriate floating
9791	point classification depending on the value of the floating point
9792	number passed in. The possible return values must be supplied as
9793	int arguments to the call in the following order: FP_NAN, FP_INFINITE,
9794	FP_NORMAL, FP_SUBNORMAL and FP_ZERO. The ellipses is for exactly
9795	one floating point argument which is "type generic". */
9796
9797	static tree
9798	fold_builtin_fpclassify (location_t loc, tree *args, int nargs)
9799	{
9800	tree fp_nan, fp_infinite, fp_normal, fp_subnormal, fp_zero,
9801	arg, type, res, tmp;
9802	machine_mode mode;
9803	REAL_VALUE_TYPE r;
9804	char buf[128];
9805
9806	/* Verify the required arguments in the original call. */
9807	if (nargs != 6
9808	|| !validate_arg (args[0], code: INTEGER_TYPE)
9809	|| !validate_arg (args[1], code: INTEGER_TYPE)
9810	|| !validate_arg (args[2], code: INTEGER_TYPE)
9811	|| !validate_arg (args[3], code: INTEGER_TYPE)
9812	|| !validate_arg (args[4], code: INTEGER_TYPE)
9813	|| !validate_arg (args[5], code: REAL_TYPE))
9814	return NULL_TREE;
9815
9816	fp_nan = args[0];
9817	fp_infinite = args[1];
9818	fp_normal = args[2];
9819	fp_subnormal = args[3];
9820	fp_zero = args[4];
9821	arg = args[5];
9822	type = TREE_TYPE (arg);
9823	mode = TYPE_MODE (type);
9824	arg = builtin_save_expr (exp: fold_build1_loc (loc, ABS_EXPR, type, arg));
9825
9826	/* fpclassify(x) ->
9827	isnan(x) ? FP_NAN :
9828	(fabs(x) == Inf ? FP_INFINITE :
9829	(fabs(x) >= DBL_MIN ? FP_NORMAL :
9830	(x == 0 ? FP_ZERO : FP_SUBNORMAL))). */
9831
9832	tmp = fold_build2_loc (loc, EQ_EXPR, integer_type_node, arg,
9833	build_real (type, dconst0));
9834	res = fold_build3_loc (loc, COND_EXPR, integer_type_node,
9835	tmp, fp_zero, fp_subnormal);
9836
9837	sprintf (s: buf, format: "0x1p%d", REAL_MODE_FORMAT (mode)->emin - 1);
9838	real_from_string (&r, buf);
9839	tmp = fold_build2_loc (loc, GE_EXPR, integer_type_node,
9840	arg, build_real (type, r));
9841	res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp, fp_normal, res);
9842
9843	if (tree_expr_maybe_infinite_p (arg))
9844	{
9845	tmp = fold_build2_loc (loc, EQ_EXPR, integer_type_node, arg,
9846	build_real (type, dconstinf));
9847	res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
9848	fp_infinite, res);
9849	}
9850
9851	if (tree_expr_maybe_nan_p (arg))
9852	{
9853	tmp = fold_build2_loc (loc, ORDERED_EXPR, integer_type_node, arg, arg);
9854	res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp, res, fp_nan);
9855	}
9856
9857	return res;
9858	}
9859
9860	/* Fold a call to an unordered comparison function such as
9861	__builtin_isgreater(). FNDECL is the FUNCTION_DECL for the function
9862	being called and ARG0 and ARG1 are the arguments for the call.
9863	UNORDERED_CODE and ORDERED_CODE are comparison codes that give
9864	the opposite of the desired result. UNORDERED_CODE is used
9865	for modes that can hold NaNs and ORDERED_CODE is used for
9866	the rest. */
9867
9868	static tree
9869	fold_builtin_unordered_cmp (location_t loc, tree fndecl, tree arg0, tree arg1,
9870	enum tree_code unordered_code,
9871	enum tree_code ordered_code)
9872	{
9873	tree type = TREE_TYPE (TREE_TYPE (fndecl));
9874	enum tree_code code;
9875	tree type0, type1;
9876	enum tree_code code0, code1;
9877	tree cmp_type = NULL_TREE;
9878
9879	type0 = TREE_TYPE (arg0);
9880	type1 = TREE_TYPE (arg1);
9881
9882	code0 = TREE_CODE (type0);
9883	code1 = TREE_CODE (type1);
9884
9885	if (code0 == REAL_TYPE && code1 == REAL_TYPE)
9886	/* Choose the wider of two real types. */
9887	cmp_type = TYPE_PRECISION (type0) >= TYPE_PRECISION (type1)
9888	? type0 : type1;
9889	else if (code0 == REAL_TYPE
9890	&& (code1 == INTEGER_TYPE || code1 == BITINT_TYPE))
9891	cmp_type = type0;
9892	else if ((code0 == INTEGER_TYPE || code0 == BITINT_TYPE)
9893	&& code1 == REAL_TYPE)
9894	cmp_type = type1;
9895
9896	arg0 = fold_convert_loc (loc, cmp_type, arg0);
9897	arg1 = fold_convert_loc (loc, cmp_type, arg1);
9898
9899	if (unordered_code == UNORDERED_EXPR)
9900	{
9901	if (tree_expr_nan_p (arg0) || tree_expr_nan_p (arg1))
9902	return omit_two_operands_loc (loc, type, integer_one_node, arg0, arg1);
9903	if (!tree_expr_maybe_nan_p (arg0) && !tree_expr_maybe_nan_p (arg1))
9904	return omit_two_operands_loc (loc, type, integer_zero_node, arg0, arg1);
9905	return fold_build2_loc (loc, UNORDERED_EXPR, type, arg0, arg1);
9906	}
9907
9908	code = (tree_expr_maybe_nan_p (arg0) || tree_expr_maybe_nan_p (arg1))
9909	? unordered_code : ordered_code;
9910	return fold_build1_loc (loc, TRUTH_NOT_EXPR, type,
9911	fold_build2_loc (loc, code, type, arg0, arg1));
9912	}
9913
9914	/* Fold a call to __builtin_iseqsig(). ARG0 and ARG1 are the arguments.
9915	After choosing the wider floating-point type for the comparison,
9916	the code is folded to:
9917	SAVE_EXPR<ARG0> >= SAVE_EXPR<ARG1> && SAVE_EXPR<ARG0> <= SAVE_EXPR<ARG1> */
9918
9919	static tree
9920	fold_builtin_iseqsig (location_t loc, tree arg0, tree arg1)
9921	{
9922	tree type0, type1;
9923	enum tree_code code0, code1;
9924	tree cmp1, cmp2, cmp_type = NULL_TREE;
9925
9926	type0 = TREE_TYPE (arg0);
9927	type1 = TREE_TYPE (arg1);
9928
9929	code0 = TREE_CODE (type0);
9930	code1 = TREE_CODE (type1);
9931
9932	if (code0 == REAL_TYPE && code1 == REAL_TYPE)
9933	/* Choose the wider of two real types. */
9934	cmp_type = TYPE_PRECISION (type0) >= TYPE_PRECISION (type1)
9935	? type0 : type1;
9936	else if (code0 == REAL_TYPE && code1 == INTEGER_TYPE)
9937	cmp_type = type0;
9938	else if (code0 == INTEGER_TYPE && code1 == REAL_TYPE)
9939	cmp_type = type1;
9940
9941	arg0 = builtin_save_expr (exp: fold_convert_loc (loc, cmp_type, arg0));
9942	arg1 = builtin_save_expr (exp: fold_convert_loc (loc, cmp_type, arg1));
9943
9944	cmp1 = fold_build2_loc (loc, GE_EXPR, integer_type_node, arg0, arg1);
9945	cmp2 = fold_build2_loc (loc, LE_EXPR, integer_type_node, arg0, arg1);
9946
9947	return fold_build2_loc (loc, TRUTH_AND_EXPR, integer_type_node, cmp1, cmp2);
9948	}
9949
9950	/* Fold __builtin_{,s,u}{add,sub,mul}{,l,ll}_overflow, either into normal
9951	arithmetics if it can never overflow, or into internal functions that
9952	return both result of arithmetics and overflowed boolean flag in
9953	a complex integer result, or some other check for overflow.
9954	Similarly fold __builtin_{add,sub,mul}_overflow_p to just the overflow
9955	checking part of that. */
9956
9957	static tree
9958	fold_builtin_arith_overflow (location_t loc, enum built_in_function fcode,
9959	tree arg0, tree arg1, tree arg2)
9960	{
9961	enum internal_fn ifn = IFN_LAST;
9962	/* The code of the expression corresponding to the built-in. */
9963	enum tree_code opcode = ERROR_MARK;
9964	bool ovf_only = false;
9965
9966	switch (fcode)
9967	{
9968	case BUILT_IN_ADD_OVERFLOW_P:
9969	ovf_only = true;
9970	/* FALLTHRU */
9971	case BUILT_IN_ADD_OVERFLOW:
9972	case BUILT_IN_SADD_OVERFLOW:
9973	case BUILT_IN_SADDL_OVERFLOW:
9974	case BUILT_IN_SADDLL_OVERFLOW:
9975	case BUILT_IN_UADD_OVERFLOW:
9976	case BUILT_IN_UADDL_OVERFLOW:
9977	case BUILT_IN_UADDLL_OVERFLOW:
9978	opcode = PLUS_EXPR;
9979	ifn = IFN_ADD_OVERFLOW;
9980	break;
9981	case BUILT_IN_SUB_OVERFLOW_P:
9982	ovf_only = true;
9983	/* FALLTHRU */
9984	case BUILT_IN_SUB_OVERFLOW:
9985	case BUILT_IN_SSUB_OVERFLOW:
9986	case BUILT_IN_SSUBL_OVERFLOW:
9987	case BUILT_IN_SSUBLL_OVERFLOW:
9988	case BUILT_IN_USUB_OVERFLOW:
9989	case BUILT_IN_USUBL_OVERFLOW:
9990	case BUILT_IN_USUBLL_OVERFLOW:
9991	opcode = MINUS_EXPR;
9992	ifn = IFN_SUB_OVERFLOW;
9993	break;
9994	case BUILT_IN_MUL_OVERFLOW_P:
9995	ovf_only = true;
9996	/* FALLTHRU */
9997	case BUILT_IN_MUL_OVERFLOW:
9998	case BUILT_IN_SMUL_OVERFLOW:
9999	case BUILT_IN_SMULL_OVERFLOW:
10000	case BUILT_IN_SMULLL_OVERFLOW:
10001	case BUILT_IN_UMUL_OVERFLOW:
10002	case BUILT_IN_UMULL_OVERFLOW:
10003	case BUILT_IN_UMULLL_OVERFLOW:
10004	opcode = MULT_EXPR;
10005	ifn = IFN_MUL_OVERFLOW;
10006	break;
10007	default:
10008	gcc_unreachable ();
10009	}
10010
10011	/* For the "generic" overloads, the first two arguments can have different
10012	types and the last argument determines the target type to use to check
10013	for overflow. The arguments of the other overloads all have the same
10014	type. */
10015	tree type = ovf_only ? TREE_TYPE (arg2) : TREE_TYPE (TREE_TYPE (arg2));
10016
10017	/* For the __builtin_{add,sub,mul}_overflow_p builtins, when the first two
10018	arguments are constant, attempt to fold the built-in call into a constant
10019	expression indicating whether or not it detected an overflow. */
10020	if (ovf_only
10021	&& TREE_CODE (arg0) == INTEGER_CST
10022	&& TREE_CODE (arg1) == INTEGER_CST)
10023	/* Perform the computation in the target type and check for overflow. */
10024	return omit_one_operand_loc (loc, boolean_type_node,
10025	arith_overflowed_p (opcode, type, arg0, arg1)
10026	? boolean_true_node : boolean_false_node,
10027	arg2);
10028
10029	tree intres, ovfres;
10030	if (TREE_CODE (arg0) == INTEGER_CST && TREE_CODE (arg1) == INTEGER_CST)
10031	{
10032	intres = fold_binary_loc (loc, opcode, type,
10033	fold_convert_loc (loc, type, arg0),
10034	fold_convert_loc (loc, type, arg1));
10035	if (TREE_OVERFLOW (intres))
10036	intres = drop_tree_overflow (intres);
10037	ovfres = (arith_overflowed_p (opcode, type, arg0, arg1)
10038	? boolean_true_node : boolean_false_node);
10039	}
10040	else
10041	{
10042	tree ctype = build_complex_type (type);
10043	tree call = build_call_expr_internal_loc (loc, ifn, ctype, 2,
10044	arg0, arg1);
10045	tree tgt = save_expr (call);
10046	intres = build1_loc (loc, code: REALPART_EXPR, type, arg1: tgt);
10047	ovfres = build1_loc (loc, code: IMAGPART_EXPR, type, arg1: tgt);
10048	ovfres = fold_convert_loc (loc, boolean_type_node, ovfres);
10049	}
10050
10051	if (ovf_only)
10052	return omit_one_operand_loc (loc, boolean_type_node, ovfres, arg2);
10053
10054	tree mem_arg2 = build_fold_indirect_ref_loc (loc, arg2);
10055	tree store
10056	= fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg2, intres);
10057	return build2_loc (loc, code: COMPOUND_EXPR, boolean_type_node, arg0: store, arg1: ovfres);
10058	}
10059
10060	/* Fold __builtin_{clz,ctz,clrsb,ffs,parity,popcount}g into corresponding
10061	internal function. */
10062
10063	static tree
10064	fold_builtin_bit_query (location_t loc, enum built_in_function fcode,
10065	tree arg0, tree arg1)
10066	{
10067	enum internal_fn ifn;
10068	enum built_in_function fcodei, fcodel, fcodell;
10069	tree arg0_type = TREE_TYPE (arg0);
10070	tree cast_type = NULL_TREE;
10071	int addend = 0;
10072
10073	switch (fcode)
10074	{
10075	case BUILT_IN_CLZG:
10076	if (arg1 && TREE_CODE (arg1) != INTEGER_CST)
10077	return NULL_TREE;
10078	ifn = IFN_CLZ;
10079	fcodei = BUILT_IN_CLZ;
10080	fcodel = BUILT_IN_CLZL;
10081	fcodell = BUILT_IN_CLZLL;
10082	break;
10083	case BUILT_IN_CTZG:
10084	if (arg1 && TREE_CODE (arg1) != INTEGER_CST)
10085	return NULL_TREE;
10086	ifn = IFN_CTZ;
10087	fcodei = BUILT_IN_CTZ;
10088	fcodel = BUILT_IN_CTZL;
10089	fcodell = BUILT_IN_CTZLL;
10090	break;
10091	case BUILT_IN_CLRSBG:
10092	ifn = IFN_CLRSB;
10093	fcodei = BUILT_IN_CLRSB;
10094	fcodel = BUILT_IN_CLRSBL;
10095	fcodell = BUILT_IN_CLRSBLL;
10096	break;
10097	case BUILT_IN_FFSG:
10098	ifn = IFN_FFS;
10099	fcodei = BUILT_IN_FFS;
10100	fcodel = BUILT_IN_FFSL;
10101	fcodell = BUILT_IN_FFSLL;
10102	break;
10103	case BUILT_IN_PARITYG:
10104	ifn = IFN_PARITY;
10105	fcodei = BUILT_IN_PARITY;
10106	fcodel = BUILT_IN_PARITYL;
10107	fcodell = BUILT_IN_PARITYLL;
10108	break;
10109	case BUILT_IN_POPCOUNTG:
10110	ifn = IFN_POPCOUNT;
10111	fcodei = BUILT_IN_POPCOUNT;
10112	fcodel = BUILT_IN_POPCOUNTL;
10113	fcodell = BUILT_IN_POPCOUNTLL;
10114	break;
10115	default:
10116	gcc_unreachable ();
10117	}
10118
10119	if (TYPE_PRECISION (arg0_type)
10120	<= TYPE_PRECISION (long_long_unsigned_type_node))
10121	{
10122	if (TYPE_PRECISION (arg0_type) <= TYPE_PRECISION (unsigned_type_node))
10123
10124	cast_type = (TYPE_UNSIGNED (arg0_type)
10125	? unsigned_type_node : integer_type_node);
10126	else if (TYPE_PRECISION (arg0_type)
10127	<= TYPE_PRECISION (long_unsigned_type_node))
10128	{
10129	cast_type = (TYPE_UNSIGNED (arg0_type)
10130	? long_unsigned_type_node : long_integer_type_node);
10131	fcodei = fcodel;
10132	}
10133	else
10134	{
10135	cast_type = (TYPE_UNSIGNED (arg0_type)
10136	? long_long_unsigned_type_node
10137	: long_long_integer_type_node);
10138	fcodei = fcodell;
10139	}
10140	}
10141	else if (TYPE_PRECISION (arg0_type) <= MAX_FIXED_MODE_SIZE)
10142	{
10143	cast_type
10144	= build_nonstandard_integer_type (MAX_FIXED_MODE_SIZE,
10145	TYPE_UNSIGNED (arg0_type));
10146	gcc_assert (TYPE_PRECISION (cast_type)
10147	== 2 * TYPE_PRECISION (long_long_unsigned_type_node));
10148	fcodei = END_BUILTINS;
10149	}
10150	else
10151	fcodei = END_BUILTINS;
10152	if (cast_type)
10153	{
10154	switch (fcode)
10155	{
10156	case BUILT_IN_CLZG:
10157	case BUILT_IN_CLRSBG:
10158	addend = TYPE_PRECISION (arg0_type) - TYPE_PRECISION (cast_type);
10159	break;
10160	default:
10161	break;
10162	}
10163	arg0 = fold_convert (cast_type, arg0);
10164	arg0_type = cast_type;
10165	}
10166
10167	if (arg1)
10168	arg1 = fold_convert (integer_type_node, arg1);
10169
10170	tree arg2 = arg1;
10171	if (fcode == BUILT_IN_CLZG && addend)
10172	{
10173	if (arg1)
10174	arg0 = save_expr (arg0);
10175	arg2 = NULL_TREE;
10176	}
10177	tree call = NULL_TREE, tem;
10178	if (TYPE_PRECISION (arg0_type) == MAX_FIXED_MODE_SIZE
10179	&& (TYPE_PRECISION (arg0_type)
10180	== 2 * TYPE_PRECISION (long_long_unsigned_type_node)))
10181	{
10182	/* __int128 expansions using up to 2 long long builtins. */
10183	arg0 = save_expr (arg0);
10184	tree type = (TYPE_UNSIGNED (arg0_type)
10185	? long_long_unsigned_type_node
10186	: long_long_integer_type_node);
10187	tree hi = fold_build2 (RSHIFT_EXPR, arg0_type, arg0,
10188	build_int_cst (integer_type_node,
10189	MAX_FIXED_MODE_SIZE / 2));
10190	hi = fold_convert (type, hi);
10191	tree lo = fold_convert (type, arg0);
10192	switch (fcode)
10193	{
10194	case BUILT_IN_CLZG:
10195	call = fold_builtin_bit_query (loc, fcode, arg0: lo, NULL_TREE);
10196	call = fold_build2 (PLUS_EXPR, integer_type_node, call,
10197	build_int_cst (integer_type_node,
10198	MAX_FIXED_MODE_SIZE / 2));
10199	if (arg2)
10200	call = fold_build3 (COND_EXPR, integer_type_node,
10201	fold_build2 (NE_EXPR, boolean_type_node,
10202	lo, build_zero_cst (type)),
10203	call, arg2);
10204	call = fold_build3 (COND_EXPR, integer_type_node,
10205	fold_build2 (NE_EXPR, boolean_type_node,
10206	hi, build_zero_cst (type)),
10207	fold_builtin_bit_query (loc, fcode, hi,
10208	NULL_TREE),
10209	call);
10210	break;
10211	case BUILT_IN_CTZG:
10212	call = fold_builtin_bit_query (loc, fcode, arg0: hi, NULL_TREE);
10213	call = fold_build2 (PLUS_EXPR, integer_type_node, call,
10214	build_int_cst (integer_type_node,
10215	MAX_FIXED_MODE_SIZE / 2));
10216	if (arg2)
10217	call = fold_build3 (COND_EXPR, integer_type_node,
10218	fold_build2 (NE_EXPR, boolean_type_node,
10219	hi, build_zero_cst (type)),
10220	call, arg2);
10221	call = fold_build3 (COND_EXPR, integer_type_node,
10222	fold_build2 (NE_EXPR, boolean_type_node,
10223	lo, build_zero_cst (type)),
10224	fold_builtin_bit_query (loc, fcode, lo,
10225	NULL_TREE),
10226	call);
10227	break;
10228	case BUILT_IN_CLRSBG:
10229	tem = fold_builtin_bit_query (loc, fcode, arg0: lo, NULL_TREE);
10230	tem = fold_build2 (PLUS_EXPR, integer_type_node, tem,
10231	build_int_cst (integer_type_node,
10232	MAX_FIXED_MODE_SIZE / 2));
10233	tem = fold_build3 (COND_EXPR, integer_type_node,
10234	fold_build2 (LT_EXPR, boolean_type_node,
10235	fold_build2 (BIT_XOR_EXPR, type,
10236	lo, hi),
10237	build_zero_cst (type)),
10238	build_int_cst (integer_type_node,
10239	MAX_FIXED_MODE_SIZE / 2 - 1),
10240	tem);
10241	call = fold_builtin_bit_query (loc, fcode, arg0: hi, NULL_TREE);
10242	call = save_expr (call);
10243	call = fold_build3 (COND_EXPR, integer_type_node,
10244	fold_build2 (NE_EXPR, boolean_type_node,
10245	call,
10246	build_int_cst (integer_type_node,
10247	MAX_FIXED_MODE_SIZE
10248	/ 2 - 1)),
10249	call, tem);
10250	break;
10251	case BUILT_IN_FFSG:
10252	call = fold_builtin_bit_query (loc, fcode, arg0: hi, NULL_TREE);
10253	call = fold_build2 (PLUS_EXPR, integer_type_node, call,
10254	build_int_cst (integer_type_node,
10255	MAX_FIXED_MODE_SIZE / 2));
10256	call = fold_build3 (COND_EXPR, integer_type_node,
10257	fold_build2 (NE_EXPR, boolean_type_node,
10258	hi, build_zero_cst (type)),
10259	call, integer_zero_node);
10260	call = fold_build3 (COND_EXPR, integer_type_node,
10261	fold_build2 (NE_EXPR, boolean_type_node,
10262	lo, build_zero_cst (type)),
10263	fold_builtin_bit_query (loc, fcode, lo,
10264	NULL_TREE),
10265	call);
10266	break;
10267	case BUILT_IN_PARITYG:
10268	call = fold_builtin_bit_query (loc, fcode,
10269	fold_build2 (BIT_XOR_EXPR, type,
10270	lo, hi), NULL_TREE);
10271	break;
10272	case BUILT_IN_POPCOUNTG:
10273	call = fold_build2 (PLUS_EXPR, integer_type_node,
10274	fold_builtin_bit_query (loc, fcode, hi,
10275	NULL_TREE),
10276	fold_builtin_bit_query (loc, fcode, lo,
10277	NULL_TREE));
10278	break;
10279	default:
10280	gcc_unreachable ();
10281	}
10282	}
10283	else
10284	{
10285	/* Only keep second argument to IFN_CLZ/IFN_CTZ if it is the
10286	value defined at zero during GIMPLE, or for large/huge _BitInt
10287	(which are then lowered during bitint lowering). */
10288	if (arg2 && TREE_CODE (TREE_TYPE (arg0)) != BITINT_TYPE)
10289	{
10290	int val;
10291	if (fcode == BUILT_IN_CLZG)
10292	{
10293	if (CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_TYPE_MODE (arg0_type),
10294	val) != 2
10295	|| wi::to_widest (t: arg2) != val)
10296	arg2 = NULL_TREE;
10297	}
10298	else if (CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_TYPE_MODE (arg0_type),
10299	val) != 2
10300	|| wi::to_widest (t: arg2) != val)
10301	arg2 = NULL_TREE;
10302	if (!direct_internal_fn_supported_p (ifn, arg0_type,
10303	OPTIMIZE_FOR_BOTH))
10304	arg2 = NULL_TREE;
10305	if (arg2 == NULL_TREE)
10306	arg0 = save_expr (arg0);
10307	}
10308	if (fcodei == END_BUILTINS || arg2)
10309	call = build_call_expr_internal_loc (loc, ifn, integer_type_node,
10310	arg2 ? 2 : 1, arg0, arg2);
10311	else
10312	call = build_call_expr_loc (loc, builtin_decl_explicit (fncode: fcodei), 1,
10313	arg0);
10314	}
10315	if (addend)
10316	call = fold_build2 (PLUS_EXPR, integer_type_node, call,
10317	build_int_cst (integer_type_node, addend));
10318	if (arg1 && arg2 == NULL_TREE)
10319	call = fold_build3 (COND_EXPR, integer_type_node,
10320	fold_build2 (NE_EXPR, boolean_type_node,
10321	arg0, build_zero_cst (arg0_type)),
10322	call, arg1);
10323
10324	return call;
10325	}
10326
10327	/* Fold __builtin_{add,sub}c{,l,ll} into pair of internal functions
10328	that return both result of arithmetics and overflowed boolean
10329	flag in a complex integer result. */
10330
10331	static tree
10332	fold_builtin_addc_subc (location_t loc, enum built_in_function fcode,
10333	tree *args)
10334	{
10335	enum internal_fn ifn;
10336
10337	switch (fcode)
10338	{
10339	case BUILT_IN_ADDC:
10340	case BUILT_IN_ADDCL:
10341	case BUILT_IN_ADDCLL:
10342	ifn = IFN_ADD_OVERFLOW;
10343	break;
10344	case BUILT_IN_SUBC:
10345	case BUILT_IN_SUBCL:
10346	case BUILT_IN_SUBCLL:
10347	ifn = IFN_SUB_OVERFLOW;
10348	break;
10349	default:
10350	gcc_unreachable ();
10351	}
10352
10353	tree type = TREE_TYPE (args[0]);
10354	tree ctype = build_complex_type (type);
10355	tree call = build_call_expr_internal_loc (loc, ifn, ctype, 2,
10356	args[0], args[1]);
10357	tree tgt = save_expr (call);
10358	tree intres = build1_loc (loc, code: REALPART_EXPR, type, arg1: tgt);
10359	tree ovfres = build1_loc (loc, code: IMAGPART_EXPR, type, arg1: tgt);
10360	call = build_call_expr_internal_loc (loc, ifn, ctype, 2,
10361	intres, args[2]);
10362	tgt = save_expr (call);
10363	intres = build1_loc (loc, code: REALPART_EXPR, type, arg1: tgt);
10364	tree ovfres2 = build1_loc (loc, code: IMAGPART_EXPR, type, arg1: tgt);
10365	ovfres = build2_loc (loc, code: BIT_IOR_EXPR, type, arg0: ovfres, arg1: ovfres2);
10366	tree mem_arg3 = build_fold_indirect_ref_loc (loc, args[3]);
10367	tree store
10368	= fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg3, ovfres);
10369	return build2_loc (loc, code: COMPOUND_EXPR, type, arg0: store, arg1: intres);
10370	}
10371
10372	/* Fold a call to __builtin_FILE to a constant string. */
10373
10374	static inline tree
10375	fold_builtin_FILE (location_t loc)
10376	{
10377	if (const char *fname = LOCATION_FILE (loc))
10378	{
10379	/* The documentation says this builtin is equivalent to the preprocessor
10380	__FILE__ macro so it appears appropriate to use the same file prefix
10381	mappings. */
10382	fname = remap_macro_filename (fname);
10383	return build_string_literal (p: fname);
10384	}
10385
10386	return build_string_literal (p: "");
10387	}
10388
10389	/* Fold a call to __builtin_FUNCTION to a constant string. */
10390
10391	static inline tree
10392	fold_builtin_FUNCTION ()
10393	{
10394	const char *name = "";
10395
10396	if (current_function_decl)
10397	name = lang_hooks.decl_printable_name (current_function_decl, 0);
10398
10399	return build_string_literal (p: name);
10400	}
10401
10402	/* Fold a call to __builtin_LINE to an integer constant. */
10403
10404	static inline tree
10405	fold_builtin_LINE (location_t loc, tree type)
10406	{
10407	return build_int_cst (type, LOCATION_LINE (loc));
10408	}
10409
10410	/* Fold a call to built-in function FNDECL with 0 arguments.
10411	This function returns NULL_TREE if no simplification was possible. */
10412
10413	static tree
10414	fold_builtin_0 (location_t loc, tree fndecl)
10415	{
10416	tree type = TREE_TYPE (TREE_TYPE (fndecl));
10417	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
10418	switch (fcode)
10419	{
10420	case BUILT_IN_FILE:
10421	return fold_builtin_FILE (loc);
10422
10423	case BUILT_IN_FUNCTION:
10424	return fold_builtin_FUNCTION ();
10425
10426	case BUILT_IN_LINE:
10427	return fold_builtin_LINE (loc, type);
10428
10429	CASE_FLT_FN (BUILT_IN_INF):
10430	CASE_FLT_FN_FLOATN_NX (BUILT_IN_INF):
10431	case BUILT_IN_INFD32:
10432	case BUILT_IN_INFD64:
10433	case BUILT_IN_INFD128:
10434	return fold_builtin_inf (loc, type, warn: true);
10435
10436	CASE_FLT_FN (BUILT_IN_HUGE_VAL):
10437	CASE_FLT_FN_FLOATN_NX (BUILT_IN_HUGE_VAL):
10438	return fold_builtin_inf (loc, type, warn: false);
10439
10440	case BUILT_IN_CLASSIFY_TYPE:
10441	return fold_builtin_classify_type (NULL_TREE);
10442
10443	case BUILT_IN_UNREACHABLE:
10444	/* Rewrite any explicit calls to __builtin_unreachable. */
10445	if (sanitize_flags_p (flag: SANITIZE_UNREACHABLE))
10446	return build_builtin_unreachable (loc);
10447	break;
10448
10449	default:
10450	break;
10451	}
10452	return NULL_TREE;
10453	}
10454
10455	/* Fold a call to built-in function FNDECL with 1 argument, ARG0.
10456	This function returns NULL_TREE if no simplification was possible. */
10457
10458	static tree
10459	fold_builtin_1 (location_t loc, tree expr, tree fndecl, tree arg0)
10460	{
10461	tree type = TREE_TYPE (TREE_TYPE (fndecl));
10462	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
10463
10464	if (error_operand_p (t: arg0))
10465	return NULL_TREE;
10466
10467	if (tree ret = fold_const_call (as_combined_fn (fn: fcode), type, arg0))
10468	return ret;
10469
10470	switch (fcode)
10471	{
10472	case BUILT_IN_CONSTANT_P:
10473	{
10474	tree val = fold_builtin_constant_p (arg: arg0);
10475
10476	/* Gimplification will pull the CALL_EXPR for the builtin out of
10477	an if condition. When not optimizing, we'll not CSE it back.
10478	To avoid link error types of regressions, return false now. */
10479	if (!val && !optimize)
10480	val = integer_zero_node;
10481
10482	return val;
10483	}
10484
10485	case BUILT_IN_CLASSIFY_TYPE:
10486	return fold_builtin_classify_type (arg: arg0);
10487
10488	case BUILT_IN_STRLEN:
10489	return fold_builtin_strlen (loc, expr, type, arg: arg0);
10490
10491	CASE_FLT_FN (BUILT_IN_FABS):
10492	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
10493	case BUILT_IN_FABSD32:
10494	case BUILT_IN_FABSD64:
10495	case BUILT_IN_FABSD128:
10496	return fold_builtin_fabs (loc, arg: arg0, type);
10497
10498	case BUILT_IN_ABS:
10499	case BUILT_IN_LABS:
10500	case BUILT_IN_LLABS:
10501	case BUILT_IN_IMAXABS:
10502	return fold_builtin_abs (loc, arg: arg0, type);
10503
10504	CASE_FLT_FN (BUILT_IN_CONJ):
10505	if (validate_arg (arg0, code: COMPLEX_TYPE)
10506	&& TREE_CODE (TREE_TYPE (TREE_TYPE (arg0))) == REAL_TYPE)
10507	return fold_build1_loc (loc, CONJ_EXPR, type, arg0);
10508	break;
10509
10510	CASE_FLT_FN (BUILT_IN_CREAL):
10511	if (validate_arg (arg0, code: COMPLEX_TYPE)
10512	&& TREE_CODE (TREE_TYPE (TREE_TYPE (arg0))) == REAL_TYPE)
10513	return non_lvalue_loc (loc, fold_build1_loc (loc, REALPART_EXPR, type, arg0));
10514	break;
10515
10516	CASE_FLT_FN (BUILT_IN_CIMAG):
10517	if (validate_arg (arg0, code: COMPLEX_TYPE)
10518	&& TREE_CODE (TREE_TYPE (TREE_TYPE (arg0))) == REAL_TYPE)
10519	return non_lvalue_loc (loc, fold_build1_loc (loc, IMAGPART_EXPR, type, arg0));
10520	break;
10521
10522	CASE_FLT_FN (BUILT_IN_CARG):
10523	CASE_FLT_FN_FLOATN_NX (BUILT_IN_CARG):
10524	return fold_builtin_carg (loc, arg: arg0, type);
10525
10526	case BUILT_IN_ISASCII:
10527	return fold_builtin_isascii (loc, arg: arg0);
10528
10529	case BUILT_IN_TOASCII:
10530	return fold_builtin_toascii (loc, arg: arg0);
10531
10532	case BUILT_IN_ISDIGIT:
10533	return fold_builtin_isdigit (loc, arg: arg0);
10534
10535	CASE_FLT_FN (BUILT_IN_FINITE):
10536	case BUILT_IN_FINITED32:
10537	case BUILT_IN_FINITED64:
10538	case BUILT_IN_FINITED128:
10539	case BUILT_IN_ISFINITE:
10540	{
10541	tree ret = fold_builtin_classify (loc, fndecl, arg: arg0, builtin_index: BUILT_IN_ISFINITE);
10542	if (ret)
10543	return ret;
10544	return fold_builtin_interclass_mathfn (loc, fndecl, arg: arg0);
10545	}
10546
10547	CASE_FLT_FN (BUILT_IN_ISINF):
10548	case BUILT_IN_ISINFD32:
10549	case BUILT_IN_ISINFD64:
10550	case BUILT_IN_ISINFD128:
10551	{
10552	tree ret = fold_builtin_classify (loc, fndecl, arg: arg0, builtin_index: BUILT_IN_ISINF);
10553	if (ret)
10554	return ret;
10555	return fold_builtin_interclass_mathfn (loc, fndecl, arg: arg0);
10556	}
10557
10558	case BUILT_IN_ISNORMAL:
10559	return fold_builtin_interclass_mathfn (loc, fndecl, arg: arg0);
10560
10561	case BUILT_IN_ISINF_SIGN:
10562	return fold_builtin_classify (loc, fndecl, arg: arg0, builtin_index: BUILT_IN_ISINF_SIGN);
10563
10564	CASE_FLT_FN (BUILT_IN_ISNAN):
10565	case BUILT_IN_ISNAND32:
10566	case BUILT_IN_ISNAND64:
10567	case BUILT_IN_ISNAND128:
10568	return fold_builtin_classify (loc, fndecl, arg: arg0, builtin_index: BUILT_IN_ISNAN);
10569
10570	case BUILT_IN_ISSIGNALING:
10571	return fold_builtin_classify (loc, fndecl, arg: arg0, builtin_index: BUILT_IN_ISSIGNALING);
10572
10573	case BUILT_IN_FREE:
10574	if (integer_zerop (arg0))
10575	return build_empty_stmt (loc);
10576	break;
10577
10578	case BUILT_IN_CLZG:
10579	case BUILT_IN_CTZG:
10580	case BUILT_IN_CLRSBG:
10581	case BUILT_IN_FFSG:
10582	case BUILT_IN_PARITYG:
10583	case BUILT_IN_POPCOUNTG:
10584	return fold_builtin_bit_query (loc, fcode, arg0, NULL_TREE);
10585
10586	default:
10587	break;
10588	}
10589
10590	return NULL_TREE;
10591
10592	}
10593
10594	/* Folds a call EXPR (which may be null) to built-in function FNDECL
10595	with 2 arguments, ARG0 and ARG1. This function returns NULL_TREE
10596	if no simplification was possible. */
10597
10598	static tree
10599	fold_builtin_2 (location_t loc, tree expr, tree fndecl, tree arg0, tree arg1)
10600	{
10601	tree type = TREE_TYPE (TREE_TYPE (fndecl));
10602	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
10603
10604	if (error_operand_p (t: arg0)
10605	|| error_operand_p (t: arg1))
10606	return NULL_TREE;
10607
10608	if (tree ret = fold_const_call (as_combined_fn (fn: fcode), type, arg0, arg1))
10609	return ret;
10610
10611	switch (fcode)
10612	{
10613	CASE_FLT_FN_REENT (BUILT_IN_GAMMA): /* GAMMA_R */
10614	CASE_FLT_FN_REENT (BUILT_IN_LGAMMA): /* LGAMMA_R */
10615	if (validate_arg (arg0, code: REAL_TYPE)
10616	&& validate_arg (arg1, code: POINTER_TYPE))
10617	return do_mpfr_lgamma_r (arg0, arg1, type);
10618	break;
10619
10620	CASE_FLT_FN (BUILT_IN_FREXP):
10621	return fold_builtin_frexp (loc, arg0, arg1, rettype: type);
10622
10623	CASE_FLT_FN (BUILT_IN_MODF):
10624	return fold_builtin_modf (loc, arg0, arg1, rettype: type);
10625
10626	case BUILT_IN_STRSPN:
10627	return fold_builtin_strspn (loc, expr, arg0, arg1);
10628
10629	case BUILT_IN_STRCSPN:
10630	return fold_builtin_strcspn (loc, expr, arg0, arg1);
10631
10632	case BUILT_IN_STRPBRK:
10633	return fold_builtin_strpbrk (loc, expr, arg0, arg1, type);
10634
10635	case BUILT_IN_EXPECT:
10636	return fold_builtin_expect (loc, arg0, arg1, NULL_TREE, NULL_TREE);
10637
10638	case BUILT_IN_ISGREATER:
10639	return fold_builtin_unordered_cmp (loc, fndecl,
10640	arg0, arg1, unordered_code: UNLE_EXPR, ordered_code: LE_EXPR);
10641	case BUILT_IN_ISGREATEREQUAL:
10642	return fold_builtin_unordered_cmp (loc, fndecl,
10643	arg0, arg1, unordered_code: UNLT_EXPR, ordered_code: LT_EXPR);
10644	case BUILT_IN_ISLESS:
10645	return fold_builtin_unordered_cmp (loc, fndecl,
10646	arg0, arg1, unordered_code: UNGE_EXPR, ordered_code: GE_EXPR);
10647	case BUILT_IN_ISLESSEQUAL:
10648	return fold_builtin_unordered_cmp (loc, fndecl,
10649	arg0, arg1, unordered_code: UNGT_EXPR, ordered_code: GT_EXPR);
10650	case BUILT_IN_ISLESSGREATER:
10651	return fold_builtin_unordered_cmp (loc, fndecl,
10652	arg0, arg1, unordered_code: UNEQ_EXPR, ordered_code: EQ_EXPR);
10653	case BUILT_IN_ISUNORDERED:
10654	return fold_builtin_unordered_cmp (loc, fndecl,
10655	arg0, arg1, unordered_code: UNORDERED_EXPR,
10656	ordered_code: NOP_EXPR);
10657
10658	case BUILT_IN_ISEQSIG:
10659	return fold_builtin_iseqsig (loc, arg0, arg1);
10660
10661	/* We do the folding for va_start in the expander. */
10662	case BUILT_IN_VA_START:
10663	break;
10664
10665	case BUILT_IN_OBJECT_SIZE:
10666	case BUILT_IN_DYNAMIC_OBJECT_SIZE:
10667	return fold_builtin_object_size (arg0, arg1, fcode);
10668
10669	case BUILT_IN_ATOMIC_ALWAYS_LOCK_FREE:
10670	return fold_builtin_atomic_always_lock_free (arg0, arg1);
10671
10672	case BUILT_IN_ATOMIC_IS_LOCK_FREE:
10673	return fold_builtin_atomic_is_lock_free (arg0, arg1);
10674
10675	case BUILT_IN_CLZG:
10676	case BUILT_IN_CTZG:
10677	return fold_builtin_bit_query (loc, fcode, arg0, arg1);
10678
10679	default:
10680	break;
10681	}
10682	return NULL_TREE;
10683	}
10684
10685	/* Fold a call to built-in function FNDECL with 3 arguments, ARG0, ARG1,
10686	and ARG2.
10687	This function returns NULL_TREE if no simplification was possible. */
10688
10689	static tree
10690	fold_builtin_3 (location_t loc, tree fndecl,
10691	tree arg0, tree arg1, tree arg2)
10692	{
10693	tree type = TREE_TYPE (TREE_TYPE (fndecl));
10694	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
10695
10696	if (error_operand_p (t: arg0)
10697	|| error_operand_p (t: arg1)
10698	|| error_operand_p (t: arg2))
10699	return NULL_TREE;
10700
10701	if (tree ret = fold_const_call (as_combined_fn (fn: fcode), type,
10702	arg0, arg1, arg2))
10703	return ret;
10704
10705	switch (fcode)
10706	{
10707
10708	CASE_FLT_FN (BUILT_IN_SINCOS):
10709	return fold_builtin_sincos (loc, arg0, arg1, arg2);
10710
10711	CASE_FLT_FN (BUILT_IN_REMQUO):
10712	if (validate_arg (arg0, code: REAL_TYPE)
10713	&& validate_arg (arg1, code: REAL_TYPE)
10714	&& validate_arg (arg2, code: POINTER_TYPE))
10715	return do_mpfr_remquo (arg0, arg1, arg2);
10716	break;
10717
10718	case BUILT_IN_MEMCMP:
10719	return fold_builtin_memcmp (loc, arg1: arg0, arg2: arg1, len: arg2);
10720
10721	case BUILT_IN_EXPECT:
10722	return fold_builtin_expect (loc, arg0, arg1, arg2, NULL_TREE);
10723
10724	case BUILT_IN_EXPECT_WITH_PROBABILITY:
10725	return fold_builtin_expect (loc, arg0, arg1, NULL_TREE, arg3: arg2);
10726
10727	case BUILT_IN_ADD_OVERFLOW:
10728	case BUILT_IN_SUB_OVERFLOW:
10729	case BUILT_IN_MUL_OVERFLOW:
10730	case BUILT_IN_ADD_OVERFLOW_P:
10731	case BUILT_IN_SUB_OVERFLOW_P:
10732	case BUILT_IN_MUL_OVERFLOW_P:
10733	case BUILT_IN_SADD_OVERFLOW:
10734	case BUILT_IN_SADDL_OVERFLOW:
10735	case BUILT_IN_SADDLL_OVERFLOW:
10736	case BUILT_IN_SSUB_OVERFLOW:
10737	case BUILT_IN_SSUBL_OVERFLOW:
10738	case BUILT_IN_SSUBLL_OVERFLOW:
10739	case BUILT_IN_SMUL_OVERFLOW:
10740	case BUILT_IN_SMULL_OVERFLOW:
10741	case BUILT_IN_SMULLL_OVERFLOW:
10742	case BUILT_IN_UADD_OVERFLOW:
10743	case BUILT_IN_UADDL_OVERFLOW:
10744	case BUILT_IN_UADDLL_OVERFLOW:
10745	case BUILT_IN_USUB_OVERFLOW:
10746	case BUILT_IN_USUBL_OVERFLOW:
10747	case BUILT_IN_USUBLL_OVERFLOW:
10748	case BUILT_IN_UMUL_OVERFLOW:
10749	case BUILT_IN_UMULL_OVERFLOW:
10750	case BUILT_IN_UMULLL_OVERFLOW:
10751	return fold_builtin_arith_overflow (loc, fcode, arg0, arg1, arg2);
10752
10753	default:
10754	break;
10755	}
10756	return NULL_TREE;
10757	}
10758
10759	/* Folds a call EXPR (which may be null) to built-in function FNDECL.
10760	ARGS is an array of NARGS arguments. IGNORE is true if the result
10761	of the function call is ignored. This function returns NULL_TREE
10762	if no simplification was possible. */
10763
10764	static tree
10765	fold_builtin_n (location_t loc, tree expr, tree fndecl, tree *args,
10766	int nargs, bool)
10767	{
10768	tree ret = NULL_TREE;
10769
10770	switch (nargs)
10771	{
10772	case 0:
10773	ret = fold_builtin_0 (loc, fndecl);
10774	break;
10775	case 1:
10776	ret = fold_builtin_1 (loc, expr, fndecl, arg0: args[0]);
10777	break;
10778	case 2:
10779	ret = fold_builtin_2 (loc, expr, fndecl, arg0: args[0], arg1: args[1]);
10780	break;
10781	case 3:
10782	ret = fold_builtin_3 (loc, fndecl, arg0: args[0], arg1: args[1], arg2: args[2]);
10783	break;
10784	default:
10785	ret = fold_builtin_varargs (loc, fndecl, args, nargs);
10786	break;
10787	}
10788	if (ret)
10789	{
10790	ret = build1 (NOP_EXPR, TREE_TYPE (ret), ret);
10791	SET_EXPR_LOCATION (ret, loc);
10792	return ret;
10793	}
10794	return NULL_TREE;
10795	}
10796
10797	/* Construct a new CALL_EXPR to FNDECL using the tail of the argument
10798	list ARGS along with N new arguments in NEWARGS. SKIP is the number
10799	of arguments in ARGS to be omitted. OLDNARGS is the number of
10800	elements in ARGS. */
10801
10802	static tree
10803	rewrite_call_expr_valist (location_t loc, int oldnargs, tree *args,
10804	int skip, tree fndecl, int n, va_list newargs)
10805	{
10806	int nargs = oldnargs - skip + n;
10807	tree *buffer;
10808
10809	if (n > 0)
10810	{
10811	int i, j;
10812
10813	buffer = XALLOCAVEC (tree, nargs);
10814	for (i = 0; i < n; i++)
10815	buffer[i] = va_arg (newargs, tree);
10816	for (j = skip; j < oldnargs; j++, i++)
10817	buffer[i] = args[j];
10818	}
10819	else
10820	buffer = args + skip;
10821
10822	return build_call_expr_loc_array (loc, fndecl, nargs, buffer);
10823	}
10824
10825	/* Return true if FNDECL shouldn't be folded right now.
10826	If a built-in function has an inline attribute always_inline
10827	wrapper, defer folding it after always_inline functions have
10828	been inlined, otherwise e.g. -D_FORTIFY_SOURCE checking
10829	might not be performed. */
10830
10831	bool
10832	avoid_folding_inline_builtin (tree fndecl)
10833	{
10834	return (DECL_DECLARED_INLINE_P (fndecl)
10835	&& DECL_DISREGARD_INLINE_LIMITS (fndecl)
10836	&& cfun
10837	&& !cfun->always_inline_functions_inlined
10838	&& lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (fndecl)));
10839	}
10840
10841	/* A wrapper function for builtin folding that prevents warnings for
10842	"statement without effect" and the like, caused by removing the
10843	call node earlier than the warning is generated. */
10844
10845	tree
10846	fold_call_expr (location_t loc, tree exp, bool ignore)
10847	{
10848	tree ret = NULL_TREE;
10849	tree fndecl = get_callee_fndecl (exp);
10850	if (fndecl && fndecl_built_in_p (node: fndecl)
10851	/* If CALL_EXPR_VA_ARG_PACK is set, the arguments aren't finalized
10852	yet. Defer folding until we see all the arguments
10853	(after inlining). */
10854	&& !CALL_EXPR_VA_ARG_PACK (exp))
10855	{
10856	int nargs = call_expr_nargs (exp);
10857
10858	/* Before gimplification CALL_EXPR_VA_ARG_PACK is not set, but
10859	instead last argument is __builtin_va_arg_pack (). Defer folding
10860	even in that case, until arguments are finalized. */
10861	if (nargs && TREE_CODE (CALL_EXPR_ARG (exp, nargs - 1)) == CALL_EXPR)
10862	{
10863	tree fndecl2 = get_callee_fndecl (CALL_EXPR_ARG (exp, nargs - 1));
10864	if (fndecl2 && fndecl_built_in_p (node: fndecl2, name1: BUILT_IN_VA_ARG_PACK))
10865	return NULL_TREE;
10866	}
10867
10868	if (avoid_folding_inline_builtin (fndecl))
10869	return NULL_TREE;
10870
10871	if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
10872	return targetm.fold_builtin (fndecl, call_expr_nargs (exp),
10873	CALL_EXPR_ARGP (exp), ignore);
10874	else
10875	{
10876	tree *args = CALL_EXPR_ARGP (exp);
10877	ret = fold_builtin_n (loc, expr: exp, fndecl, args, nargs, ignore);
10878	if (ret)
10879	return ret;
10880	}
10881	}
10882	return NULL_TREE;
10883	}
10884
10885	/* Fold a CALL_EXPR with type TYPE with FN as the function expression.
10886	N arguments are passed in the array ARGARRAY. Return a folded
10887	expression or NULL_TREE if no simplification was possible. */
10888
10889	tree
10890	fold_builtin_call_array (location_t loc, tree,
10891	tree fn,
10892	int n,
10893	tree *argarray)
10894	{
10895	if (TREE_CODE (fn) != ADDR_EXPR)
10896	return NULL_TREE;
10897
10898	tree fndecl = TREE_OPERAND (fn, 0);
10899	if (TREE_CODE (fndecl) == FUNCTION_DECL
10900	&& fndecl_built_in_p (node: fndecl))
10901	{
10902	/* If last argument is __builtin_va_arg_pack (), arguments to this
10903	function are not finalized yet. Defer folding until they are. */
10904	if (n && TREE_CODE (argarray[n - 1]) == CALL_EXPR)
10905	{
10906	tree fndecl2 = get_callee_fndecl (argarray[n - 1]);
10907	if (fndecl2 && fndecl_built_in_p (node: fndecl2, name1: BUILT_IN_VA_ARG_PACK))
10908	return NULL_TREE;
10909	}
10910	if (avoid_folding_inline_builtin (fndecl))
10911	return NULL_TREE;
10912	if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
10913	return targetm.fold_builtin (fndecl, n, argarray, false);
10914	else
10915	return fold_builtin_n (loc, NULL_TREE, fndecl, args: argarray, nargs: n, false);
10916	}
10917
10918	return NULL_TREE;
10919	}
10920
10921	/* Construct a new CALL_EXPR using the tail of the argument list of EXP
10922	along with N new arguments specified as the "..." parameters. SKIP
10923	is the number of arguments in EXP to be omitted. This function is used
10924	to do varargs-to-varargs transformations. */
10925
10926	static tree
10927	rewrite_call_expr (location_t loc, tree exp, int skip, tree fndecl, int n, ...)
10928	{
10929	va_list ap;
10930	tree t;
10931
10932	va_start (ap, n);
10933	t = rewrite_call_expr_valist (loc, call_expr_nargs (exp),
10934	CALL_EXPR_ARGP (exp), skip, fndecl, n, newargs: ap);
10935	va_end (ap);
10936
10937	return t;
10938	}
10939
10940	/* Validate a single argument ARG against a tree code CODE representing
10941	a type. Return true when argument is valid. */
10942
10943	static bool
10944	validate_arg (const_tree arg, enum tree_code code)
10945	{
10946	if (!arg)
10947	return false;
10948	else if (code == POINTER_TYPE)
10949	return POINTER_TYPE_P (TREE_TYPE (arg));
10950	else if (code == INTEGER_TYPE)
10951	return INTEGRAL_TYPE_P (TREE_TYPE (arg));
10952	return code == TREE_CODE (TREE_TYPE (arg));
10953	}
10954
10955	/* This function validates the types of a function call argument list
10956	against a specified list of tree_codes. If the last specifier is a 0,
10957	that represents an ellipses, otherwise the last specifier must be a
10958	VOID_TYPE.
10959
10960	This is the GIMPLE version of validate_arglist. Eventually we want to
10961	completely convert builtins.cc to work from GIMPLEs and the tree based
10962	validate_arglist will then be removed. */
10963
10964	bool
10965	validate_gimple_arglist (const gcall *call, ...)
10966	{
10967	enum tree_code code;
10968	bool res = 0;
10969	va_list ap;
10970	const_tree arg;
10971	size_t i;
10972
10973	va_start (ap, call);
10974	i = 0;
10975
10976	do
10977	{
10978	code = (enum tree_code) va_arg (ap, int);
10979	switch (code)
10980	{
10981	case 0:
10982	/* This signifies an ellipses, any further arguments are all ok. */
10983	res = true;
10984	goto end;
10985	case VOID_TYPE:
10986	/* This signifies an endlink, if no arguments remain, return
10987	true, otherwise return false. */
10988	res = (i == gimple_call_num_args (gs: call));
10989	goto end;
10990	default:
10991	/* If no parameters remain or the parameter's code does not
10992	match the specified code, return false. Otherwise continue
10993	checking any remaining arguments. */
10994	arg = gimple_call_arg (gs: call, index: i++);
10995	if (!validate_arg (arg, code))
10996	goto end;
10997	break;
10998	}
10999	}
11000	while (1);
11001
11002	/* We need gotos here since we can only have one VA_CLOSE in a
11003	function. */
11004	end: ;
11005	va_end (ap);
11006
11007	return res;
11008	}
11009
11010	/* Default target-specific builtin expander that does nothing. */
11011
11012	rtx
11013	default_expand_builtin (tree exp ATTRIBUTE_UNUSED,
11014	rtx target ATTRIBUTE_UNUSED,
11015	rtx subtarget ATTRIBUTE_UNUSED,
11016	machine_mode mode ATTRIBUTE_UNUSED,
11017	int ignore ATTRIBUTE_UNUSED)
11018	{
11019	return NULL_RTX;
11020	}
11021
11022	/* Returns true is EXP represents data that would potentially reside
11023	in a readonly section. */
11024
11025	bool
11026	readonly_data_expr (tree exp)
11027	{
11028	STRIP_NOPS (exp);
11029
11030	if (TREE_CODE (exp) != ADDR_EXPR)
11031	return false;
11032
11033	exp = get_base_address (TREE_OPERAND (exp, 0));
11034	if (!exp)
11035	return false;
11036
11037	/* Make sure we call decl_readonly_section only for trees it
11038	can handle (since it returns true for everything it doesn't
11039	understand). */
11040	if (TREE_CODE (exp) == STRING_CST
11041	|| TREE_CODE (exp) == CONSTRUCTOR
11042	|| (VAR_P (exp) && TREE_STATIC (exp)))
11043	return decl_readonly_section (exp, 0);
11044	else
11045	return false;
11046	}
11047
11048	/* Simplify a call to the strpbrk builtin. S1 and S2 are the arguments
11049	to the call, and TYPE is its return type.
11050
11051	Return NULL_TREE if no simplification was possible, otherwise return the
11052	simplified form of the call as a tree.
11053
11054	The simplified form may be a constant or other expression which
11055	computes the same value, but in a more efficient manner (including
11056	calls to other builtin functions).
11057
11058	The call may contain arguments which need to be evaluated, but
11059	which are not useful to determine the result of the call. In
11060	this case we return a chain of COMPOUND_EXPRs. The LHS of each
11061	COMPOUND_EXPR will be an argument which must be evaluated.
11062	COMPOUND_EXPRs are chained through their RHS. The RHS of the last
11063	COMPOUND_EXPR in the chain will contain the tree for the simplified
11064	form of the builtin function call. */
11065
11066	static tree
11067	fold_builtin_strpbrk (location_t loc, tree, tree s1, tree s2, tree type)
11068	{
11069	if (!validate_arg (arg: s1, code: POINTER_TYPE)
11070	|| !validate_arg (arg: s2, code: POINTER_TYPE))
11071	return NULL_TREE;
11072
11073	tree fn;
11074	const char *p1, *p2;
11075
11076	p2 = c_getstr (s2);
11077	if (p2 == NULL)
11078	return NULL_TREE;
11079
11080	p1 = c_getstr (s1);
11081	if (p1 != NULL)
11082	{
11083	const char *r = strpbrk (s: p1, accept: p2);
11084	tree tem;
11085
11086	if (r == NULL)
11087	return build_int_cst (TREE_TYPE (s1), 0);
11088
11089	/* Return an offset into the constant string argument. */
11090	tem = fold_build_pointer_plus_hwi_loc (loc, ptr: s1, off: r - p1);
11091	return fold_convert_loc (loc, type, tem);
11092	}
11093
11094	if (p2[0] == '\0')
11095	/* strpbrk(x, "") == NULL.
11096	Evaluate and ignore s1 in case it had side-effects. */
11097	return omit_one_operand_loc (loc, type, integer_zero_node, s1);
11098
11099	if (p2[1] != '\0')
11100	return NULL_TREE; /* Really call strpbrk. */
11101
11102	fn = builtin_decl_implicit (fncode: BUILT_IN_STRCHR);
11103	if (!fn)
11104	return NULL_TREE;
11105
11106	/* New argument list transforming strpbrk(s1, s2) to
11107	strchr(s1, s2[0]). */
11108	return build_call_expr_loc (loc, fn, 2, s1,
11109	build_int_cst (integer_type_node, p2[0]));
11110	}
11111
11112	/* Simplify a call to the strspn builtin. S1 and S2 are the arguments
11113	to the call.
11114
11115	Return NULL_TREE if no simplification was possible, otherwise return the
11116	simplified form of the call as a tree.
11117
11118	The simplified form may be a constant or other expression which
11119	computes the same value, but in a more efficient manner (including
11120	calls to other builtin functions).
11121
11122	The call may contain arguments which need to be evaluated, but
11123	which are not useful to determine the result of the call. In
11124	this case we return a chain of COMPOUND_EXPRs. The LHS of each
11125	COMPOUND_EXPR will be an argument which must be evaluated.
11126	COMPOUND_EXPRs are chained through their RHS. The RHS of the last
11127	COMPOUND_EXPR in the chain will contain the tree for the simplified
11128	form of the builtin function call. */
11129
11130	static tree
11131	fold_builtin_strspn (location_t loc, tree expr, tree s1, tree s2)
11132	{
11133	if (!validate_arg (arg: s1, code: POINTER_TYPE)
11134	|| !validate_arg (arg: s2, code: POINTER_TYPE))
11135	return NULL_TREE;
11136
11137	if (!check_nul_terminated_array (expr, s1)
11138	|| !check_nul_terminated_array (expr, s2))
11139	return NULL_TREE;
11140
11141	const char *p1 = c_getstr (s1), *p2 = c_getstr (s2);
11142
11143	/* If either argument is "", return NULL_TREE. */
11144	if ((p1 && *p1 == '\0') || (p2 && *p2 == '\0'))
11145	/* Evaluate and ignore both arguments in case either one has
11146	side-effects. */
11147	return omit_two_operands_loc (loc, size_type_node, size_zero_node,
11148	s1, s2);
11149	return NULL_TREE;
11150	}
11151
11152	/* Simplify a call to the strcspn builtin. S1 and S2 are the arguments
11153	to the call.
11154
11155	Return NULL_TREE if no simplification was possible, otherwise return the
11156	simplified form of the call as a tree.
11157
11158	The simplified form may be a constant or other expression which
11159	computes the same value, but in a more efficient manner (including
11160	calls to other builtin functions).
11161
11162	The call may contain arguments which need to be evaluated, but
11163	which are not useful to determine the result of the call. In
11164	this case we return a chain of COMPOUND_EXPRs. The LHS of each
11165	COMPOUND_EXPR will be an argument which must be evaluated.
11166	COMPOUND_EXPRs are chained through their RHS. The RHS of the last
11167	COMPOUND_EXPR in the chain will contain the tree for the simplified
11168	form of the builtin function call. */
11169
11170	static tree
11171	fold_builtin_strcspn (location_t loc, tree expr, tree s1, tree s2)
11172	{
11173	if (!validate_arg (arg: s1, code: POINTER_TYPE)
11174	|| !validate_arg (arg: s2, code: POINTER_TYPE))
11175	return NULL_TREE;
11176
11177	if (!check_nul_terminated_array (expr, s1)
11178	|| !check_nul_terminated_array (expr, s2))
11179	return NULL_TREE;
11180
11181	/* If the first argument is "", return NULL_TREE. */
11182	const char *p1 = c_getstr (s1);
11183	if (p1 && *p1 == '\0')
11184	{
11185	/* Evaluate and ignore argument s2 in case it has
11186	side-effects. */
11187	return omit_one_operand_loc (loc, size_type_node,
11188	size_zero_node, s2);
11189	}
11190
11191	/* If the second argument is "", return __builtin_strlen(s1). */
11192	const char *p2 = c_getstr (s2);
11193	if (p2 && *p2 == '\0')
11194	{
11195	tree fn = builtin_decl_implicit (fncode: BUILT_IN_STRLEN);
11196
11197	/* If the replacement _DECL isn't initialized, don't do the
11198	transformation. */
11199	if (!fn)
11200	return NULL_TREE;
11201
11202	return build_call_expr_loc (loc, fn, 1, s1);
11203	}
11204	return NULL_TREE;
11205	}
11206
11207	/* Fold the next_arg or va_start call EXP. Returns true if there was an error
11208	produced. False otherwise. This is done so that we don't output the error
11209	or warning twice or three times. */
11210
11211	bool
11212	fold_builtin_next_arg (tree exp, bool va_start_p)
11213	{
11214	tree fntype = TREE_TYPE (current_function_decl);
11215	int nargs = call_expr_nargs (exp);
11216	tree arg;
11217	/* There is good chance the current input_location points inside the
11218	definition of the va_start macro (perhaps on the token for
11219	builtin) in a system header, so warnings will not be emitted.
11220	Use the location in real source code. */
11221	location_t current_location =
11222	linemap_unwind_to_first_non_reserved_loc (line_table, loc: input_location,
11223	NULL);
11224
11225	if (!stdarg_p (fntype))
11226	{
11227	error ("%<va_start%> used in function with fixed arguments");
11228	return true;
11229	}
11230
11231	if (va_start_p)
11232	{
11233	if (va_start_p && (nargs != 2))
11234	{
11235	error ("wrong number of arguments to function %<va_start%>");
11236	return true;
11237	}
11238	arg = CALL_EXPR_ARG (exp, 1);
11239	}
11240	/* We use __builtin_va_start (ap, 0, 0) or __builtin_next_arg (0, 0)
11241	when we checked the arguments and if needed issued a warning. */
11242	else
11243	{
11244	if (nargs == 0)
11245	{
11246	/* Evidently an out of date version of <stdarg.h>; can't validate
11247	va_start's second argument, but can still work as intended. */
11248	warning_at (current_location,
11249	OPT_Wvarargs,
11250	"%<__builtin_next_arg%> called without an argument");
11251	return true;
11252	}
11253	else if (nargs > 1)
11254	{
11255	error ("wrong number of arguments to function %<__builtin_next_arg%>");
11256	return true;
11257	}
11258	arg = CALL_EXPR_ARG (exp, 0);
11259	}
11260
11261	if (TREE_CODE (arg) == SSA_NAME
11262	&& SSA_NAME_VAR (arg))
11263	arg = SSA_NAME_VAR (arg);
11264
11265	/* We destructively modify the call to be __builtin_va_start (ap, 0)
11266	or __builtin_next_arg (0) the first time we see it, after checking
11267	the arguments and if needed issuing a warning. */
11268	if (!integer_zerop (arg))
11269	{
11270	tree last_parm = tree_last (DECL_ARGUMENTS (current_function_decl));
11271
11272	/* Strip off all nops for the sake of the comparison. This
11273	is not quite the same as STRIP_NOPS. It does more.
11274	We must also strip off INDIRECT_EXPR for C++ reference
11275	parameters. */
11276	while (CONVERT_EXPR_P (arg)
11277	|| INDIRECT_REF_P (arg))
11278	arg = TREE_OPERAND (arg, 0);
11279	if (arg != last_parm)
11280	{
11281	/* FIXME: Sometimes with the tree optimizers we can get the
11282	not the last argument even though the user used the last
11283	argument. We just warn and set the arg to be the last
11284	argument so that we will get wrong-code because of
11285	it. */
11286	warning_at (current_location,
11287	OPT_Wvarargs,
11288	"second parameter of %<va_start%> not last named argument");
11289	}
11290
11291	/* Undefined by C99 7.15.1.4p4 (va_start):
11292	"If the parameter parmN is declared with the register storage
11293	class, with a function or array type, or with a type that is
11294	not compatible with the type that results after application of
11295	the default argument promotions, the behavior is undefined."
11296	*/
11297	else if (DECL_REGISTER (arg))
11298	{
11299	warning_at (current_location,
11300	OPT_Wvarargs,
11301	"undefined behavior when second parameter of "
11302	"%<va_start%> is declared with %<register%> storage");
11303	}
11304
11305	/* We want to verify the second parameter just once before the tree
11306	optimizers are run and then avoid keeping it in the tree,
11307	as otherwise we could warn even for correct code like:
11308	void foo (int i, ...)
11309	{ va_list ap; i++; va_start (ap, i); va_end (ap); } */
11310	if (va_start_p)
11311	CALL_EXPR_ARG (exp, 1) = integer_zero_node;
11312	else
11313	CALL_EXPR_ARG (exp, 0) = integer_zero_node;
11314	}
11315	return false;
11316	}
11317
11318
11319	/* Expand a call EXP to __builtin_object_size. */
11320
11321	static rtx
11322	expand_builtin_object_size (tree exp)
11323	{
11324	tree ost;
11325	int object_size_type;
11326	tree fndecl = get_callee_fndecl (exp);
11327
11328	if (!validate_arglist (callexpr: exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
11329	{
11330	error ("first argument of %qD must be a pointer, second integer constant",
11331	fndecl);
11332	expand_builtin_trap ();
11333	return const0_rtx;
11334	}
11335
11336	ost = CALL_EXPR_ARG (exp, 1);
11337	STRIP_NOPS (ost);
11338
11339	if (TREE_CODE (ost) != INTEGER_CST
11340	|| tree_int_cst_sgn (ost) < 0
11341	|| compare_tree_int (ost, 3) > 0)
11342	{
11343	error ("last argument of %qD is not integer constant between 0 and 3",
11344	fndecl);
11345	expand_builtin_trap ();
11346	return const0_rtx;
11347	}
11348
11349	object_size_type = tree_to_shwi (ost);
11350
11351	return object_size_type < 2 ? constm1_rtx : const0_rtx;
11352	}
11353
11354	/* Expand EXP, a call to the __mem{cpy,pcpy,move,set}_chk builtin.
11355	FCODE is the BUILT_IN_* to use.
11356	Return NULL_RTX if we failed; the caller should emit a normal call,
11357	otherwise try to get the result in TARGET, if convenient (and in
11358	mode MODE if that's convenient). */
11359
11360	static rtx
11361	expand_builtin_memory_chk (tree exp, rtx target, machine_mode mode,
11362	enum built_in_function fcode)
11363	{
11364	if (!validate_arglist (callexpr: exp,
11365	POINTER_TYPE,
11366	fcode == BUILT_IN_MEMSET_CHK
11367	? INTEGER_TYPE : POINTER_TYPE,
11368	INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE))
11369	return NULL_RTX;
11370
11371	tree dest = CALL_EXPR_ARG (exp, 0);
11372	tree src = CALL_EXPR_ARG (exp, 1);
11373	tree len = CALL_EXPR_ARG (exp, 2);
11374	tree size = CALL_EXPR_ARG (exp, 3);
11375
11376	/* FIXME: Set access mode to write only for memset et al. */
11377	bool sizes_ok = check_access (exp, len, /*maxread=*/NULL_TREE,
11378	/*srcstr=*/NULL_TREE, size, access_read_write);
11379
11380	if (!tree_fits_uhwi_p (size))
11381	return NULL_RTX;
11382
11383	if (tree_fits_uhwi_p (len) || integer_all_onesp (size))
11384	{
11385	/* Avoid transforming the checking call to an ordinary one when
11386	an overflow has been detected or when the call couldn't be
11387	validated because the size is not constant. */
11388	if (!sizes_ok && !integer_all_onesp (size) && tree_int_cst_lt (t1: size, t2: len))
11389	return NULL_RTX;
11390
11391	tree fn = NULL_TREE;
11392	/* If __builtin_mem{cpy,pcpy,move,set}_chk is used, assume
11393	mem{cpy,pcpy,move,set} is available. */
11394	switch (fcode)
11395	{
11396	case BUILT_IN_MEMCPY_CHK:
11397	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMCPY);
11398	break;
11399	case BUILT_IN_MEMPCPY_CHK:
11400	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMPCPY);
11401	break;
11402	case BUILT_IN_MEMMOVE_CHK:
11403	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMMOVE);
11404	break;
11405	case BUILT_IN_MEMSET_CHK:
11406	fn = builtin_decl_explicit (fncode: BUILT_IN_MEMSET);
11407	break;
11408	default:
11409	break;
11410	}
11411
11412	if (! fn)
11413	return NULL_RTX;
11414
11415	fn = build_call_nofold_loc (EXPR_LOCATION (exp), fndecl: fn, n: 3, dest, src, len);
11416	gcc_assert (TREE_CODE (fn) == CALL_EXPR);
11417	CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (exp);
11418	return expand_expr (exp: fn, target, mode, modifier: EXPAND_NORMAL);
11419	}
11420	else if (fcode == BUILT_IN_MEMSET_CHK)
11421	return NULL_RTX;
11422	else
11423	{
11424	unsigned int dest_align = get_pointer_alignment (exp: dest);
11425
11426	/* If DEST is not a pointer type, call the normal function. */
11427	if (dest_align == 0)
11428	return NULL_RTX;
11429
11430	/* If SRC and DEST are the same (and not volatile), do nothing. */
11431	if (operand_equal_p (src, dest, flags: 0))
11432	{
11433	tree expr;
11434
11435	if (fcode != BUILT_IN_MEMPCPY_CHK)
11436	{
11437	/* Evaluate and ignore LEN in case it has side-effects. */
11438	expand_expr (exp: len, const0_rtx, VOIDmode, modifier: EXPAND_NORMAL);
11439	return expand_expr (exp: dest, target, mode, modifier: EXPAND_NORMAL);
11440	}
11441
11442	expr = fold_build_pointer_plus (dest, len);
11443	return expand_expr (exp: expr, target, mode, modifier: EXPAND_NORMAL);
11444	}
11445
11446	/* __memmove_chk special case. */
11447	if (fcode == BUILT_IN_MEMMOVE_CHK)
11448	{
11449	unsigned int src_align = get_pointer_alignment (exp: src);
11450
11451	if (src_align == 0)
11452	return NULL_RTX;
11453
11454	/* If src is categorized for a readonly section we can use
11455	normal __memcpy_chk. */
11456	if (readonly_data_expr (exp: src))
11457	{
11458	tree fn = builtin_decl_explicit (fncode: BUILT_IN_MEMCPY_CHK);
11459	if (!fn)
11460	return NULL_RTX;
11461	fn = build_call_nofold_loc (EXPR_LOCATION (exp), fndecl: fn, n: 4,
11462	dest, src, len, size);
11463	gcc_assert (TREE_CODE (fn) == CALL_EXPR);
11464	CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (exp);
11465	return expand_expr (exp: fn, target, mode, modifier: EXPAND_NORMAL);
11466	}
11467	}
11468	return NULL_RTX;
11469	}
11470	}
11471
11472	/* Emit warning if a buffer overflow is detected at compile time. */
11473
11474	static void
11475	maybe_emit_chk_warning (tree exp, enum built_in_function fcode)
11476	{
11477	/* The source string. */
11478	tree srcstr = NULL_TREE;
11479	/* The size of the destination object returned by __builtin_object_size. */
11480	tree objsize = NULL_TREE;
11481	/* The string that is being concatenated with (as in __strcat_chk)
11482	or null if it isn't. */
11483	tree catstr = NULL_TREE;
11484	/* The maximum length of the source sequence in a bounded operation
11485	(such as __strncat_chk) or null if the operation isn't bounded
11486	(such as __strcat_chk). */
11487	tree maxread = NULL_TREE;
11488	/* The exact size of the access (such as in __strncpy_chk). */
11489	tree size = NULL_TREE;
11490	/* The access by the function that's checked. Except for snprintf
11491	both writing and reading is checked. */
11492	access_mode mode = access_read_write;
11493
11494	switch (fcode)
11495	{
11496	case BUILT_IN_STRCPY_CHK:
11497	case BUILT_IN_STPCPY_CHK:
11498	srcstr = CALL_EXPR_ARG (exp, 1);
11499	objsize = CALL_EXPR_ARG (exp, 2);
11500	break;
11501
11502	case BUILT_IN_STRCAT_CHK:
11503	/* For __strcat_chk the warning will be emitted only if overflowing
11504	by at least strlen (dest) + 1 bytes. */
11505	catstr = CALL_EXPR_ARG (exp, 0);
11506	srcstr = CALL_EXPR_ARG (exp, 1);
11507	objsize = CALL_EXPR_ARG (exp, 2);
11508	break;
11509
11510	case BUILT_IN_STRNCAT_CHK:
11511	catstr = CALL_EXPR_ARG (exp, 0);
11512	srcstr = CALL_EXPR_ARG (exp, 1);
11513	maxread = CALL_EXPR_ARG (exp, 2);
11514	objsize = CALL_EXPR_ARG (exp, 3);
11515	break;
11516
11517	case BUILT_IN_STRNCPY_CHK:
11518	case BUILT_IN_STPNCPY_CHK:
11519	srcstr = CALL_EXPR_ARG (exp, 1);
11520	size = CALL_EXPR_ARG (exp, 2);
11521	objsize = CALL_EXPR_ARG (exp, 3);
11522	break;
11523
11524	case BUILT_IN_SNPRINTF_CHK:
11525	case BUILT_IN_VSNPRINTF_CHK:
11526	maxread = CALL_EXPR_ARG (exp, 1);
11527	objsize = CALL_EXPR_ARG (exp, 3);
11528	/* The only checked access the write to the destination. */
11529	mode = access_write_only;
11530	break;
11531	default:
11532	gcc_unreachable ();
11533	}
11534
11535	if (catstr && maxread)
11536	{
11537	/* Check __strncat_chk. There is no way to determine the length
11538	of the string to which the source string is being appended so
11539	just warn when the length of the source string is not known. */
11540	check_strncat_sizes (exp, objsize);
11541	return;
11542	}
11543
11544	check_access (exp, size, maxread, srcstr, objsize, mode);
11545	}
11546
11547	/* Emit warning if a buffer overflow is detected at compile time
11548	in __sprintf_chk/__vsprintf_chk calls. */
11549
11550	static void
11551	maybe_emit_sprintf_chk_warning (tree exp, enum built_in_function fcode)
11552	{
11553	tree size, len, fmt;
11554	const char *fmt_str;
11555	int nargs = call_expr_nargs (exp);
11556
11557	/* Verify the required arguments in the original call. */
11558
11559	if (nargs < 4)
11560	return;
11561	size = CALL_EXPR_ARG (exp, 2);
11562	fmt = CALL_EXPR_ARG (exp, 3);
11563
11564	if (! tree_fits_uhwi_p (size) || integer_all_onesp (size))
11565	return;
11566
11567	/* Check whether the format is a literal string constant. */
11568	fmt_str = c_getstr (fmt);
11569	if (fmt_str == NULL)
11570	return;
11571
11572	if (!init_target_chars ())
11573	return;
11574
11575	/* If the format doesn't contain % args or %%, we know its size. */
11576	if (strchr (s: fmt_str, c: target_percent) == 0)
11577	len = build_int_cstu (size_type_node, strlen (s: fmt_str));
11578	/* If the format is "%s" and first ... argument is a string literal,
11579	we know it too. */
11580	else if (fcode == BUILT_IN_SPRINTF_CHK
11581	&& strcmp (s1: fmt_str, s2: target_percent_s) == 0)
11582	{
11583	tree arg;
11584
11585	if (nargs < 5)
11586	return;
11587	arg = CALL_EXPR_ARG (exp, 4);
11588	if (! POINTER_TYPE_P (TREE_TYPE (arg)))
11589	return;
11590
11591	len = c_strlen (arg, only_value: 1);
11592	if (!len || ! tree_fits_uhwi_p (len))
11593	return;
11594	}
11595	else
11596	return;
11597
11598	/* Add one for the terminating nul. */
11599	len = fold_build2 (PLUS_EXPR, TREE_TYPE (len), len, size_one_node);
11600
11601	check_access (exp, /*size=*/NULL_TREE, /*maxread=*/NULL_TREE, len, size,
11602	access_write_only);
11603	}
11604
11605	/* Fold a call to __builtin_object_size with arguments PTR and OST,
11606	if possible. */
11607
11608	static tree
11609	fold_builtin_object_size (tree ptr, tree ost, enum built_in_function fcode)
11610	{
11611	tree bytes;
11612	int object_size_type;
11613
11614	if (!validate_arg (arg: ptr, code: POINTER_TYPE)
11615	|| !validate_arg (arg: ost, code: INTEGER_TYPE))
11616	return NULL_TREE;
11617
11618	STRIP_NOPS (ost);
11619
11620	if (TREE_CODE (ost) != INTEGER_CST
11621	|| tree_int_cst_sgn (ost) < 0
11622	|| compare_tree_int (ost, 3) > 0)
11623	return NULL_TREE;
11624
11625	object_size_type = tree_to_shwi (ost);
11626
11627	/* __builtin_object_size doesn't evaluate side-effects in its arguments;
11628	if there are any side-effects, it returns (size_t) -1 for types 0 and 1
11629	and (size_t) 0 for types 2 and 3. */
11630	if (TREE_SIDE_EFFECTS (ptr))
11631	return build_int_cst_type (size_type_node, object_size_type < 2 ? -1 : 0);
11632
11633	if (fcode == BUILT_IN_DYNAMIC_OBJECT_SIZE)
11634	object_size_type |= OST_DYNAMIC;
11635
11636	if (TREE_CODE (ptr) == ADDR_EXPR)
11637	{
11638	compute_builtin_object_size (ptr, object_size_type, &bytes);
11639	if ((object_size_type & OST_DYNAMIC)
11640	|| int_fits_type_p (bytes, size_type_node))
11641	return fold_convert (size_type_node, bytes);
11642	}
11643	else if (TREE_CODE (ptr) == SSA_NAME)
11644	{
11645	/* If object size is not known yet, delay folding until
11646	later. Maybe subsequent passes will help determining
11647	it. */
11648	if (compute_builtin_object_size (ptr, object_size_type, &bytes)
11649	&& ((object_size_type & OST_DYNAMIC)
11650	|| int_fits_type_p (bytes, size_type_node)))
11651	return fold_convert (size_type_node, bytes);
11652	}
11653
11654	return NULL_TREE;
11655	}
11656
11657	/* Builtins with folding operations that operate on "..." arguments
11658	need special handling; we need to store the arguments in a convenient
11659	data structure before attempting any folding. Fortunately there are
11660	only a few builtins that fall into this category. FNDECL is the
11661	function, EXP is the CALL_EXPR for the call. */
11662
11663	static tree
11664	fold_builtin_varargs (location_t loc, tree fndecl, tree *args, int nargs)
11665	{
11666	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: fndecl);
11667	tree ret = NULL_TREE;
11668
11669	switch (fcode)
11670	{
11671	case BUILT_IN_FPCLASSIFY:
11672	ret = fold_builtin_fpclassify (loc, args, nargs);
11673	break;
11674
11675	case BUILT_IN_ADDC:
11676	case BUILT_IN_ADDCL:
11677	case BUILT_IN_ADDCLL:
11678	case BUILT_IN_SUBC:
11679	case BUILT_IN_SUBCL:
11680	case BUILT_IN_SUBCLL:
11681	return fold_builtin_addc_subc (loc, fcode, args);
11682
11683	default:
11684	break;
11685	}
11686	if (ret)
11687	{
11688	ret = build1 (NOP_EXPR, TREE_TYPE (ret), ret);
11689	SET_EXPR_LOCATION (ret, loc);
11690	suppress_warning (ret);
11691	return ret;
11692	}
11693	return NULL_TREE;
11694	}
11695
11696	/* Initialize format string characters in the target charset. */
11697
11698	bool
11699	init_target_chars (void)
11700	{
11701	static bool init;
11702	if (!init)
11703	{
11704	target_newline = lang_hooks.to_target_charset ('\n');
11705	target_percent = lang_hooks.to_target_charset ('%');
11706	target_c = lang_hooks.to_target_charset ('c');
11707	target_s = lang_hooks.to_target_charset ('s');
11708	if (target_newline == 0 || target_percent == 0 || target_c == 0
11709	|| target_s == 0)
11710	return false;
11711
11712	target_percent_c[0] = target_percent;
11713	target_percent_c[1] = target_c;
11714	target_percent_c[2] = '\0';
11715
11716	target_percent_s[0] = target_percent;
11717	target_percent_s[1] = target_s;
11718	target_percent_s[2] = '\0';
11719
11720	target_percent_s_newline[0] = target_percent;
11721	target_percent_s_newline[1] = target_s;
11722	target_percent_s_newline[2] = target_newline;
11723	target_percent_s_newline[3] = '\0';
11724
11725	init = true;
11726	}
11727	return true;
11728	}
11729
11730	/* Helper function for do_mpfr_arg*(). Ensure M is a normal number
11731	and no overflow/underflow occurred. INEXACT is true if M was not
11732	exactly calculated. TYPE is the tree type for the result. This
11733	function assumes that you cleared the MPFR flags and then
11734	calculated M to see if anything subsequently set a flag prior to
11735	entering this function. Return NULL_TREE if any checks fail. */
11736
11737	static tree
11738	do_mpfr_ckconv (mpfr_srcptr m, tree type, int inexact)
11739	{
11740	/* Proceed iff we get a normal number, i.e. not NaN or Inf and no
11741	overflow/underflow occurred. If -frounding-math, proceed iff the
11742	result of calling FUNC was exact. */
11743	if (mpfr_number_p (m) && !mpfr_overflow_p () && !mpfr_underflow_p ()
11744	&& (!flag_rounding_math || !inexact))
11745	{
11746	REAL_VALUE_TYPE rr;
11747
11748	real_from_mpfr (&rr, m, type, MPFR_RNDN);
11749	/* Proceed iff GCC's REAL_VALUE_TYPE can hold the MPFR value,
11750	check for overflow/underflow. If the REAL_VALUE_TYPE is zero
11751	but the mpfr_t is not, then we underflowed in the
11752	conversion. */
11753	if (real_isfinite (&rr)
11754	&& (rr.cl == rvc_zero) == (mpfr_zero_p (m) != 0))
11755	{
11756	REAL_VALUE_TYPE rmode;
11757
11758	real_convert (&rmode, TYPE_MODE (type), &rr);
11759	/* Proceed iff the specified mode can hold the value. */
11760	if (real_identical (&rmode, &rr))
11761	return build_real (type, rmode);
11762	}
11763	}
11764	return NULL_TREE;
11765	}
11766
11767	/* Helper function for do_mpc_arg*(). Ensure M is a normal complex
11768	number and no overflow/underflow occurred. INEXACT is true if M
11769	was not exactly calculated. TYPE is the tree type for the result.
11770	This function assumes that you cleared the MPFR flags and then
11771	calculated M to see if anything subsequently set a flag prior to
11772	entering this function. Return NULL_TREE if any checks fail, if
11773	FORCE_CONVERT is true, then bypass the checks. */
11774
11775	static tree
11776	do_mpc_ckconv (mpc_srcptr m, tree type, int inexact, int force_convert)
11777	{
11778	/* Proceed iff we get a normal number, i.e. not NaN or Inf and no
11779	overflow/underflow occurred. If -frounding-math, proceed iff the
11780	result of calling FUNC was exact. */
11781	if (force_convert
11782	|| (mpfr_number_p (mpc_realref (m)) && mpfr_number_p (mpc_imagref (m))
11783	&& !mpfr_overflow_p () && !mpfr_underflow_p ()
11784	&& (!flag_rounding_math || !inexact)))
11785	{
11786	REAL_VALUE_TYPE re, im;
11787
11788	real_from_mpfr (&re, mpc_realref (m), TREE_TYPE (type), MPFR_RNDN);
11789	real_from_mpfr (&im, mpc_imagref (m), TREE_TYPE (type), MPFR_RNDN);
11790	/* Proceed iff GCC's REAL_VALUE_TYPE can hold the MPFR values,
11791	check for overflow/underflow. If the REAL_VALUE_TYPE is zero
11792	but the mpfr_t is not, then we underflowed in the
11793	conversion. */
11794	if (force_convert
11795	|| (real_isfinite (&re) && real_isfinite (&im)
11796	&& (re.cl == rvc_zero) == (mpfr_zero_p (mpc_realref (m)) != 0)
11797	&& (im.cl == rvc_zero) == (mpfr_zero_p (mpc_imagref (m)) != 0)))
11798	{
11799	REAL_VALUE_TYPE re_mode, im_mode;
11800
11801	real_convert (&re_mode, TYPE_MODE (TREE_TYPE (type)), &re);
11802	real_convert (&im_mode, TYPE_MODE (TREE_TYPE (type)), &im);
11803	/* Proceed iff the specified mode can hold the value. */
11804	if (force_convert
11805	|| (real_identical (&re_mode, &re)
11806	&& real_identical (&im_mode, &im)))
11807	return build_complex (type, build_real (TREE_TYPE (type), re_mode),
11808	build_real (TREE_TYPE (type), im_mode));
11809	}
11810	}
11811	return NULL_TREE;
11812	}
11813
11814	/* If arguments ARG0 and ARG1 are REAL_CSTs, call mpfr_remquo() to set
11815	the pointer *(ARG_QUO) and return the result. The type is taken
11816	from the type of ARG0 and is used for setting the precision of the
11817	calculation and results. */
11818
11819	static tree
11820	do_mpfr_remquo (tree arg0, tree arg1, tree arg_quo)
11821	{
11822	tree const type = TREE_TYPE (arg0);
11823	tree result = NULL_TREE;
11824
11825	STRIP_NOPS (arg0);
11826	STRIP_NOPS (arg1);
11827
11828	/* To proceed, MPFR must exactly represent the target floating point
11829	format, which only happens when the target base equals two. */
11830	if (REAL_MODE_FORMAT (TYPE_MODE (type))->b == 2
11831	&& TREE_CODE (arg0) == REAL_CST && !TREE_OVERFLOW (arg0)
11832	&& TREE_CODE (arg1) == REAL_CST && !TREE_OVERFLOW (arg1))
11833	{
11834	const REAL_VALUE_TYPE *const ra0 = TREE_REAL_CST_PTR (arg0);
11835	const REAL_VALUE_TYPE *const ra1 = TREE_REAL_CST_PTR (arg1);
11836
11837	if (real_isfinite (ra0) && real_isfinite (ra1))
11838	{
11839	const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type));
11840	const int prec = fmt->p;
11841	const mpfr_rnd_t rnd = fmt->round_towards_zero? MPFR_RNDZ : MPFR_RNDN;
11842	tree result_rem;
11843	long integer_quo;
11844	mpfr_t m0, m1;
11845
11846	mpfr_inits2 (prec, m0, m1, NULL);
11847	mpfr_from_real (m0, ra0, MPFR_RNDN);
11848	mpfr_from_real (m1, ra1, MPFR_RNDN);
11849	mpfr_clear_flags ();
11850	mpfr_remquo (m0, &integer_quo, m0, m1, rnd);
11851	/* Remquo is independent of the rounding mode, so pass
11852	inexact=0 to do_mpfr_ckconv(). */
11853	result_rem = do_mpfr_ckconv (m: m0, type, /*inexact=*/ 0);
11854	mpfr_clears (m0, m1, NULL);
11855	if (result_rem)
11856	{
11857	/* MPFR calculates quo in the host's long so it may
11858	return more bits in quo than the target int can hold
11859	if sizeof(host long) > sizeof(target int). This can
11860	happen even for native compilers in LP64 mode. In
11861	these cases, modulo the quo value with the largest
11862	number that the target int can hold while leaving one
11863	bit for the sign. */
11864	if (sizeof (integer_quo) * CHAR_BIT > INT_TYPE_SIZE)
11865	integer_quo %= (long)(1UL << (INT_TYPE_SIZE - 1));
11866
11867	/* Dereference the quo pointer argument. */
11868	arg_quo = build_fold_indirect_ref (arg_quo);
11869	/* Proceed iff a valid pointer type was passed in. */
11870	if (TYPE_MAIN_VARIANT (TREE_TYPE (arg_quo)) == integer_type_node)
11871	{
11872	/* Set the value. */
11873	tree result_quo
11874	= fold_build2 (MODIFY_EXPR, TREE_TYPE (arg_quo), arg_quo,
11875	build_int_cst (TREE_TYPE (arg_quo),
11876	integer_quo));
11877	TREE_SIDE_EFFECTS (result_quo) = 1;
11878	/* Combine the quo assignment with the rem. */
11879	result = fold_build2 (COMPOUND_EXPR, type,
11880	result_quo, result_rem);
11881	suppress_warning (result, OPT_Wunused_value);
11882	result = non_lvalue (result);
11883	}
11884	}
11885	}
11886	}
11887	return result;
11888	}
11889
11890	/* If ARG is a REAL_CST, call mpfr_lgamma() on it and return the
11891	resulting value as a tree with type TYPE. The mpfr precision is
11892	set to the precision of TYPE. We assume that this mpfr function
11893	returns zero if the result could be calculated exactly within the
11894	requested precision. In addition, the integer pointer represented
11895	by ARG_SG will be dereferenced and set to the appropriate signgam
11896	(-1,1) value. */
11897
11898	static tree
11899	do_mpfr_lgamma_r (tree arg, tree arg_sg, tree type)
11900	{
11901	tree result = NULL_TREE;
11902
11903	STRIP_NOPS (arg);
11904
11905	/* To proceed, MPFR must exactly represent the target floating point
11906	format, which only happens when the target base equals two. Also
11907	verify ARG is a constant and that ARG_SG is an int pointer. */
11908	if (REAL_MODE_FORMAT (TYPE_MODE (type))->b == 2
11909	&& TREE_CODE (arg) == REAL_CST && !TREE_OVERFLOW (arg)
11910	&& TREE_CODE (TREE_TYPE (arg_sg)) == POINTER_TYPE
11911	&& TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (arg_sg))) == integer_type_node)
11912	{
11913	const REAL_VALUE_TYPE *const ra = TREE_REAL_CST_PTR (arg);
11914
11915	/* In addition to NaN and Inf, the argument cannot be zero or a
11916	negative integer. */
11917	if (real_isfinite (ra)
11918	&& ra->cl != rvc_zero
11919	&& !(real_isneg (ra) && real_isinteger (ra, TYPE_MODE (type))))
11920	{
11921	const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type));
11922	const int prec = fmt->p;
11923	const mpfr_rnd_t rnd = fmt->round_towards_zero? MPFR_RNDZ : MPFR_RNDN;
11924	int inexact, sg;
11925	tree result_lg;
11926
11927	auto_mpfr m (prec);
11928	mpfr_from_real (m, ra, MPFR_RNDN);
11929	mpfr_clear_flags ();
11930	inexact = mpfr_lgamma (m, &sg, m, rnd);
11931	result_lg = do_mpfr_ckconv (m, type, inexact);
11932	if (result_lg)
11933	{
11934	tree result_sg;
11935
11936	/* Dereference the arg_sg pointer argument. */
11937	arg_sg = build_fold_indirect_ref (arg_sg);
11938	/* Assign the signgam value into *arg_sg. */
11939	result_sg = fold_build2 (MODIFY_EXPR,
11940	TREE_TYPE (arg_sg), arg_sg,
11941	build_int_cst (TREE_TYPE (arg_sg), sg));
11942	TREE_SIDE_EFFECTS (result_sg) = 1;
11943	/* Combine the signgam assignment with the lgamma result. */
11944	result = non_lvalue (fold_build2 (COMPOUND_EXPR, type,
11945	result_sg, result_lg));
11946	}
11947	}
11948	}
11949
11950	return result;
11951	}
11952
11953	/* If arguments ARG0 and ARG1 are a COMPLEX_CST, call the two-argument
11954	mpc function FUNC on it and return the resulting value as a tree
11955	with type TYPE. The mpfr precision is set to the precision of
11956	TYPE. We assume that function FUNC returns zero if the result
11957	could be calculated exactly within the requested precision. If
11958	DO_NONFINITE is true, then fold expressions containing Inf or NaN
11959	in the arguments and/or results. */
11960
11961	tree
11962	do_mpc_arg2 (tree arg0, tree arg1, tree type, int do_nonfinite,
11963	int (*func)(mpc_ptr, mpc_srcptr, mpc_srcptr, mpc_rnd_t))
11964	{
11965	tree result = NULL_TREE;
11966
11967	STRIP_NOPS (arg0);
11968	STRIP_NOPS (arg1);
11969
11970	/* To proceed, MPFR must exactly represent the target floating point
11971	format, which only happens when the target base equals two. */
11972	if (TREE_CODE (arg0) == COMPLEX_CST && !TREE_OVERFLOW (arg0)
11973	&& SCALAR_FLOAT_TYPE_P (TREE_TYPE (TREE_TYPE (arg0)))
11974	&& TREE_CODE (arg1) == COMPLEX_CST && !TREE_OVERFLOW (arg1)
11975	&& SCALAR_FLOAT_TYPE_P (TREE_TYPE (TREE_TYPE (arg1)))
11976	&& REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0))))->b == 2)
11977	{
11978	const REAL_VALUE_TYPE *const re0 = TREE_REAL_CST_PTR (TREE_REALPART (arg0));
11979	const REAL_VALUE_TYPE *const im0 = TREE_REAL_CST_PTR (TREE_IMAGPART (arg0));
11980	const REAL_VALUE_TYPE *const re1 = TREE_REAL_CST_PTR (TREE_REALPART (arg1));
11981	const REAL_VALUE_TYPE *const im1 = TREE_REAL_CST_PTR (TREE_IMAGPART (arg1));
11982
11983	if (do_nonfinite
11984	|| (real_isfinite (re0) && real_isfinite (im0)
11985	&& real_isfinite (re1) && real_isfinite (im1)))
11986	{
11987	const struct real_format *const fmt =
11988	REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (type)));
11989	const int prec = fmt->p;
11990	const mpfr_rnd_t rnd = fmt->round_towards_zero
11991	? MPFR_RNDZ : MPFR_RNDN;
11992	const mpc_rnd_t crnd = fmt->round_towards_zero ? MPC_RNDZZ : MPC_RNDNN;
11993	int inexact;
11994	mpc_t m0, m1;
11995
11996	mpc_init2 (m0, prec);
11997	mpc_init2 (m1, prec);
11998	mpfr_from_real (mpc_realref (m0), re0, rnd);
11999	mpfr_from_real (mpc_imagref (m0), im0, rnd);
12000	mpfr_from_real (mpc_realref (m1), re1, rnd);
12001	mpfr_from_real (mpc_imagref (m1), im1, rnd);
12002	mpfr_clear_flags ();
12003	inexact = func (m0, m0, m1, crnd);
12004	result = do_mpc_ckconv (m: m0, type, inexact, force_convert: do_nonfinite);
12005	mpc_clear (m0);
12006	mpc_clear (m1);
12007	}
12008	}
12009
12010	return result;
12011	}
12012
12013	/* A wrapper function for builtin folding that prevents warnings for
12014	"statement without effect" and the like, caused by removing the
12015	call node earlier than the warning is generated. */
12016
12017	tree
12018	fold_call_stmt (gcall *stmt, bool ignore)
12019	{
12020	tree ret = NULL_TREE;
12021	tree fndecl = gimple_call_fndecl (gs: stmt);
12022	location_t loc = gimple_location (g: stmt);
12023	if (fndecl && fndecl_built_in_p (node: fndecl)
12024	&& !gimple_call_va_arg_pack_p (s: stmt))
12025	{
12026	int nargs = gimple_call_num_args (gs: stmt);
12027	tree *args = (nargs > 0
12028	? gimple_call_arg_ptr (gs: stmt, index: 0)
12029	: &error_mark_node);
12030
12031	if (avoid_folding_inline_builtin (fndecl))
12032	return NULL_TREE;
12033	if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
12034	{
12035	return targetm.fold_builtin (fndecl, nargs, args, ignore);
12036	}
12037	else
12038	{
12039	ret = fold_builtin_n (loc, NULL_TREE, fndecl, args, nargs, ignore);
12040	if (ret)
12041	{
12042	/* Propagate location information from original call to
12043	expansion of builtin. Otherwise things like
12044	maybe_emit_chk_warning, that operate on the expansion
12045	of a builtin, will use the wrong location information. */
12046	if (gimple_has_location (g: stmt))
12047	{
12048	tree realret = ret;
12049	if (TREE_CODE (ret) == NOP_EXPR)
12050	realret = TREE_OPERAND (ret, 0);
12051	if (CAN_HAVE_LOCATION_P (realret)
12052	&& !EXPR_HAS_LOCATION (realret))
12053	SET_EXPR_LOCATION (realret, loc);
12054	return realret;
12055	}
12056	return ret;
12057	}
12058	}
12059	}
12060	return NULL_TREE;
12061	}
12062
12063	/* Look up the function in builtin_decl that corresponds to DECL
12064	and set ASMSPEC as its user assembler name. DECL must be a
12065	function decl that declares a builtin. */
12066
12067	void
12068	set_builtin_user_assembler_name (tree decl, const char *asmspec)
12069	{
12070	gcc_assert (fndecl_built_in_p (decl, BUILT_IN_NORMAL)
12071	&& asmspec != 0);
12072
12073	tree builtin = builtin_decl_explicit (fncode: DECL_FUNCTION_CODE (decl));
12074	set_user_assembler_name (builtin, asmspec);
12075
12076	if (DECL_FUNCTION_CODE (decl) == BUILT_IN_FFS
12077	&& INT_TYPE_SIZE < BITS_PER_WORD)
12078	{
12079	scalar_int_mode mode = int_mode_for_size (INT_TYPE_SIZE, limit: 0).require ();
12080	set_user_assembler_libfunc ("ffs", asmspec);
12081	set_optab_libfunc (ffs_optab, mode, "ffs");
12082	}
12083	}
12084
12085	/* Return true if DECL is a builtin that expands to a constant or similarly
12086	simple code. */
12087	bool
12088	is_simple_builtin (tree decl)
12089	{
12090	if (decl && fndecl_built_in_p (node: decl, klass: BUILT_IN_NORMAL))
12091	switch (DECL_FUNCTION_CODE (decl))
12092	{
12093	/* Builtins that expand to constants. */
12094	case BUILT_IN_CONSTANT_P:
12095	case BUILT_IN_EXPECT:
12096	case BUILT_IN_OBJECT_SIZE:
12097	case BUILT_IN_UNREACHABLE:
12098	/* Simple register moves or loads from stack. */
12099	case BUILT_IN_ASSUME_ALIGNED:
12100	case BUILT_IN_RETURN_ADDRESS:
12101	case BUILT_IN_EXTRACT_RETURN_ADDR:
12102	case BUILT_IN_FROB_RETURN_ADDR:
12103	case BUILT_IN_RETURN:
12104	case BUILT_IN_AGGREGATE_INCOMING_ADDRESS:
12105	case BUILT_IN_FRAME_ADDRESS:
12106	case BUILT_IN_VA_END:
12107	case BUILT_IN_STACK_SAVE:
12108	case BUILT_IN_STACK_RESTORE:
12109	case BUILT_IN_DWARF_CFA:
12110	/* Exception state returns or moves registers around. */
12111	case BUILT_IN_EH_FILTER:
12112	case BUILT_IN_EH_POINTER:
12113	case BUILT_IN_EH_COPY_VALUES:
12114	return true;
12115
12116	default:
12117	return false;
12118	}
12119
12120	return false;
12121	}
12122
12123	/* Return true if DECL is a builtin that is not expensive, i.e., they are
12124	most probably expanded inline into reasonably simple code. This is a
12125	superset of is_simple_builtin. */
12126	bool
12127	is_inexpensive_builtin (tree decl)
12128	{
12129	if (!decl)
12130	return false;
12131	else if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_MD)
12132	return true;
12133	else if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
12134	switch (DECL_FUNCTION_CODE (decl))
12135	{
12136	case BUILT_IN_ABS:
12137	CASE_BUILT_IN_ALLOCA:
12138	case BUILT_IN_BSWAP16:
12139	case BUILT_IN_BSWAP32:
12140	case BUILT_IN_BSWAP64:
12141	case BUILT_IN_BSWAP128:
12142	case BUILT_IN_CLZ:
12143	case BUILT_IN_CLZIMAX:
12144	case BUILT_IN_CLZL:
12145	case BUILT_IN_CLZLL:
12146	case BUILT_IN_CTZ:
12147	case BUILT_IN_CTZIMAX:
12148	case BUILT_IN_CTZL:
12149	case BUILT_IN_CTZLL:
12150	case BUILT_IN_FFS:
12151	case BUILT_IN_FFSIMAX:
12152	case BUILT_IN_FFSL:
12153	case BUILT_IN_FFSLL:
12154	case BUILT_IN_IMAXABS:
12155	case BUILT_IN_FINITE:
12156	case BUILT_IN_FINITEF:
12157	case BUILT_IN_FINITEL:
12158	case BUILT_IN_FINITED32:
12159	case BUILT_IN_FINITED64:
12160	case BUILT_IN_FINITED128:
12161	case BUILT_IN_FPCLASSIFY:
12162	case BUILT_IN_ISFINITE:
12163	case BUILT_IN_ISINF_SIGN:
12164	case BUILT_IN_ISINF:
12165	case BUILT_IN_ISINFF:
12166	case BUILT_IN_ISINFL:
12167	case BUILT_IN_ISINFD32:
12168	case BUILT_IN_ISINFD64:
12169	case BUILT_IN_ISINFD128:
12170	case BUILT_IN_ISNAN:
12171	case BUILT_IN_ISNANF:
12172	case BUILT_IN_ISNANL:
12173	case BUILT_IN_ISNAND32:
12174	case BUILT_IN_ISNAND64:
12175	case BUILT_IN_ISNAND128:
12176	case BUILT_IN_ISNORMAL:
12177	case BUILT_IN_ISGREATER:
12178	case BUILT_IN_ISGREATEREQUAL:
12179	case BUILT_IN_ISLESS:
12180	case BUILT_IN_ISLESSEQUAL:
12181	case BUILT_IN_ISLESSGREATER:
12182	case BUILT_IN_ISUNORDERED:
12183	case BUILT_IN_ISEQSIG:
12184	case BUILT_IN_VA_ARG_PACK:
12185	case BUILT_IN_VA_ARG_PACK_LEN:
12186	case BUILT_IN_VA_COPY:
12187	case BUILT_IN_TRAP:
12188	case BUILT_IN_UNREACHABLE_TRAP:
12189	case BUILT_IN_SAVEREGS:
12190	case BUILT_IN_POPCOUNTL:
12191	case BUILT_IN_POPCOUNTLL:
12192	case BUILT_IN_POPCOUNTIMAX:
12193	case BUILT_IN_POPCOUNT:
12194	case BUILT_IN_PARITYL:
12195	case BUILT_IN_PARITYLL:
12196	case BUILT_IN_PARITYIMAX:
12197	case BUILT_IN_PARITY:
12198	case BUILT_IN_LABS:
12199	case BUILT_IN_LLABS:
12200	case BUILT_IN_PREFETCH:
12201	case BUILT_IN_ACC_ON_DEVICE:
12202	return true;
12203
12204	default:
12205	return is_simple_builtin (decl);
12206	}
12207
12208	return false;
12209	}
12210
12211	/* Return true if T is a constant and the value cast to a target char
12212	can be represented by a host char.
12213	Store the casted char constant in *P if so. */
12214
12215	bool
12216	target_char_cst_p (tree t, char *p)
12217	{
12218	if (!tree_fits_uhwi_p (t) || CHAR_TYPE_SIZE != HOST_BITS_PER_CHAR)
12219	return false;
12220
12221	*p = (char)tree_to_uhwi (t);
12222	return true;
12223	}
12224
12225	/* Return true if the builtin DECL is implemented in a standard library.
12226	Otherwise return false which doesn't guarantee it is not (thus the list
12227	of handled builtins below may be incomplete). */
12228
12229	bool
12230	builtin_with_linkage_p (tree decl)
12231	{
12232	if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
12233	switch (DECL_FUNCTION_CODE (decl))
12234	{
12235	CASE_FLT_FN (BUILT_IN_ACOS):
12236	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ACOS):
12237	CASE_FLT_FN (BUILT_IN_ACOSH):
12238	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ACOSH):
12239	CASE_FLT_FN (BUILT_IN_ASIN):
12240	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ASIN):
12241	CASE_FLT_FN (BUILT_IN_ASINH):
12242	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ASINH):
12243	CASE_FLT_FN (BUILT_IN_ATAN):
12244	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATAN):
12245	CASE_FLT_FN (BUILT_IN_ATANH):
12246	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATANH):
12247	CASE_FLT_FN (BUILT_IN_ATAN2):
12248	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATAN2):
12249	CASE_FLT_FN (BUILT_IN_CBRT):
12250	CASE_FLT_FN_FLOATN_NX (BUILT_IN_CBRT):
12251	CASE_FLT_FN (BUILT_IN_CEIL):
12252	CASE_FLT_FN_FLOATN_NX (BUILT_IN_CEIL):
12253	CASE_FLT_FN (BUILT_IN_COPYSIGN):
12254	CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
12255	CASE_FLT_FN (BUILT_IN_COS):
12256	CASE_FLT_FN_FLOATN_NX (BUILT_IN_COS):
12257	CASE_FLT_FN (BUILT_IN_COSH):
12258	CASE_FLT_FN_FLOATN_NX (BUILT_IN_COSH):
12259	CASE_FLT_FN (BUILT_IN_ERF):
12260	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ERF):
12261	CASE_FLT_FN (BUILT_IN_ERFC):
12262	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ERFC):
12263	CASE_FLT_FN (BUILT_IN_EXP):
12264	CASE_FLT_FN_FLOATN_NX (BUILT_IN_EXP):
12265	CASE_FLT_FN (BUILT_IN_EXP2):
12266	CASE_FLT_FN_FLOATN_NX (BUILT_IN_EXP2):
12267	CASE_FLT_FN (BUILT_IN_EXPM1):
12268	CASE_FLT_FN_FLOATN_NX (BUILT_IN_EXPM1):
12269	CASE_FLT_FN (BUILT_IN_FABS):
12270	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
12271	CASE_FLT_FN (BUILT_IN_FDIM):
12272	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FDIM):
12273	CASE_FLT_FN (BUILT_IN_FLOOR):
12274	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FLOOR):
12275	CASE_FLT_FN (BUILT_IN_FMA):
12276	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
12277	CASE_FLT_FN (BUILT_IN_FMAX):
12278	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMAX):
12279	CASE_FLT_FN (BUILT_IN_FMIN):
12280	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMIN):
12281	CASE_FLT_FN (BUILT_IN_FMOD):
12282	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMOD):
12283	CASE_FLT_FN (BUILT_IN_FREXP):
12284	CASE_FLT_FN_FLOATN_NX (BUILT_IN_FREXP):
12285	CASE_FLT_FN (BUILT_IN_HYPOT):
12286	CASE_FLT_FN_FLOATN_NX (BUILT_IN_HYPOT):
12287	CASE_FLT_FN (BUILT_IN_ILOGB):
12288	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ILOGB):
12289	CASE_FLT_FN (BUILT_IN_LDEXP):
12290	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LDEXP):
12291	CASE_FLT_FN (BUILT_IN_LGAMMA):
12292	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LGAMMA):
12293	CASE_FLT_FN (BUILT_IN_LLRINT):
12294	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LLRINT):
12295	CASE_FLT_FN (BUILT_IN_LLROUND):
12296	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LLROUND):
12297	CASE_FLT_FN (BUILT_IN_LOG):
12298	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG):
12299	CASE_FLT_FN (BUILT_IN_LOG10):
12300	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG10):
12301	CASE_FLT_FN (BUILT_IN_LOG1P):
12302	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG1P):
12303	CASE_FLT_FN (BUILT_IN_LOG2):
12304	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG2):
12305	CASE_FLT_FN (BUILT_IN_LOGB):
12306	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOGB):
12307	CASE_FLT_FN (BUILT_IN_LRINT):
12308	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LRINT):
12309	CASE_FLT_FN (BUILT_IN_LROUND):
12310	CASE_FLT_FN_FLOATN_NX (BUILT_IN_LROUND):
12311	CASE_FLT_FN (BUILT_IN_MODF):
12312	CASE_FLT_FN_FLOATN_NX (BUILT_IN_MODF):
12313	CASE_FLT_FN (BUILT_IN_NAN):
12314	CASE_FLT_FN_FLOATN_NX (BUILT_IN_NAN):
12315	CASE_FLT_FN (BUILT_IN_NEARBYINT):
12316	CASE_FLT_FN_FLOATN_NX (BUILT_IN_NEARBYINT):
12317	CASE_FLT_FN (BUILT_IN_NEXTAFTER):
12318	CASE_FLT_FN_FLOATN_NX (BUILT_IN_NEXTAFTER):
12319	CASE_FLT_FN (BUILT_IN_NEXTTOWARD):
12320	CASE_FLT_FN (BUILT_IN_POW):
12321	CASE_FLT_FN_FLOATN_NX (BUILT_IN_POW):
12322	CASE_FLT_FN (BUILT_IN_REMAINDER):
12323	CASE_FLT_FN_FLOATN_NX (BUILT_IN_REMAINDER):
12324	CASE_FLT_FN (BUILT_IN_REMQUO):
12325	CASE_FLT_FN_FLOATN_NX (BUILT_IN_REMQUO):
12326	CASE_FLT_FN (BUILT_IN_RINT):
12327	CASE_FLT_FN_FLOATN_NX (BUILT_IN_RINT):
12328	CASE_FLT_FN (BUILT_IN_ROUND):
12329	CASE_FLT_FN_FLOATN_NX (BUILT_IN_ROUND):
12330	CASE_FLT_FN (BUILT_IN_SCALBLN):
12331	CASE_FLT_FN_FLOATN_NX (BUILT_IN_SCALBLN):
12332	CASE_FLT_FN (BUILT_IN_SCALBN):
12333	CASE_FLT_FN_FLOATN_NX (BUILT_IN_SCALBN):
12334	CASE_FLT_FN (BUILT_IN_SIN):
12335	CASE_FLT_FN_FLOATN_NX (BUILT_IN_SIN):
12336	CASE_FLT_FN (BUILT_IN_SINH):
12337	CASE_FLT_FN_FLOATN_NX (BUILT_IN_SINH):
12338	CASE_FLT_FN (BUILT_IN_SINCOS):
12339	CASE_FLT_FN (BUILT_IN_SQRT):
12340	CASE_FLT_FN_FLOATN_NX (BUILT_IN_SQRT):
12341	CASE_FLT_FN (BUILT_IN_TAN):
12342	CASE_FLT_FN_FLOATN_NX (BUILT_IN_TAN):
12343	CASE_FLT_FN (BUILT_IN_TANH):
12344	CASE_FLT_FN_FLOATN_NX (BUILT_IN_TANH):
12345	CASE_FLT_FN (BUILT_IN_TGAMMA):
12346	CASE_FLT_FN_FLOATN_NX (BUILT_IN_TGAMMA):
12347	CASE_FLT_FN (BUILT_IN_TRUNC):
12348	CASE_FLT_FN_FLOATN_NX (BUILT_IN_TRUNC):
12349	return true;
12350
12351	case BUILT_IN_STPCPY:
12352	case BUILT_IN_STPNCPY:
12353	/* stpcpy is both referenced in libiberty's pex-win32.c and provided
12354	by libiberty's stpcpy.c for MinGW targets so we need to return true
12355	in order to be able to build libiberty in LTO mode for them. */
12356	return true;
12357
12358	default:
12359	break;
12360	}
12361	return false;
12362	}
12363
12364	/* Return true if OFFRNG is bounded to a subrange of offset values
12365	valid for the largest possible object. */
12366
12367	bool
12368	access_ref::offset_bounded () const
12369	{
12370	tree min = TYPE_MIN_VALUE (ptrdiff_type_node);
12371	tree max = TYPE_MAX_VALUE (ptrdiff_type_node);
12372	return wi::to_offset (t: min) <= offrng[0] && offrng[1] <= wi::to_offset (t: max);
12373	}
12374
12375	/* If CALLEE has known side effects, fill in INFO and return true.
12376	See tree-ssa-structalias.cc:find_func_aliases
12377	for the list of builtins we might need to handle here. */
12378
12379	attr_fnspec
12380	builtin_fnspec (tree callee)
12381	{
12382	built_in_function code = DECL_FUNCTION_CODE (decl: callee);
12383
12384	switch (code)
12385	{
12386	/* All the following functions read memory pointed to by
12387	their second argument and write memory pointed to by first
12388	argument.
12389	strcat/strncat additionally reads memory pointed to by the first
12390	argument. */
12391	case BUILT_IN_STRCAT:
12392	case BUILT_IN_STRCAT_CHK:
12393	return "1cW 1 ";
12394	case BUILT_IN_STRNCAT:
12395	case BUILT_IN_STRNCAT_CHK:
12396	return "1cW 13";
12397	case BUILT_IN_STRCPY:
12398	case BUILT_IN_STRCPY_CHK:
12399	return "1cO 1 ";
12400	case BUILT_IN_STPCPY:
12401	case BUILT_IN_STPCPY_CHK:
12402	return ".cO 1 ";
12403	case BUILT_IN_STRNCPY:
12404	case BUILT_IN_MEMCPY:
12405	case BUILT_IN_MEMMOVE:
12406	case BUILT_IN_TM_MEMCPY:
12407	case BUILT_IN_TM_MEMMOVE:
12408	case BUILT_IN_STRNCPY_CHK:
12409	case BUILT_IN_MEMCPY_CHK:
12410	case BUILT_IN_MEMMOVE_CHK:
12411	return "1cO313";
12412	case BUILT_IN_MEMPCPY:
12413	case BUILT_IN_MEMPCPY_CHK:
12414	return ".cO313";
12415	case BUILT_IN_STPNCPY:
12416	case BUILT_IN_STPNCPY_CHK:
12417	return ".cO313";
12418	case BUILT_IN_BCOPY:
12419	return ".c23O3";
12420	case BUILT_IN_BZERO:
12421	return ".cO2";
12422	case BUILT_IN_MEMCMP:
12423	case BUILT_IN_MEMCMP_EQ:
12424	case BUILT_IN_BCMP:
12425	case BUILT_IN_STRNCMP:
12426	case BUILT_IN_STRNCMP_EQ:
12427	case BUILT_IN_STRNCASECMP:
12428	return ".cR3R3";
12429
12430	/* The following functions read memory pointed to by their
12431	first argument. */
12432	CASE_BUILT_IN_TM_LOAD (1):
12433	CASE_BUILT_IN_TM_LOAD (2):
12434	CASE_BUILT_IN_TM_LOAD (4):
12435	CASE_BUILT_IN_TM_LOAD (8):
12436	CASE_BUILT_IN_TM_LOAD (FLOAT):
12437	CASE_BUILT_IN_TM_LOAD (DOUBLE):
12438	CASE_BUILT_IN_TM_LOAD (LDOUBLE):
12439	CASE_BUILT_IN_TM_LOAD (M64):
12440	CASE_BUILT_IN_TM_LOAD (M128):
12441	CASE_BUILT_IN_TM_LOAD (M256):
12442	case BUILT_IN_TM_LOG:
12443	case BUILT_IN_TM_LOG_1:
12444	case BUILT_IN_TM_LOG_2:
12445	case BUILT_IN_TM_LOG_4:
12446	case BUILT_IN_TM_LOG_8:
12447	case BUILT_IN_TM_LOG_FLOAT:
12448	case BUILT_IN_TM_LOG_DOUBLE:
12449	case BUILT_IN_TM_LOG_LDOUBLE:
12450	case BUILT_IN_TM_LOG_M64:
12451	case BUILT_IN_TM_LOG_M128:
12452	case BUILT_IN_TM_LOG_M256:
12453	return ".cR ";
12454
12455	case BUILT_IN_INDEX:
12456	case BUILT_IN_RINDEX:
12457	case BUILT_IN_STRCHR:
12458	case BUILT_IN_STRLEN:
12459	case BUILT_IN_STRRCHR:
12460	return ".cR ";
12461	case BUILT_IN_STRNLEN:
12462	return ".cR2";
12463
12464	/* These read memory pointed to by the first argument.
12465	Allocating memory does not have any side-effects apart from
12466	being the definition point for the pointer.
12467	Unix98 specifies that errno is set on allocation failure. */
12468	case BUILT_IN_STRDUP:
12469	return "mCR ";
12470	case BUILT_IN_STRNDUP:
12471	return "mCR2";
12472	/* Allocating memory does not have any side-effects apart from
12473	being the definition point for the pointer. */
12474	case BUILT_IN_MALLOC:
12475	case BUILT_IN_ALIGNED_ALLOC:
12476	case BUILT_IN_CALLOC:
12477	case BUILT_IN_GOMP_ALLOC:
12478	return "mC";
12479	CASE_BUILT_IN_ALLOCA:
12480	return "mc";
12481	/* These read memory pointed to by the first argument with size
12482	in the third argument. */
12483	case BUILT_IN_MEMCHR:
12484	return ".cR3";
12485	/* These read memory pointed to by the first and second arguments. */
12486	case BUILT_IN_STRSTR:
12487	case BUILT_IN_STRPBRK:
12488	case BUILT_IN_STRCASECMP:
12489	case BUILT_IN_STRCSPN:
12490	case BUILT_IN_STRSPN:
12491	case BUILT_IN_STRCMP:
12492	case BUILT_IN_STRCMP_EQ:
12493	return ".cR R ";
12494	/* Freeing memory kills the pointed-to memory. More importantly
12495	the call has to serve as a barrier for moving loads and stores
12496	across it. */
12497	case BUILT_IN_STACK_RESTORE:
12498	case BUILT_IN_FREE:
12499	case BUILT_IN_GOMP_FREE:
12500	return ".co ";
12501	case BUILT_IN_VA_END:
12502	return ".cO ";
12503	/* Realloc serves both as allocation point and deallocation point. */
12504	case BUILT_IN_REALLOC:
12505	case BUILT_IN_GOMP_REALLOC:
12506	return ".Cw ";
12507	case BUILT_IN_GAMMA_R:
12508	case BUILT_IN_GAMMAF_R:
12509	case BUILT_IN_GAMMAL_R:
12510	case BUILT_IN_LGAMMA_R:
12511	case BUILT_IN_LGAMMAF_R:
12512	case BUILT_IN_LGAMMAL_R:
12513	return ".C. Ot";
12514	case BUILT_IN_FREXP:
12515	case BUILT_IN_FREXPF:
12516	case BUILT_IN_FREXPL:
12517	case BUILT_IN_MODF:
12518	case BUILT_IN_MODFF:
12519	case BUILT_IN_MODFL:
12520	return ".c. Ot";
12521	case BUILT_IN_REMQUO:
12522	case BUILT_IN_REMQUOF:
12523	case BUILT_IN_REMQUOL:
12524	return ".c. . Ot";
12525	case BUILT_IN_SINCOS:
12526	case BUILT_IN_SINCOSF:
12527	case BUILT_IN_SINCOSL:
12528	return ".c. OtOt";
12529	case BUILT_IN_MEMSET:
12530	case BUILT_IN_MEMSET_CHK:
12531	case BUILT_IN_TM_MEMSET:
12532	return "1cO3";
12533	CASE_BUILT_IN_TM_STORE (1):
12534	CASE_BUILT_IN_TM_STORE (2):
12535	CASE_BUILT_IN_TM_STORE (4):
12536	CASE_BUILT_IN_TM_STORE (8):
12537	CASE_BUILT_IN_TM_STORE (FLOAT):
12538	CASE_BUILT_IN_TM_STORE (DOUBLE):
12539	CASE_BUILT_IN_TM_STORE (LDOUBLE):
12540	CASE_BUILT_IN_TM_STORE (M64):
12541	CASE_BUILT_IN_TM_STORE (M128):
12542	CASE_BUILT_IN_TM_STORE (M256):
12543	return ".cO ";
12544	case BUILT_IN_STACK_SAVE:
12545	case BUILT_IN_RETURN:
12546	case BUILT_IN_EH_POINTER:
12547	case BUILT_IN_EH_FILTER:
12548	case BUILT_IN_UNWIND_RESUME:
12549	case BUILT_IN_CXA_END_CLEANUP:
12550	case BUILT_IN_EH_COPY_VALUES:
12551	case BUILT_IN_FRAME_ADDRESS:
12552	case BUILT_IN_APPLY_ARGS:
12553	case BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT:
12554	case BUILT_IN_ASAN_AFTER_DYNAMIC_INIT:
12555	case BUILT_IN_PREFETCH:
12556	case BUILT_IN_DWARF_CFA:
12557	case BUILT_IN_RETURN_ADDRESS:
12558	return ".c";
12559	case BUILT_IN_ASSUME_ALIGNED:
12560	case BUILT_IN_EXPECT:
12561	case BUILT_IN_EXPECT_WITH_PROBABILITY:
12562	return "1cX ";
12563	/* But posix_memalign stores a pointer into the memory pointed to
12564	by its first argument. */
12565	case BUILT_IN_POSIX_MEMALIGN:
12566	return ".cOt";
12567
12568	default:
12569	return "";
12570	}
12571	}
12572