1	/* Internal functions.
2	Copyright (C) 2011-2023 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it under
7	the terms of the GNU General Public License as published by the Free
8	Software Foundation; either version 3, or (at your option) any later
9	version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14	for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#include "system.h"
22	#include "coretypes.h"
23	#include "backend.h"
24	#include "target.h"
25	#include "rtl.h"
26	#include "tree.h"
27	#include "gimple.h"
28	#include "predict.h"
29	#include "stringpool.h"
30	#include "tree-vrp.h"
31	#include "tree-ssanames.h"
32	#include "expmed.h"
33	#include "memmodel.h"
34	#include "optabs.h"
35	#include "emit-rtl.h"
36	#include "diagnostic-core.h"
37	#include "fold-const.h"
38	#include "internal-fn.h"
39	#include "stor-layout.h"
40	#include "dojump.h"
41	#include "expr.h"
42	#include "stringpool.h"
43	#include "attribs.h"
44	#include "asan.h"
45	#include "ubsan.h"
46	#include "recog.h"
47	#include "builtins.h"
48	#include "optabs-tree.h"
49	#include "gimple-ssa.h"
50	#include "tree-phinodes.h"
51	#include "ssa-iterators.h"
52	#include "explow.h"
53	#include "rtl-iter.h"
54	#include "gimple-range.h"
55
56	/* For lang_hooks.types.type_for_mode. */
57	#include "langhooks.h"
58
59	/* The names of each internal function, indexed by function number. */
60	const char *const internal_fn_name_array[] = {
61	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
62	#include "internal-fn.def"
63	"<invalid-fn>"
64	};
65
66	/* The ECF_* flags of each internal function, indexed by function number. */
67	const int internal_fn_flags_array[] = {
68	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
69	#include "internal-fn.def"
70	0
71	};
72
73	/* Return the internal function called NAME, or IFN_LAST if there's
74	no such function. */
75
76	internal_fn
77	lookup_internal_fn (const char *name)
78	{
79	typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
80	static name_to_fn_map_type *name_to_fn_map;
81
82	if (!name_to_fn_map)
83	{
84	name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
85	for (unsigned int i = 0; i < IFN_LAST; ++i)
86	name_to_fn_map->put (k: internal_fn_name (fn: internal_fn (i)),
87	v: internal_fn (i));
88	}
89	internal_fn *entry = name_to_fn_map->get (k: name);
90	return entry ? *entry : IFN_LAST;
91	}
92
93	/* Geven an internal_fn IFN that is a widening function, return its
94	corresponding LO and HI internal_fns. */
95
96	extern void
97	lookup_hilo_internal_fn (internal_fn ifn, internal_fn *lo, internal_fn *hi)
98	{
99	gcc_assert (widening_fn_p (ifn));
100
101	switch (ifn)
102	{
103	default:
104	gcc_unreachable ();
105	#undef DEF_INTERNAL_FN
106	#undef DEF_INTERNAL_WIDENING_OPTAB_FN
107	#define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
108	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
109	case IFN_##NAME: \
110	*lo = internal_fn (IFN_##NAME##_LO); \
111	*hi = internal_fn (IFN_##NAME##_HI); \
112	break;
113	#include "internal-fn.def"
114	#undef DEF_INTERNAL_FN
115	#undef DEF_INTERNAL_WIDENING_OPTAB_FN
116	}
117	}
118
119	/* Given an internal_fn IFN that is a widening function, return its
120	corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
121
122	extern void
123	lookup_evenodd_internal_fn (internal_fn ifn, internal_fn *even,
124	internal_fn *odd)
125	{
126	gcc_assert (widening_fn_p (ifn));
127
128	switch (ifn)
129	{
130	default:
131	gcc_unreachable ();
132	#undef DEF_INTERNAL_FN
133	#undef DEF_INTERNAL_WIDENING_OPTAB_FN
134	#define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
135	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
136	case IFN_##NAME: \
137	*even = internal_fn (IFN_##NAME##_EVEN); \
138	*odd = internal_fn (IFN_##NAME##_ODD); \
139	break;
140	#include "internal-fn.def"
141	#undef DEF_INTERNAL_FN
142	#undef DEF_INTERNAL_WIDENING_OPTAB_FN
143	}
144	}
145
146
147	/* Fnspec of each internal function, indexed by function number. */
148	const_tree internal_fn_fnspec_array[IFN_LAST + 1];
149
150	void
151	init_internal_fns ()
152	{
153	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
154	if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
155	build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
156	#include "internal-fn.def"
157	internal_fn_fnspec_array[IFN_LAST] = 0;
158	}
159
160	/* Create static initializers for the information returned by
161	direct_internal_fn. */
162	#define not_direct { -2, -2, false }
163	#define mask_load_direct { -1, 2, false }
164	#define load_lanes_direct { -1, -1, false }
165	#define mask_load_lanes_direct { -1, -1, false }
166	#define gather_load_direct { 3, 1, false }
167	#define len_load_direct { -1, -1, false }
168	#define mask_len_load_direct { -1, 4, false }
169	#define mask_store_direct { 3, 2, false }
170	#define store_lanes_direct { 0, 0, false }
171	#define mask_store_lanes_direct { 0, 0, false }
172	#define vec_cond_mask_direct { 1, 0, false }
173	#define vec_cond_mask_len_direct { 1, 1, false }
174	#define vec_cond_direct { 2, 0, false }
175	#define scatter_store_direct { 3, 1, false }
176	#define len_store_direct { 3, 3, false }
177	#define mask_len_store_direct { 4, 5, false }
178	#define vec_set_direct { 3, 3, false }
179	#define vec_extract_direct { 0, -1, false }
180	#define unary_direct { 0, 0, true }
181	#define unary_convert_direct { -1, 0, true }
182	#define binary_direct { 0, 0, true }
183	#define ternary_direct { 0, 0, true }
184	#define cond_unary_direct { 1, 1, true }
185	#define cond_binary_direct { 1, 1, true }
186	#define cond_ternary_direct { 1, 1, true }
187	#define cond_len_unary_direct { 1, 1, true }
188	#define cond_len_binary_direct { 1, 1, true }
189	#define cond_len_ternary_direct { 1, 1, true }
190	#define while_direct { 0, 2, false }
191	#define fold_extract_direct { 2, 2, false }
192	#define fold_len_extract_direct { 2, 2, false }
193	#define fold_left_direct { 1, 1, false }
194	#define mask_fold_left_direct { 1, 1, false }
195	#define mask_len_fold_left_direct { 1, 1, false }
196	#define check_ptrs_direct { 0, 0, false }
197
198	const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
199	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
200	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
201	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
202	UNSIGNED_OPTAB, TYPE) TYPE##_direct,
203	#include "internal-fn.def"
204	not_direct
205	};
206
207	/* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
208	output operands and NINPUTS input operands, where NOUTPUTS is either
209	0 or 1. The output operand (if any) comes first, followed by the
210	NINPUTS input operands. */
211
212	static void
213	expand_fn_using_insn (gcall *stmt, insn_code icode, unsigned int noutputs,
214	unsigned int ninputs)
215	{
216	gcc_assert (icode != CODE_FOR_nothing);
217
218	expand_operand *ops = XALLOCAVEC (expand_operand, noutputs + ninputs);
219	unsigned int opno = 0;
220	rtx lhs_rtx = NULL_RTX;
221	tree lhs = gimple_call_lhs (gs: stmt);
222
223	if (noutputs)
224	{
225	gcc_assert (noutputs == 1);
226	if (lhs)
227	lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
228
229	/* Do not assign directly to a promoted subreg, since there is no
230	guarantee that the instruction will leave the upper bits of the
231	register in the state required by SUBREG_PROMOTED_SIGN. */
232	rtx dest = lhs_rtx;
233	if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
234	dest = NULL_RTX;
235	create_output_operand (op: &ops[opno], x: dest,
236	mode: insn_data[icode].operand[opno].mode);
237	opno += 1;
238	}
239	else
240	gcc_assert (!lhs);
241
242	for (unsigned int i = 0; i < ninputs; ++i)
243	{
244	tree rhs = gimple_call_arg (gs: stmt, index: i);
245	tree rhs_type = TREE_TYPE (rhs);
246	rtx rhs_rtx = expand_normal (exp: rhs);
247	if (INTEGRAL_TYPE_P (rhs_type))
248	create_convert_operand_from (op: &ops[opno], value: rhs_rtx,
249	TYPE_MODE (rhs_type),
250	TYPE_UNSIGNED (rhs_type));
251	else if (TREE_CODE (rhs) == SSA_NAME
252	&& SSA_NAME_IS_DEFAULT_DEF (rhs)
253	&& VAR_P (SSA_NAME_VAR (rhs)))
254	create_undefined_input_operand (op: &ops[opno], TYPE_MODE (rhs_type));
255	else
256	create_input_operand (op: &ops[opno], value: rhs_rtx, TYPE_MODE (rhs_type));
257	opno += 1;
258	}
259
260	gcc_assert (opno == noutputs + ninputs);
261	expand_insn (icode, nops: opno, ops);
262	if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
263	{
264	/* If the return value has an integral type, convert the instruction
265	result to that type. This is useful for things that return an
266	int regardless of the size of the input. If the instruction result
267	is smaller than required, assume that it is signed.
268
269	If the return value has a nonintegral type, its mode must match
270	the instruction result. */
271	if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
272	{
273	/* If this is a scalar in a register that is stored in a wider
274	mode than the declared mode, compute the result into its
275	declared mode and then convert to the wider mode. */
276	gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
277	rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
278	convert_move (SUBREG_REG (lhs_rtx), tmp,
279	SUBREG_PROMOTED_SIGN (lhs_rtx));
280	}
281	else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
282	emit_move_insn (lhs_rtx, ops[0].value);
283	else
284	{
285	gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
286	convert_move (lhs_rtx, ops[0].value, 0);
287	}
288	}
289	}
290
291	/* ARRAY_TYPE is an array of vector modes. Return the associated insn
292	for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
293
294	static enum insn_code
295	get_multi_vector_move (tree array_type, convert_optab optab)
296	{
297	machine_mode imode;
298	machine_mode vmode;
299
300	gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
301	imode = TYPE_MODE (array_type);
302	vmode = TYPE_MODE (TREE_TYPE (array_type));
303
304	return convert_optab_handler (op: optab, to_mode: imode, from_mode: vmode);
305	}
306
307	/* Add mask and len arguments according to the STMT. */
308
309	static unsigned int
310	add_mask_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
311	{
312	internal_fn ifn = gimple_call_internal_fn (gs: stmt);
313	int len_index = internal_fn_len_index (ifn);
314	/* BIAS is always consecutive next of LEN. */
315	int bias_index = len_index + 1;
316	int mask_index = internal_fn_mask_index (ifn);
317	/* The order of arguments are always {len,bias,mask}. */
318	if (mask_index >= 0)
319	{
320	tree mask = gimple_call_arg (gs: stmt, index: mask_index);
321	rtx mask_rtx = expand_normal (exp: mask);
322	create_input_operand (op: &ops[opno++], value: mask_rtx,
323	TYPE_MODE (TREE_TYPE (mask)));
324	}
325	if (len_index >= 0)
326	{
327	tree len = gimple_call_arg (gs: stmt, index: len_index);
328	rtx len_rtx = expand_normal (exp: len);
329	create_convert_operand_from (op: &ops[opno++], value: len_rtx,
330	TYPE_MODE (TREE_TYPE (len)),
331	TYPE_UNSIGNED (TREE_TYPE (len)));
332	tree biast = gimple_call_arg (gs: stmt, index: bias_index);
333	rtx bias = expand_normal (exp: biast);
334	create_input_operand (op: &ops[opno++], value: bias, QImode);
335	}
336	return opno;
337	}
338
339	/* Expand LOAD_LANES call STMT using optab OPTAB. */
340
341	static void
342	expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
343	{
344	class expand_operand ops[2];
345	tree type, lhs, rhs;
346	rtx target, mem;
347
348	lhs = gimple_call_lhs (gs: stmt);
349	rhs = gimple_call_arg (gs: stmt, index: 0);
350	type = TREE_TYPE (lhs);
351
352	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
353	mem = expand_normal (exp: rhs);
354
355	gcc_assert (MEM_P (mem));
356	PUT_MODE (x: mem, TYPE_MODE (type));
357
358	create_output_operand (op: &ops[0], x: target, TYPE_MODE (type));
359	create_fixed_operand (op: &ops[1], x: mem);
360	expand_insn (icode: get_multi_vector_move (array_type: type, optab), nops: 2, ops);
361	if (!rtx_equal_p (target, ops[0].value))
362	emit_move_insn (target, ops[0].value);
363	}
364
365	/* Expand STORE_LANES call STMT using optab OPTAB. */
366
367	static void
368	expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
369	{
370	class expand_operand ops[2];
371	tree type, lhs, rhs;
372	rtx target, reg;
373
374	lhs = gimple_call_lhs (gs: stmt);
375	rhs = gimple_call_arg (gs: stmt, index: 0);
376	type = TREE_TYPE (rhs);
377
378	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
379	reg = expand_normal (exp: rhs);
380
381	gcc_assert (MEM_P (target));
382	PUT_MODE (x: target, TYPE_MODE (type));
383
384	create_fixed_operand (op: &ops[0], x: target);
385	create_input_operand (op: &ops[1], value: reg, TYPE_MODE (type));
386	expand_insn (icode: get_multi_vector_move (array_type: type, optab), nops: 2, ops);
387	}
388
389	static void
390	expand_ANNOTATE (internal_fn, gcall *)
391	{
392	gcc_unreachable ();
393	}
394
395	/* This should get expanded in omp_device_lower pass. */
396
397	static void
398	expand_GOMP_USE_SIMT (internal_fn, gcall *)
399	{
400	gcc_unreachable ();
401	}
402
403	/* This should get expanded in omp_device_lower pass. */
404
405	static void
406	expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
407	{
408	gcc_unreachable ();
409	}
410
411	/* Allocate per-lane storage and begin non-uniform execution region. */
412
413	static void
414	expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
415	{
416	rtx target;
417	tree lhs = gimple_call_lhs (gs: stmt);
418	if (lhs)
419	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
420	else
421	target = gen_reg_rtx (Pmode);
422	rtx size = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
423	rtx align = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
424	class expand_operand ops[3];
425	create_output_operand (op: &ops[0], x: target, Pmode);
426	create_input_operand (op: &ops[1], value: size, Pmode);
427	create_input_operand (op: &ops[2], value: align, Pmode);
428	gcc_assert (targetm.have_omp_simt_enter ());
429	expand_insn (icode: targetm.code_for_omp_simt_enter, nops: 3, ops);
430	if (!rtx_equal_p (target, ops[0].value))
431	emit_move_insn (target, ops[0].value);
432	}
433
434	/* Deallocate per-lane storage and leave non-uniform execution region. */
435
436	static void
437	expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
438	{
439	gcc_checking_assert (!gimple_call_lhs (stmt));
440	rtx arg = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
441	class expand_operand ops[1];
442	create_input_operand (op: &ops[0], value: arg, Pmode);
443	gcc_assert (targetm.have_omp_simt_exit ());
444	expand_insn (icode: targetm.code_for_omp_simt_exit, nops: 1, ops);
445	}
446
447	/* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
448	without SIMT execution this should be expanded in omp_device_lower pass. */
449
450	static void
451	expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
452	{
453	tree lhs = gimple_call_lhs (gs: stmt);
454	if (!lhs)
455	return;
456
457	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
458	gcc_assert (targetm.have_omp_simt_lane ());
459	emit_insn (targetm.gen_omp_simt_lane (target));
460	}
461
462	/* This should get expanded in omp_device_lower pass. */
463
464	static void
465	expand_GOMP_SIMT_VF (internal_fn, gcall *)
466	{
467	gcc_unreachable ();
468	}
469
470	/* This should get expanded in omp_device_lower pass. */
471
472	static void
473	expand_GOMP_TARGET_REV (internal_fn, gcall *)
474	{
475	gcc_unreachable ();
476	}
477
478	/* Lane index of the first SIMT lane that supplies a non-zero argument.
479	This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
480	lane that executed the last iteration for handling OpenMP lastprivate. */
481
482	static void
483	expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
484	{
485	tree lhs = gimple_call_lhs (gs: stmt);
486	if (!lhs)
487	return;
488
489	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
490	rtx cond = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
491	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
492	class expand_operand ops[2];
493	create_output_operand (op: &ops[0], x: target, mode);
494	create_input_operand (op: &ops[1], value: cond, mode);
495	gcc_assert (targetm.have_omp_simt_last_lane ());
496	expand_insn (icode: targetm.code_for_omp_simt_last_lane, nops: 2, ops);
497	if (!rtx_equal_p (target, ops[0].value))
498	emit_move_insn (target, ops[0].value);
499	}
500
501	/* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
502
503	static void
504	expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
505	{
506	tree lhs = gimple_call_lhs (gs: stmt);
507	if (!lhs)
508	return;
509
510	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
511	rtx ctr = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
512	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
513	class expand_operand ops[2];
514	create_output_operand (op: &ops[0], x: target, mode);
515	create_input_operand (op: &ops[1], value: ctr, mode);
516	gcc_assert (targetm.have_omp_simt_ordered ());
517	expand_insn (icode: targetm.code_for_omp_simt_ordered, nops: 2, ops);
518	if (!rtx_equal_p (target, ops[0].value))
519	emit_move_insn (target, ops[0].value);
520	}
521
522	/* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
523	any lane supplies a non-zero argument. */
524
525	static void
526	expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
527	{
528	tree lhs = gimple_call_lhs (gs: stmt);
529	if (!lhs)
530	return;
531
532	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
533	rtx cond = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
534	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
535	class expand_operand ops[2];
536	create_output_operand (op: &ops[0], x: target, mode);
537	create_input_operand (op: &ops[1], value: cond, mode);
538	gcc_assert (targetm.have_omp_simt_vote_any ());
539	expand_insn (icode: targetm.code_for_omp_simt_vote_any, nops: 2, ops);
540	if (!rtx_equal_p (target, ops[0].value))
541	emit_move_insn (target, ops[0].value);
542	}
543
544	/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
545	is destination lane index XOR given offset. */
546
547	static void
548	expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
549	{
550	tree lhs = gimple_call_lhs (gs: stmt);
551	if (!lhs)
552	return;
553
554	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
555	rtx src = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
556	rtx idx = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
557	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
558	class expand_operand ops[3];
559	create_output_operand (op: &ops[0], x: target, mode);
560	create_input_operand (op: &ops[1], value: src, mode);
561	create_input_operand (op: &ops[2], value: idx, SImode);
562	gcc_assert (targetm.have_omp_simt_xchg_bfly ());
563	expand_insn (icode: targetm.code_for_omp_simt_xchg_bfly, nops: 3, ops);
564	if (!rtx_equal_p (target, ops[0].value))
565	emit_move_insn (target, ops[0].value);
566	}
567
568	/* Exchange between SIMT lanes according to given source lane index. */
569
570	static void
571	expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
572	{
573	tree lhs = gimple_call_lhs (gs: stmt);
574	if (!lhs)
575	return;
576
577	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
578	rtx src = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
579	rtx idx = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
580	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
581	class expand_operand ops[3];
582	create_output_operand (op: &ops[0], x: target, mode);
583	create_input_operand (op: &ops[1], value: src, mode);
584	create_input_operand (op: &ops[2], value: idx, SImode);
585	gcc_assert (targetm.have_omp_simt_xchg_idx ());
586	expand_insn (icode: targetm.code_for_omp_simt_xchg_idx, nops: 3, ops);
587	if (!rtx_equal_p (target, ops[0].value))
588	emit_move_insn (target, ops[0].value);
589	}
590
591	/* This should get expanded in adjust_simduid_builtins. */
592
593	static void
594	expand_GOMP_SIMD_LANE (internal_fn, gcall *)
595	{
596	gcc_unreachable ();
597	}
598
599	/* This should get expanded in adjust_simduid_builtins. */
600
601	static void
602	expand_GOMP_SIMD_VF (internal_fn, gcall *)
603	{
604	gcc_unreachable ();
605	}
606
607	/* This should get expanded in adjust_simduid_builtins. */
608
609	static void
610	expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
611	{
612	gcc_unreachable ();
613	}
614
615	/* This should get expanded in adjust_simduid_builtins. */
616
617	static void
618	expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
619	{
620	gcc_unreachable ();
621	}
622
623	/* This should get expanded in adjust_simduid_builtins. */
624
625	static void
626	expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
627	{
628	gcc_unreachable ();
629	}
630
631	/* This should get expanded in the sanopt pass. */
632
633	static void
634	expand_UBSAN_NULL (internal_fn, gcall *)
635	{
636	gcc_unreachable ();
637	}
638
639	/* This should get expanded in the sanopt pass. */
640
641	static void
642	expand_UBSAN_BOUNDS (internal_fn, gcall *)
643	{
644	gcc_unreachable ();
645	}
646
647	/* This should get expanded in the sanopt pass. */
648
649	static void
650	expand_UBSAN_VPTR (internal_fn, gcall *)
651	{
652	gcc_unreachable ();
653	}
654
655	/* This should get expanded in the sanopt pass. */
656
657	static void
658	expand_UBSAN_PTR (internal_fn, gcall *)
659	{
660	gcc_unreachable ();
661	}
662
663	/* This should get expanded in the sanopt pass. */
664
665	static void
666	expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
667	{
668	gcc_unreachable ();
669	}
670
671	/* This should get expanded in the sanopt pass. */
672
673	static void
674	expand_HWASAN_CHECK (internal_fn, gcall *)
675	{
676	gcc_unreachable ();
677	}
678
679	/* For hwasan stack tagging:
680	Clear tags on the dynamically allocated space.
681	For use after an object dynamically allocated on the stack goes out of
682	scope. */
683	static void
684	expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
685	{
686	gcc_assert (Pmode == ptr_mode);
687	tree restored_position = gimple_call_arg (gs: gc, index: 0);
688	rtx restored_rtx = expand_expr (exp: restored_position, NULL_RTX, VOIDmode,
689	modifier: EXPAND_NORMAL);
690	rtx func = init_one_libfunc ("__hwasan_tag_memory");
691	rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
692	stack_pointer_rtx, NULL_RTX, 0,
693	OPTAB_WIDEN);
694	emit_library_call_value (fun: func, NULL_RTX, fn_type: LCT_NORMAL, VOIDmode,
695	virtual_stack_dynamic_rtx, Pmode,
696	HWASAN_STACK_BACKGROUND, QImode,
697	arg3: off, Pmode);
698	}
699
700	/* For hwasan stack tagging:
701	Return a tag to be used for a dynamic allocation. */
702	static void
703	expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
704	{
705	tree tag = gimple_call_lhs (gs: gc);
706	rtx target = expand_expr (exp: tag, NULL_RTX, VOIDmode, modifier: EXPAND_NORMAL);
707	machine_mode mode = GET_MODE (target);
708	gcc_assert (mode == QImode);
709
710	rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
711	gcc_assert (base_tag);
712	rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
713	rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
714	target, /* unsignedp = */1,
715	OPTAB_WIDEN);
716	chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
717
718	/* Really need to put the tag into the `target` RTX. */
719	if (chosen_tag != target)
720	{
721	rtx temp = chosen_tag;
722	gcc_assert (GET_MODE (chosen_tag) == mode);
723	emit_move_insn (target, temp);
724	}
725
726	hwasan_increment_frame_tag ();
727	}
728
729	/* For hwasan stack tagging:
730	Tag a region of space in the shadow stack according to the base pointer of
731	an object on the stack. N.b. the length provided in the internal call is
732	required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
733	static void
734	expand_HWASAN_MARK (internal_fn, gcall *gc)
735	{
736	gcc_assert (ptr_mode == Pmode);
737	HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gs: gc, index: 0));
738	bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
739
740	tree base = gimple_call_arg (gs: gc, index: 1);
741	gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
742	rtx base_rtx = expand_normal (exp: base);
743
744	rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
745	: targetm.memtag.extract_tag (base_rtx, NULL_RTX);
746	rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
747
748	tree len = gimple_call_arg (gs: gc, index: 2);
749	rtx r_len = expand_normal (exp: len);
750
751	rtx func = init_one_libfunc ("__hwasan_tag_memory");
752	emit_library_call (fun: func, fn_type: LCT_NORMAL, VOIDmode, arg1: address, Pmode,
753	arg2: tag, QImode, arg3: r_len, Pmode);
754	}
755
756	/* For hwasan stack tagging:
757	Store a tag into a pointer. */
758	static void
759	expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
760	{
761	gcc_assert (ptr_mode == Pmode);
762	tree g_target = gimple_call_lhs (gs: gc);
763	tree g_ptr = gimple_call_arg (gs: gc, index: 0);
764	tree g_tag = gimple_call_arg (gs: gc, index: 1);
765
766	rtx ptr = expand_normal (exp: g_ptr);
767	rtx tag = expand_expr (exp: g_tag, NULL_RTX, QImode, modifier: EXPAND_NORMAL);
768	rtx target = expand_normal (exp: g_target);
769
770	rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
771	rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
772	if (tagged_value != target)
773	emit_move_insn (target, tagged_value);
774	}
775
776	/* This should get expanded in the sanopt pass. */
777
778	static void
779	expand_ASAN_CHECK (internal_fn, gcall *)
780	{
781	gcc_unreachable ();
782	}
783
784	/* This should get expanded in the sanopt pass. */
785
786	static void
787	expand_ASAN_MARK (internal_fn, gcall *)
788	{
789	gcc_unreachable ();
790	}
791
792	/* This should get expanded in the sanopt pass. */
793
794	static void
795	expand_ASAN_POISON (internal_fn, gcall *)
796	{
797	gcc_unreachable ();
798	}
799
800	/* This should get expanded in the sanopt pass. */
801
802	static void
803	expand_ASAN_POISON_USE (internal_fn, gcall *)
804	{
805	gcc_unreachable ();
806	}
807
808	/* This should get expanded in the tsan pass. */
809
810	static void
811	expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
812	{
813	gcc_unreachable ();
814	}
815
816	/* This should get expanded in the lower pass. */
817
818	static void
819	expand_FALLTHROUGH (internal_fn, gcall *call)
820	{
821	error_at (gimple_location (g: call),
822	"invalid use of attribute %<fallthrough%>");
823	}
824
825	/* Return minimum precision needed to represent all values
826	of ARG in SIGNed integral type. */
827
828	static int
829	get_min_precision (tree arg, signop sign)
830	{
831	int prec = TYPE_PRECISION (TREE_TYPE (arg));
832	int cnt = 0;
833	signop orig_sign = sign;
834	if (TREE_CODE (arg) == INTEGER_CST)
835	{
836	int p;
837	if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
838	{
839	widest_int w = wi::to_widest (t: arg);
840	w = wi::ext (x: w, offset: prec, sgn: sign);
841	p = wi::min_precision (x: w, sgn: sign);
842	}
843	else
844	p = wi::min_precision (x: wi::to_wide (t: arg), sgn: sign);
845	return MIN (p, prec);
846	}
847	while (CONVERT_EXPR_P (arg)
848	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
849	&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
850	{
851	arg = TREE_OPERAND (arg, 0);
852	if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
853	{
854	if (TYPE_UNSIGNED (TREE_TYPE (arg)))
855	sign = UNSIGNED;
856	else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
857	return prec + (orig_sign != sign);
858	prec = TYPE_PRECISION (TREE_TYPE (arg));
859	}
860	if (++cnt > 30)
861	return prec + (orig_sign != sign);
862	}
863	if (CONVERT_EXPR_P (arg)
864	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
865	&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
866	{
867	/* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
868	If y_2's min precision is smaller than prec, return that. */
869	int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
870	if (oprec < prec)
871	return oprec + (orig_sign != sign);
872	}
873	if (TREE_CODE (arg) != SSA_NAME)
874	return prec + (orig_sign != sign);
875	value_range r;
876	while (!get_global_range_query ()->range_of_expr (r, expr: arg)
877	|| r.varying_p ()
878	|| r.undefined_p ())
879	{
880	gimple *g = SSA_NAME_DEF_STMT (arg);
881	if (is_gimple_assign (gs: g)
882	&& CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
883	{
884	tree t = gimple_assign_rhs1 (gs: g);
885	if (INTEGRAL_TYPE_P (TREE_TYPE (t))
886	&& TYPE_PRECISION (TREE_TYPE (t)) <= prec)
887	{
888	arg = t;
889	if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
890	{
891	if (TYPE_UNSIGNED (TREE_TYPE (arg)))
892	sign = UNSIGNED;
893	else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
894	return prec + (orig_sign != sign);
895	prec = TYPE_PRECISION (TREE_TYPE (arg));
896	}
897	if (++cnt > 30)
898	return prec + (orig_sign != sign);
899	continue;
900	}
901	}
902	return prec + (orig_sign != sign);
903	}
904	if (sign == TYPE_SIGN (TREE_TYPE (arg)))
905	{
906	int p1 = wi::min_precision (x: r.lower_bound (), sgn: sign);
907	int p2 = wi::min_precision (x: r.upper_bound (), sgn: sign);
908	p1 = MAX (p1, p2);
909	prec = MIN (prec, p1);
910	}
911	else if (sign == UNSIGNED && !wi::neg_p (x: r.lower_bound (), sgn: SIGNED))
912	{
913	int p = wi::min_precision (x: r.upper_bound (), sgn: UNSIGNED);
914	prec = MIN (prec, p);
915	}
916	return prec + (orig_sign != sign);
917	}
918
919	/* Helper for expand_*_overflow. Set the __imag__ part to true
920	(1 except for signed:1 type, in which case store -1). */
921
922	static void
923	expand_arith_set_overflow (tree lhs, rtx target)
924	{
925	if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
926	&& !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
927	write_complex_part (target, constm1_rtx, true, false);
928	else
929	write_complex_part (target, const1_rtx, true, false);
930	}
931
932	/* Helper for expand_*_overflow. Store RES into the __real__ part
933	of TARGET. If RES has larger MODE than __real__ part of TARGET,
934	set the __imag__ part to 1 if RES doesn't fit into it. Similarly
935	if LHS has smaller precision than its mode. */
936
937	static void
938	expand_arith_overflow_result_store (tree lhs, rtx target,
939	scalar_int_mode mode, rtx res)
940	{
941	scalar_int_mode tgtmode
942	= as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
943	rtx lres = res;
944	if (tgtmode != mode)
945	{
946	rtx_code_label *done_label = gen_label_rtx ();
947	int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
948	lres = convert_modes (mode: tgtmode, oldmode: mode, x: res, unsignedp: uns);
949	gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
950	do_compare_rtx_and_jump (res, convert_modes (mode, oldmode: tgtmode, x: lres, unsignedp: uns),
951	EQ, true, mode, NULL_RTX, NULL, done_label,
952	profile_probability::very_likely ());
953	expand_arith_set_overflow (lhs, target);
954	emit_label (done_label);
955	}
956	int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
957	int tgtprec = GET_MODE_PRECISION (mode: tgtmode);
958	if (prec < tgtprec)
959	{
960	rtx_code_label *done_label = gen_label_rtx ();
961	int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
962	res = lres;
963	if (uns)
964	{
965	rtx mask
966	= immed_wide_int_const (wi::shifted_mask (start: 0, width: prec, negate_p: false, precision: tgtprec),
967	tgtmode);
968	lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
969	true, OPTAB_LIB_WIDEN);
970	}
971	else
972	{
973	lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
974	NULL_RTX, 1);
975	lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
976	NULL_RTX, 0);
977	}
978	do_compare_rtx_and_jump (res, lres,
979	EQ, true, tgtmode, NULL_RTX, NULL, done_label,
980	profile_probability::very_likely ());
981	expand_arith_set_overflow (lhs, target);
982	emit_label (done_label);
983	}
984	write_complex_part (target, lres, false, false);
985	}
986
987	/* Helper for expand_*_overflow. Store RES into TARGET. */
988
989	static void
990	expand_ubsan_result_store (tree lhs, rtx target, scalar_int_mode mode,
991	rtx res, rtx_code_label *do_error)
992	{
993	if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE
994	&& TYPE_PRECISION (TREE_TYPE (lhs)) < GET_MODE_PRECISION (mode))
995	{
996	int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
997	int prec = TYPE_PRECISION (TREE_TYPE (lhs));
998	int tgtprec = GET_MODE_PRECISION (mode);
999	rtx resc = gen_reg_rtx (mode), lres;
1000	emit_move_insn (resc, res);
1001	if (uns)
1002	{
1003	rtx mask
1004	= immed_wide_int_const (wi::shifted_mask (start: 0, width: prec, negate_p: false, precision: tgtprec),
1005	mode);
1006	lres = expand_simple_binop (mode, AND, res, mask, NULL_RTX,
1007	true, OPTAB_LIB_WIDEN);
1008	}
1009	else
1010	{
1011	lres = expand_shift (LSHIFT_EXPR, mode, res, tgtprec - prec,
1012	NULL_RTX, 1);
1013	lres = expand_shift (RSHIFT_EXPR, mode, lres, tgtprec - prec,
1014	NULL_RTX, 0);
1015	}
1016	if (lres != res)
1017	emit_move_insn (res, lres);
1018	do_compare_rtx_and_jump (res, resc,
1019	NE, true, mode, NULL_RTX, NULL, do_error,
1020	profile_probability::very_unlikely ());
1021	}
1022	if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
1023	/* If this is a scalar in a register that is stored in a wider mode
1024	than the declared mode, compute the result into its declared mode
1025	and then convert to the wider mode. Our value is the computed
1026	expression. */
1027	convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
1028	else
1029	emit_move_insn (target, res);
1030	}
1031
1032	/* Add sub/add overflow checking to the statement STMT.
1033	CODE says whether the operation is +, or -. */
1034
1035	void
1036	expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
1037	tree arg0, tree arg1, bool unsr_p, bool uns0_p,
1038	bool uns1_p, bool is_ubsan, tree *datap)
1039	{
1040	rtx res, target = NULL_RTX;
1041	tree fn;
1042	rtx_code_label *done_label = gen_label_rtx ();
1043	rtx_code_label *do_error = gen_label_rtx ();
1044	do_pending_stack_adjust ();
1045	rtx op0 = expand_normal (exp: arg0);
1046	rtx op1 = expand_normal (exp: arg1);
1047	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1048	int prec = GET_MODE_PRECISION (mode);
1049	rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1050	bool do_xor = false;
1051
1052	if (is_ubsan)
1053	gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1054
1055	if (lhs)
1056	{
1057	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1058	if (!is_ubsan)
1059	write_complex_part (target, const0_rtx, true, false);
1060	}
1061
1062	/* We assume both operands and result have the same precision
1063	here (GET_MODE_BITSIZE (mode)), S stands for signed type
1064	with that precision, U for unsigned type with that precision,
1065	sgn for unsigned most significant bit in that precision.
1066	s1 is signed first operand, u1 is unsigned first operand,
1067	s2 is signed second operand, u2 is unsigned second operand,
1068	sr is signed result, ur is unsigned result and the following
1069	rules say how to compute result (which is always result of
1070	the operands as if both were unsigned, cast to the right
1071	signedness) and how to compute whether operation overflowed.
1072
1073	s1 + s2 -> sr
1074	res = (S) ((U) s1 + (U) s2)
1075	ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1076	s1 - s2 -> sr
1077	res = (S) ((U) s1 - (U) s2)
1078	ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1079	u1 + u2 -> ur
1080	res = u1 + u2
1081	ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1082	u1 - u2 -> ur
1083	res = u1 - u2
1084	ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1085	s1 + u2 -> sr
1086	res = (S) ((U) s1 + u2)
1087	ovf = ((U) res ^ sgn) < u2
1088	s1 + u2 -> ur
1089	t1 = (S) (u2 ^ sgn)
1090	t2 = s1 + t1
1091	res = (U) t2 ^ sgn
1092	ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1093	s1 - u2 -> sr
1094	res = (S) ((U) s1 - u2)
1095	ovf = u2 > ((U) s1 ^ sgn)
1096	s1 - u2 -> ur
1097	res = (U) s1 - u2
1098	ovf = s1 < 0 || u2 > (U) s1
1099	u1 - s2 -> sr
1100	res = u1 - (U) s2
1101	ovf = u1 >= ((U) s2 ^ sgn)
1102	u1 - s2 -> ur
1103	t1 = u1 ^ sgn
1104	t2 = t1 - (U) s2
1105	res = t2 ^ sgn
1106	ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1107	s1 + s2 -> ur
1108	res = (U) s1 + (U) s2
1109	ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1110	u1 + u2 -> sr
1111	res = (S) (u1 + u2)
1112	ovf = (U) res < u2 || res < 0
1113	u1 - u2 -> sr
1114	res = (S) (u1 - u2)
1115	ovf = u1 >= u2 ? res < 0 : res >= 0
1116	s1 - s2 -> ur
1117	res = (U) s1 - (U) s2
1118	ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1119
1120	if (code == PLUS_EXPR && uns0_p && !uns1_p)
1121	{
1122	/* PLUS_EXPR is commutative, if operand signedness differs,
1123	canonicalize to the first operand being signed and second
1124	unsigned to simplify following code. */
1125	std::swap (a&: op0, b&: op1);
1126	std::swap (a&: arg0, b&: arg1);
1127	uns0_p = false;
1128	uns1_p = true;
1129	}
1130
1131	/* u1 +- u2 -> ur */
1132	if (uns0_p && uns1_p && unsr_p)
1133	{
1134	insn_code icode = optab_handler (op: code == PLUS_EXPR ? uaddv4_optab
1135	: usubv4_optab, mode);
1136	if (icode != CODE_FOR_nothing)
1137	{
1138	class expand_operand ops[4];
1139	rtx_insn *last = get_last_insn ();
1140
1141	res = gen_reg_rtx (mode);
1142	create_output_operand (op: &ops[0], x: res, mode);
1143	create_input_operand (op: &ops[1], value: op0, mode);
1144	create_input_operand (op: &ops[2], value: op1, mode);
1145	create_fixed_operand (op: &ops[3], x: do_error);
1146	if (maybe_expand_insn (icode, nops: 4, ops))
1147	{
1148	last = get_last_insn ();
1149	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1150	&& JUMP_P (last)
1151	&& any_condjump_p (last)
1152	&& !find_reg_note (last, REG_BR_PROB, 0))
1153	add_reg_br_prob_note (last,
1154	profile_probability::very_unlikely ());
1155	emit_jump (done_label);
1156	goto do_error_label;
1157	}
1158
1159	delete_insns_since (last);
1160	}
1161
1162	/* Compute the operation. On RTL level, the addition is always
1163	unsigned. */
1164	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1165	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1166	rtx tem = op0;
1167	/* For PLUS_EXPR, the operation is commutative, so we can pick
1168	operand to compare against. For prec <= BITS_PER_WORD, I think
1169	preferring REG operand is better over CONST_INT, because
1170	the CONST_INT might enlarge the instruction or CSE would need
1171	to figure out we'd already loaded it into a register before.
1172	For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1173	as then the multi-word comparison can be perhaps simplified. */
1174	if (code == PLUS_EXPR
1175	&& (prec <= BITS_PER_WORD
1176	? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
1177	: CONST_SCALAR_INT_P (op1)))
1178	tem = op1;
1179	do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
1180	true, mode, NULL_RTX, NULL, done_label,
1181	profile_probability::very_likely ());
1182	goto do_error_label;
1183	}
1184
1185	/* s1 +- u2 -> sr */
1186	if (!uns0_p && uns1_p && !unsr_p)
1187	{
1188	/* Compute the operation. On RTL level, the addition is always
1189	unsigned. */
1190	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1191	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1192	rtx tem = expand_binop (mode, add_optab,
1193	code == PLUS_EXPR ? res : op0, sgn,
1194	NULL_RTX, false, OPTAB_LIB_WIDEN);
1195	do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
1196	done_label, profile_probability::very_likely ());
1197	goto do_error_label;
1198	}
1199
1200	/* s1 + u2 -> ur */
1201	if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
1202	{
1203	op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1204	OPTAB_LIB_WIDEN);
1205	/* As we've changed op1, we have to avoid using the value range
1206	for the original argument. */
1207	arg1 = error_mark_node;
1208	do_xor = true;
1209	goto do_signed;
1210	}
1211
1212	/* u1 - s2 -> ur */
1213	if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
1214	{
1215	op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
1216	OPTAB_LIB_WIDEN);
1217	/* As we've changed op0, we have to avoid using the value range
1218	for the original argument. */
1219	arg0 = error_mark_node;
1220	do_xor = true;
1221	goto do_signed;
1222	}
1223
1224	/* s1 - u2 -> ur */
1225	if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1226	{
1227	/* Compute the operation. On RTL level, the addition is always
1228	unsigned. */
1229	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1230	OPTAB_LIB_WIDEN);
1231	int pos_neg = get_range_pos_neg (arg0);
1232	if (pos_neg == 2)
1233	/* If ARG0 is known to be always negative, this is always overflow. */
1234	emit_jump (do_error);
1235	else if (pos_neg == 3)
1236	/* If ARG0 is not known to be always positive, check at runtime. */
1237	do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1238	NULL, do_error, profile_probability::very_unlikely ());
1239	do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1240	done_label, profile_probability::very_likely ());
1241	goto do_error_label;
1242	}
1243
1244	/* u1 - s2 -> sr */
1245	if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1246	{
1247	/* Compute the operation. On RTL level, the addition is always
1248	unsigned. */
1249	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1250	OPTAB_LIB_WIDEN);
1251	rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1252	OPTAB_LIB_WIDEN);
1253	do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1254	done_label, profile_probability::very_likely ());
1255	goto do_error_label;
1256	}
1257
1258	/* u1 + u2 -> sr */
1259	if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1260	{
1261	/* Compute the operation. On RTL level, the addition is always
1262	unsigned. */
1263	res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1264	OPTAB_LIB_WIDEN);
1265	do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1266	NULL, do_error, profile_probability::very_unlikely ());
1267	rtx tem = op1;
1268	/* The operation is commutative, so we can pick operand to compare
1269	against. For prec <= BITS_PER_WORD, I think preferring REG operand
1270	is better over CONST_INT, because the CONST_INT might enlarge the
1271	instruction or CSE would need to figure out we'd already loaded it
1272	into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1273	might be more beneficial, as then the multi-word comparison can be
1274	perhaps simplified. */
1275	if (prec <= BITS_PER_WORD
1276	? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1277	: CONST_SCALAR_INT_P (op0))
1278	tem = op0;
1279	do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1280	done_label, profile_probability::very_likely ());
1281	goto do_error_label;
1282	}
1283
1284	/* s1 +- s2 -> ur */
1285	if (!uns0_p && !uns1_p && unsr_p)
1286	{
1287	/* Compute the operation. On RTL level, the addition is always
1288	unsigned. */
1289	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1290	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1291	int pos_neg = get_range_pos_neg (arg1);
1292	if (code == PLUS_EXPR)
1293	{
1294	int pos_neg0 = get_range_pos_neg (arg0);
1295	if (pos_neg0 != 3 && pos_neg == 3)
1296	{
1297	std::swap (a&: op0, b&: op1);
1298	pos_neg = pos_neg0;
1299	}
1300	}
1301	rtx tem;
1302	if (pos_neg != 3)
1303	{
1304	tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1305	? and_optab : ior_optab,
1306	op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1307	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1308	NULL, done_label, profile_probability::very_likely ());
1309	}
1310	else
1311	{
1312	rtx_code_label *do_ior_label = gen_label_rtx ();
1313	do_compare_rtx_and_jump (op1, const0_rtx,
1314	code == MINUS_EXPR ? GE : LT, false, mode,
1315	NULL_RTX, NULL, do_ior_label,
1316	profile_probability::even ());
1317	tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1318	OPTAB_LIB_WIDEN);
1319	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1320	NULL, done_label, profile_probability::very_likely ());
1321	emit_jump (do_error);
1322	emit_label (do_ior_label);
1323	tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1324	OPTAB_LIB_WIDEN);
1325	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1326	NULL, done_label, profile_probability::very_likely ());
1327	}
1328	goto do_error_label;
1329	}
1330
1331	/* u1 - u2 -> sr */
1332	if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1333	{
1334	/* Compute the operation. On RTL level, the addition is always
1335	unsigned. */
1336	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1337	OPTAB_LIB_WIDEN);
1338	rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1339	do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1340	op0_geu_op1, profile_probability::even ());
1341	do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1342	NULL, done_label, profile_probability::very_likely ());
1343	emit_jump (do_error);
1344	emit_label (op0_geu_op1);
1345	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1346	NULL, done_label, profile_probability::very_likely ());
1347	goto do_error_label;
1348	}
1349
1350	gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1351
1352	/* s1 +- s2 -> sr */
1353	do_signed:
1354	{
1355	insn_code icode = optab_handler (op: code == PLUS_EXPR ? addv4_optab
1356	: subv4_optab, mode);
1357	if (icode != CODE_FOR_nothing)
1358	{
1359	class expand_operand ops[4];
1360	rtx_insn *last = get_last_insn ();
1361
1362	res = gen_reg_rtx (mode);
1363	create_output_operand (op: &ops[0], x: res, mode);
1364	create_input_operand (op: &ops[1], value: op0, mode);
1365	create_input_operand (op: &ops[2], value: op1, mode);
1366	create_fixed_operand (op: &ops[3], x: do_error);
1367	if (maybe_expand_insn (icode, nops: 4, ops))
1368	{
1369	last = get_last_insn ();
1370	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1371	&& JUMP_P (last)
1372	&& any_condjump_p (last)
1373	&& !find_reg_note (last, REG_BR_PROB, 0))
1374	add_reg_br_prob_note (last,
1375	profile_probability::very_unlikely ());
1376	emit_jump (done_label);
1377	goto do_error_label;
1378	}
1379
1380	delete_insns_since (last);
1381	}
1382
1383	/* Compute the operation. On RTL level, the addition is always
1384	unsigned. */
1385	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1386	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1387
1388	/* If we can prove that one of the arguments (for MINUS_EXPR only
1389	the second operand, as subtraction is not commutative) is always
1390	non-negative or always negative, we can do just one comparison
1391	and conditional jump. */
1392	int pos_neg = get_range_pos_neg (arg1);
1393	if (code == PLUS_EXPR)
1394	{
1395	int pos_neg0 = get_range_pos_neg (arg0);
1396	if (pos_neg0 != 3 && pos_neg == 3)
1397	{
1398	std::swap (a&: op0, b&: op1);
1399	pos_neg = pos_neg0;
1400	}
1401	}
1402
1403	/* Addition overflows if and only if the two operands have the same sign,
1404	and the result has the opposite sign. Subtraction overflows if and
1405	only if the two operands have opposite sign, and the subtrahend has
1406	the same sign as the result. Here 0 is counted as positive. */
1407	if (pos_neg == 3)
1408	{
1409	/* Compute op0 ^ op1 (operands have opposite sign). */
1410	rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1411	OPTAB_LIB_WIDEN);
1412
1413	/* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1414	rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1415	OPTAB_LIB_WIDEN);
1416
1417	rtx tem;
1418	if (code == PLUS_EXPR)
1419	{
1420	/* Compute (res ^ op1) & ~(op0 ^ op1). */
1421	tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1422	tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1423	OPTAB_LIB_WIDEN);
1424	}
1425	else
1426	{
1427	/* Compute (op0 ^ op1) & ~(res ^ op1). */
1428	tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1429	tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1430	OPTAB_LIB_WIDEN);
1431	}
1432
1433	/* No overflow if the result has bit sign cleared. */
1434	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1435	NULL, done_label, profile_probability::very_likely ());
1436	}
1437
1438	/* Compare the result of the operation with the first operand.
1439	No overflow for addition if second operand is positive and result
1440	is larger or second operand is negative and result is smaller.
1441	Likewise for subtraction with sign of second operand flipped. */
1442	else
1443	do_compare_rtx_and_jump (res, op0,
1444	(pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1445	false, mode, NULL_RTX, NULL, done_label,
1446	profile_probability::very_likely ());
1447	}
1448
1449	do_error_label:
1450	emit_label (do_error);
1451	if (is_ubsan)
1452	{
1453	/* Expand the ubsan builtin call. */
1454	push_temp_slots ();
1455	fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1456	arg0, arg1, datap);
1457	expand_normal (exp: fn);
1458	pop_temp_slots ();
1459	do_pending_stack_adjust ();
1460	}
1461	else if (lhs)
1462	expand_arith_set_overflow (lhs, target);
1463
1464	/* We're done. */
1465	emit_label (done_label);
1466
1467	if (lhs)
1468	{
1469	if (is_ubsan)
1470	expand_ubsan_result_store (lhs, target, mode, res, do_error);
1471	else
1472	{
1473	if (do_xor)
1474	res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1475	OPTAB_LIB_WIDEN);
1476
1477	expand_arith_overflow_result_store (lhs, target, mode, res);
1478	}
1479	}
1480	}
1481
1482	/* Add negate overflow checking to the statement STMT. */
1483
1484	static void
1485	expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1486	tree *datap)
1487	{
1488	rtx res, op1;
1489	tree fn;
1490	rtx_code_label *done_label, *do_error;
1491	rtx target = NULL_RTX;
1492
1493	done_label = gen_label_rtx ();
1494	do_error = gen_label_rtx ();
1495
1496	do_pending_stack_adjust ();
1497	op1 = expand_normal (exp: arg1);
1498
1499	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1500	if (lhs)
1501	{
1502	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1503	if (!is_ubsan)
1504	write_complex_part (target, const0_rtx, true, false);
1505	}
1506
1507	enum insn_code icode = optab_handler (op: negv3_optab, mode);
1508	if (icode != CODE_FOR_nothing)
1509	{
1510	class expand_operand ops[3];
1511	rtx_insn *last = get_last_insn ();
1512
1513	res = gen_reg_rtx (mode);
1514	create_output_operand (op: &ops[0], x: res, mode);
1515	create_input_operand (op: &ops[1], value: op1, mode);
1516	create_fixed_operand (op: &ops[2], x: do_error);
1517	if (maybe_expand_insn (icode, nops: 3, ops))
1518	{
1519	last = get_last_insn ();
1520	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1521	&& JUMP_P (last)
1522	&& any_condjump_p (last)
1523	&& !find_reg_note (last, REG_BR_PROB, 0))
1524	add_reg_br_prob_note (last,
1525	profile_probability::very_unlikely ());
1526	emit_jump (done_label);
1527	}
1528	else
1529	{
1530	delete_insns_since (last);
1531	icode = CODE_FOR_nothing;
1532	}
1533	}
1534
1535	if (icode == CODE_FOR_nothing)
1536	{
1537	/* Compute the operation. On RTL level, the addition is always
1538	unsigned. */
1539	res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1540
1541	/* Compare the operand with the most negative value. */
1542	rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1543	do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1544	done_label, profile_probability::very_likely ());
1545	}
1546
1547	emit_label (do_error);
1548	if (is_ubsan)
1549	{
1550	/* Expand the ubsan builtin call. */
1551	push_temp_slots ();
1552	fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1553	arg1, NULL_TREE, datap);
1554	expand_normal (exp: fn);
1555	pop_temp_slots ();
1556	do_pending_stack_adjust ();
1557	}
1558	else if (lhs)
1559	expand_arith_set_overflow (lhs, target);
1560
1561	/* We're done. */
1562	emit_label (done_label);
1563
1564	if (lhs)
1565	{
1566	if (is_ubsan)
1567	expand_ubsan_result_store (lhs, target, mode, res, do_error);
1568	else
1569	expand_arith_overflow_result_store (lhs, target, mode, res);
1570	}
1571	}
1572
1573	/* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1574	mode MODE can be expanded without using a libcall. */
1575
1576	static bool
1577	can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1578	rtx op0, rtx op1, bool uns)
1579	{
1580	if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1581	!= CODE_FOR_nothing)
1582	return true;
1583
1584	if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1585	!= CODE_FOR_nothing)
1586	return true;
1587
1588	rtx_insn *last = get_last_insn ();
1589	if (CONSTANT_P (op0))
1590	op0 = convert_modes (mode: wmode, oldmode: mode, x: op0, unsignedp: uns);
1591	else
1592	op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1593	if (CONSTANT_P (op1))
1594	op1 = convert_modes (mode: wmode, oldmode: mode, x: op1, unsignedp: uns);
1595	else
1596	op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1597	rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1598	delete_insns_since (last);
1599	return ret != NULL_RTX;
1600	}
1601
1602	/* Add mul overflow checking to the statement STMT. */
1603
1604	static void
1605	expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1606	bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1607	tree *datap)
1608	{
1609	rtx res, op0, op1;
1610	tree fn, type;
1611	rtx_code_label *done_label, *do_error;
1612	rtx target = NULL_RTX;
1613	signop sign;
1614	enum insn_code icode;
1615
1616	done_label = gen_label_rtx ();
1617	do_error = gen_label_rtx ();
1618
1619	do_pending_stack_adjust ();
1620	op0 = expand_normal (exp: arg0);
1621	op1 = expand_normal (exp: arg1);
1622
1623	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1624	bool uns = unsr_p;
1625	if (lhs)
1626	{
1627	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1628	if (!is_ubsan)
1629	write_complex_part (target, const0_rtx, true, false);
1630	}
1631
1632	if (is_ubsan)
1633	gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1634
1635	/* We assume both operands and result have the same precision
1636	here (GET_MODE_BITSIZE (mode)), S stands for signed type
1637	with that precision, U for unsigned type with that precision,
1638	sgn for unsigned most significant bit in that precision.
1639	s1 is signed first operand, u1 is unsigned first operand,
1640	s2 is signed second operand, u2 is unsigned second operand,
1641	sr is signed result, ur is unsigned result and the following
1642	rules say how to compute result (which is always result of
1643	the operands as if both were unsigned, cast to the right
1644	signedness) and how to compute whether operation overflowed.
1645	main_ovf (false) stands for jump on signed multiplication
1646	overflow or the main algorithm with uns == false.
1647	main_ovf (true) stands for jump on unsigned multiplication
1648	overflow or the main algorithm with uns == true.
1649
1650	s1 * s2 -> sr
1651	res = (S) ((U) s1 * (U) s2)
1652	ovf = main_ovf (false)
1653	u1 * u2 -> ur
1654	res = u1 * u2
1655	ovf = main_ovf (true)
1656	s1 * u2 -> ur
1657	res = (U) s1 * u2
1658	ovf = (s1 < 0 && u2) || main_ovf (true)
1659	u1 * u2 -> sr
1660	res = (S) (u1 * u2)
1661	ovf = res < 0 || main_ovf (true)
1662	s1 * u2 -> sr
1663	res = (S) ((U) s1 * u2)
1664	ovf = (S) u2 >= 0 ? main_ovf (false)
1665	: (s1 != 0 && (s1 != -1 || u2 != (U) res))
1666	s1 * s2 -> ur
1667	t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1668	t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1669	res = t1 * t2
1670	ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1671
1672	if (uns0_p && !uns1_p)
1673	{
1674	/* Multiplication is commutative, if operand signedness differs,
1675	canonicalize to the first operand being signed and second
1676	unsigned to simplify following code. */
1677	std::swap (a&: op0, b&: op1);
1678	std::swap (a&: arg0, b&: arg1);
1679	uns0_p = false;
1680	uns1_p = true;
1681	}
1682
1683	int pos_neg0 = get_range_pos_neg (arg0);
1684	int pos_neg1 = get_range_pos_neg (arg1);
1685	/* Unsigned types with smaller than mode precision, even if they have most
1686	significant bit set, are still zero-extended. */
1687	if (uns0_p && TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
1688	pos_neg0 = 1;
1689	if (uns1_p && TYPE_PRECISION (TREE_TYPE (arg1)) < GET_MODE_PRECISION (mode))
1690	pos_neg1 = 1;
1691
1692	/* s1 * u2 -> ur */
1693	if (!uns0_p && uns1_p && unsr_p)
1694	{
1695	switch (pos_neg0)
1696	{
1697	case 1:
1698	/* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1699	goto do_main;
1700	case 2:
1701	/* If s1 is negative, avoid the main code, just multiply and
1702	signal overflow if op1 is not 0. */
1703	struct separate_ops ops;
1704	ops.code = MULT_EXPR;
1705	ops.type = TREE_TYPE (arg1);
1706	ops.op0 = make_tree (ops.type, op0);
1707	ops.op1 = make_tree (ops.type, op1);
1708	ops.op2 = NULL_TREE;
1709	ops.location = loc;
1710	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1711	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1712	NULL, done_label, profile_probability::very_likely ());
1713	goto do_error_label;
1714	case 3:
1715	if (get_min_precision (arg: arg1, sign: UNSIGNED)
1716	+ get_min_precision (arg: arg0, sign: SIGNED) <= GET_MODE_PRECISION (mode))
1717	{
1718	/* If the first operand is sign extended from narrower type, the
1719	second operand is zero extended from narrower type and
1720	the sum of the two precisions is smaller or equal to the
1721	result precision: if the first argument is at runtime
1722	non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1723	and there will be no overflow, if the first argument is
1724	negative and the second argument zero, the result will be
1725	0 and there will be no overflow, if the first argument is
1726	negative and the second argument positive, the result when
1727	treated as signed will be negative (minimum -0x7f80 or
1728	-0x7f..f80..0) there will be always overflow. So, do
1729	res = (U) (s1 * u2)
1730	ovf = (S) res < 0 */
1731	struct separate_ops ops;
1732	ops.code = MULT_EXPR;
1733	ops.type
1734	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1735	1);
1736	ops.op0 = make_tree (ops.type, op0);
1737	ops.op1 = make_tree (ops.type, op1);
1738	ops.op2 = NULL_TREE;
1739	ops.location = loc;
1740	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1741	do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1742	mode, NULL_RTX, NULL, done_label,
1743	profile_probability::very_likely ());
1744	goto do_error_label;
1745	}
1746	rtx_code_label *do_main_label;
1747	do_main_label = gen_label_rtx ();
1748	do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1749	NULL, do_main_label, profile_probability::very_likely ());
1750	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1751	NULL, do_main_label, profile_probability::very_likely ());
1752	expand_arith_set_overflow (lhs, target);
1753	emit_label (do_main_label);
1754	goto do_main;
1755	default:
1756	gcc_unreachable ();
1757	}
1758	}
1759
1760	/* u1 * u2 -> sr */
1761	if (uns0_p && uns1_p && !unsr_p)
1762	{
1763	if ((pos_neg0 | pos_neg1) == 1)
1764	{
1765	/* If both arguments are zero extended from narrower types,
1766	the MSB will be clear on both and so we can pretend it is
1767	a normal s1 * s2 -> sr multiplication. */
1768	uns0_p = false;
1769	uns1_p = false;
1770	}
1771	else
1772	uns = true;
1773	/* Rest of handling of this case after res is computed. */
1774	goto do_main;
1775	}
1776
1777	/* s1 * u2 -> sr */
1778	if (!uns0_p && uns1_p && !unsr_p)
1779	{
1780	switch (pos_neg1)
1781	{
1782	case 1:
1783	goto do_main;
1784	case 2:
1785	/* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1786	avoid the main code, just multiply and signal overflow
1787	unless 0 * u2 or -1 * ((U) Smin). */
1788	struct separate_ops ops;
1789	ops.code = MULT_EXPR;
1790	ops.type = TREE_TYPE (arg1);
1791	ops.op0 = make_tree (ops.type, op0);
1792	ops.op1 = make_tree (ops.type, op1);
1793	ops.op2 = NULL_TREE;
1794	ops.location = loc;
1795	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1796	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1797	NULL, done_label, profile_probability::very_likely ());
1798	do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1799	NULL, do_error, profile_probability::very_unlikely ());
1800	int prec;
1801	prec = GET_MODE_PRECISION (mode);
1802	rtx sgn;
1803	sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1804	do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1805	NULL, done_label, profile_probability::very_likely ());
1806	goto do_error_label;
1807	case 3:
1808	/* Rest of handling of this case after res is computed. */
1809	goto do_main;
1810	default:
1811	gcc_unreachable ();
1812	}
1813	}
1814
1815	/* s1 * s2 -> ur */
1816	if (!uns0_p && !uns1_p && unsr_p)
1817	{
1818	rtx tem;
1819	switch (pos_neg0 | pos_neg1)
1820	{
1821	case 1: /* Both operands known to be non-negative. */
1822	goto do_main;
1823	case 2: /* Both operands known to be negative. */
1824	op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1825	op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1826	/* Avoid looking at arg0/arg1 ranges, as we've changed
1827	the arguments. */
1828	arg0 = error_mark_node;
1829	arg1 = error_mark_node;
1830	goto do_main;
1831	case 3:
1832	if ((pos_neg0 ^ pos_neg1) == 3)
1833	{
1834	/* If one operand is known to be negative and the other
1835	non-negative, this overflows always, unless the non-negative
1836	one is 0. Just do normal multiply and set overflow
1837	unless one of the operands is 0. */
1838	struct separate_ops ops;
1839	ops.code = MULT_EXPR;
1840	ops.type
1841	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1842	1);
1843	ops.op0 = make_tree (ops.type, op0);
1844	ops.op1 = make_tree (ops.type, op1);
1845	ops.op2 = NULL_TREE;
1846	ops.location = loc;
1847	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1848	do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1849	true, mode, NULL_RTX, NULL, done_label,
1850	profile_probability::very_likely ());
1851	goto do_error_label;
1852	}
1853	if (get_min_precision (arg: arg0, sign: SIGNED)
1854	+ get_min_precision (arg: arg1, sign: SIGNED) <= GET_MODE_PRECISION (mode))
1855	{
1856	/* If both operands are sign extended from narrower types and
1857	the sum of the two precisions is smaller or equal to the
1858	result precision: if both arguments are at runtime
1859	non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1860	and there will be no overflow, if both arguments are negative,
1861	maximum result will be 0x40..00 and there will be no overflow
1862	either, if one argument is positive and the other argument
1863	negative, the result when treated as signed will be negative
1864	and there will be always overflow, and if one argument is
1865	zero and the other negative the result will be zero and no
1866	overflow. So, do
1867	res = (U) (s1 * s2)
1868	ovf = (S) res < 0 */
1869	struct separate_ops ops;
1870	ops.code = MULT_EXPR;
1871	ops.type
1872	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1873	1);
1874	ops.op0 = make_tree (ops.type, op0);
1875	ops.op1 = make_tree (ops.type, op1);
1876	ops.op2 = NULL_TREE;
1877	ops.location = loc;
1878	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1879	do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1880	mode, NULL_RTX, NULL, done_label,
1881	profile_probability::very_likely ());
1882	goto do_error_label;
1883	}
1884	/* The general case, do all the needed comparisons at runtime. */
1885	rtx_code_label *do_main_label, *after_negate_label;
1886	rtx rop0, rop1;
1887	rop0 = gen_reg_rtx (mode);
1888	rop1 = gen_reg_rtx (mode);
1889	emit_move_insn (rop0, op0);
1890	emit_move_insn (rop1, op1);
1891	op0 = rop0;
1892	op1 = rop1;
1893	do_main_label = gen_label_rtx ();
1894	after_negate_label = gen_label_rtx ();
1895	tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1896	OPTAB_LIB_WIDEN);
1897	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1898	NULL, after_negate_label, profile_probability::very_likely ());
1899	/* Both arguments negative here, negate them and continue with
1900	normal unsigned overflow checking multiplication. */
1901	emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1902	NULL_RTX, false));
1903	emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1904	NULL_RTX, false));
1905	/* Avoid looking at arg0/arg1 ranges, as we might have changed
1906	the arguments. */
1907	arg0 = error_mark_node;
1908	arg1 = error_mark_node;
1909	emit_jump (do_main_label);
1910	emit_label (after_negate_label);
1911	tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1912	OPTAB_LIB_WIDEN);
1913	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1914	NULL, do_main_label,
1915	profile_probability::very_likely ());
1916	/* One argument is negative here, the other positive. This
1917	overflows always, unless one of the arguments is 0. But
1918	if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1919	is, thus we can keep do_main code oring in overflow as is. */
1920	if (pos_neg0 != 2)
1921	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1922	NULL, do_main_label,
1923	profile_probability::very_unlikely ());
1924	if (pos_neg1 != 2)
1925	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1926	NULL, do_main_label,
1927	profile_probability::very_unlikely ());
1928	expand_arith_set_overflow (lhs, target);
1929	emit_label (do_main_label);
1930	goto do_main;
1931	default:
1932	gcc_unreachable ();
1933	}
1934	}
1935
1936	do_main:
1937	type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1938	sign = uns ? UNSIGNED : SIGNED;
1939	icode = optab_handler (op: uns ? umulv4_optab : mulv4_optab, mode);
1940	if (uns
1941	&& (integer_pow2p (arg0) || integer_pow2p (arg1))
1942	&& (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1943	{
1944	/* Optimize unsigned multiplication by power of 2 constant
1945	using 2 shifts, one for result, one to extract the shifted
1946	out bits to see if they are all zero.
1947	Don't do this if optimizing for size and we have umulv4_optab,
1948	in that case assume multiplication will be shorter.
1949	This is heuristics based on the single target that provides
1950	umulv4 right now (i?86/x86_64), if further targets add it, this
1951	might need to be revisited.
1952	Cases where both operands are constant should be folded already
1953	during GIMPLE, and cases where one operand is constant but not
1954	power of 2 are questionable, either the WIDEN_MULT_EXPR case
1955	below can be done without multiplication, just by shifts and adds,
1956	or we'd need to divide the result (and hope it actually doesn't
1957	really divide nor multiply) and compare the result of the division
1958	with the original operand. */
1959	rtx opn0 = op0;
1960	rtx opn1 = op1;
1961	tree argn0 = arg0;
1962	tree argn1 = arg1;
1963	if (integer_pow2p (arg0))
1964	{
1965	std::swap (a&: opn0, b&: opn1);
1966	std::swap (a&: argn0, b&: argn1);
1967	}
1968	int cnt = tree_log2 (argn1);
1969	if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1970	{
1971	rtx upper = const0_rtx;
1972	res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1973	if (cnt != 0)
1974	upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1975	GET_MODE_PRECISION (mode) - cnt,
1976	NULL_RTX, uns);
1977	do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
1978	NULL_RTX, NULL, done_label,
1979	profile_probability::very_likely ());
1980	goto do_error_label;
1981	}
1982	}
1983	if (icode != CODE_FOR_nothing)
1984	{
1985	class expand_operand ops[4];
1986	rtx_insn *last = get_last_insn ();
1987
1988	res = gen_reg_rtx (mode);
1989	create_output_operand (op: &ops[0], x: res, mode);
1990	create_input_operand (op: &ops[1], value: op0, mode);
1991	create_input_operand (op: &ops[2], value: op1, mode);
1992	create_fixed_operand (op: &ops[3], x: do_error);
1993	if (maybe_expand_insn (icode, nops: 4, ops))
1994	{
1995	last = get_last_insn ();
1996	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1997	&& JUMP_P (last)
1998	&& any_condjump_p (last)
1999	&& !find_reg_note (last, REG_BR_PROB, 0))
2000	add_reg_br_prob_note (last,
2001	profile_probability::very_unlikely ());
2002	emit_jump (done_label);
2003	}
2004	else
2005	{
2006	delete_insns_since (last);
2007	icode = CODE_FOR_nothing;
2008	}
2009	}
2010
2011	if (icode == CODE_FOR_nothing)
2012	{
2013	struct separate_ops ops;
2014	int prec = GET_MODE_PRECISION (mode);
2015	scalar_int_mode hmode, wmode;
2016	ops.op0 = make_tree (type, op0);
2017	ops.op1 = make_tree (type, op1);
2018	ops.op2 = NULL_TREE;
2019	ops.location = loc;
2020
2021	/* Optimize unsigned overflow check where we don't use the
2022	multiplication result, just whether overflow happened.
2023	If we can do MULT_HIGHPART_EXPR, that followed by
2024	comparison of the result against zero is cheapest.
2025	We'll still compute res, but it should be DCEd later. */
2026	use_operand_p use;
2027	gimple *use_stmt;
2028	if (!is_ubsan
2029	&& lhs
2030	&& uns
2031	&& !(uns0_p && uns1_p && !unsr_p)
2032	&& can_mult_highpart_p (mode, uns) == 1
2033	&& single_imm_use (var: lhs, use_p: &use, stmt: &use_stmt)
2034	&& is_gimple_assign (gs: use_stmt)
2035	&& gimple_assign_rhs_code (gs: use_stmt) == IMAGPART_EXPR)
2036	goto highpart;
2037
2038	if (GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wmode)
2039	&& targetm.scalar_mode_supported_p (wmode)
2040	&& can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
2041	{
2042	twoxwider:
2043	ops.code = WIDEN_MULT_EXPR;
2044	ops.type
2045	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode: wmode), uns);
2046
2047	res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
2048	rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
2049	NULL_RTX, uns);
2050	hipart = convert_modes (mode, oldmode: wmode, x: hipart, unsignedp: uns);
2051	res = convert_modes (mode, oldmode: wmode, x: res, unsignedp: uns);
2052	if (uns)
2053	/* For the unsigned multiplication, there was overflow if
2054	HIPART is non-zero. */
2055	do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2056	NULL_RTX, NULL, done_label,
2057	profile_probability::very_likely ());
2058	else
2059	{
2060	/* RES is used more than once, place it in a pseudo. */
2061	res = force_reg (mode, res);
2062
2063	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2064	NULL_RTX, 0);
2065	/* RES is low half of the double width result, HIPART
2066	the high half. There was overflow if
2067	HIPART is different from RES < 0 ? -1 : 0. */
2068	do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2069	NULL_RTX, NULL, done_label,
2070	profile_probability::very_likely ());
2071	}
2072	}
2073	else if (can_mult_highpart_p (mode, uns) == 1)
2074	{
2075	highpart:
2076	ops.code = MULT_HIGHPART_EXPR;
2077	ops.type = type;
2078
2079	rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
2080	EXPAND_NORMAL);
2081	ops.code = MULT_EXPR;
2082	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2083	if (uns)
2084	/* For the unsigned multiplication, there was overflow if
2085	HIPART is non-zero. */
2086	do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2087	NULL_RTX, NULL, done_label,
2088	profile_probability::very_likely ());
2089	else
2090	{
2091	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2092	NULL_RTX, 0);
2093	/* RES is low half of the double width result, HIPART
2094	the high half. There was overflow if
2095	HIPART is different from RES < 0 ? -1 : 0. */
2096	do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2097	NULL_RTX, NULL, done_label,
2098	profile_probability::very_likely ());
2099	}
2100
2101	}
2102	else if (int_mode_for_size (size: prec / 2, limit: 1).exists (mode: &hmode)
2103	&& 2 * GET_MODE_PRECISION (mode: hmode) == prec)
2104	{
2105	rtx_code_label *large_op0 = gen_label_rtx ();
2106	rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
2107	rtx_code_label *one_small_one_large = gen_label_rtx ();
2108	rtx_code_label *both_ops_large = gen_label_rtx ();
2109	rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
2110	rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
2111	rtx_code_label *do_overflow = gen_label_rtx ();
2112	rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
2113
2114	unsigned int hprec = GET_MODE_PRECISION (mode: hmode);
2115	rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
2116	NULL_RTX, uns);
2117	hipart0 = convert_modes (mode: hmode, oldmode: mode, x: hipart0, unsignedp: uns);
2118	rtx lopart0 = convert_modes (mode: hmode, oldmode: mode, x: op0, unsignedp: uns);
2119	rtx signbit0 = const0_rtx;
2120	if (!uns)
2121	signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
2122	NULL_RTX, 0);
2123	rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
2124	NULL_RTX, uns);
2125	hipart1 = convert_modes (mode: hmode, oldmode: mode, x: hipart1, unsignedp: uns);
2126	rtx lopart1 = convert_modes (mode: hmode, oldmode: mode, x: op1, unsignedp: uns);
2127	rtx signbit1 = const0_rtx;
2128	if (!uns)
2129	signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
2130	NULL_RTX, 0);
2131
2132	res = gen_reg_rtx (mode);
2133
2134	/* True if op0 resp. op1 are known to be in the range of
2135	halfstype. */
2136	bool op0_small_p = false;
2137	bool op1_small_p = false;
2138	/* True if op0 resp. op1 are known to have all zeros or all ones
2139	in the upper half of bits, but are not known to be
2140	op{0,1}_small_p. */
2141	bool op0_medium_p = false;
2142	bool op1_medium_p = false;
2143	/* -1 if op{0,1} is known to be negative, 0 if it is known to be
2144	nonnegative, 1 if unknown. */
2145	int op0_sign = 1;
2146	int op1_sign = 1;
2147
2148	if (pos_neg0 == 1)
2149	op0_sign = 0;
2150	else if (pos_neg0 == 2)
2151	op0_sign = -1;
2152	if (pos_neg1 == 1)
2153	op1_sign = 0;
2154	else if (pos_neg1 == 2)
2155	op1_sign = -1;
2156
2157	unsigned int mprec0 = prec;
2158	if (arg0 != error_mark_node)
2159	mprec0 = get_min_precision (arg: arg0, sign);
2160	if (mprec0 <= hprec)
2161	op0_small_p = true;
2162	else if (!uns && mprec0 <= hprec + 1)
2163	op0_medium_p = true;
2164	unsigned int mprec1 = prec;
2165	if (arg1 != error_mark_node)
2166	mprec1 = get_min_precision (arg: arg1, sign);
2167	if (mprec1 <= hprec)
2168	op1_small_p = true;
2169	else if (!uns && mprec1 <= hprec + 1)
2170	op1_medium_p = true;
2171
2172	int smaller_sign = 1;
2173	int larger_sign = 1;
2174	if (op0_small_p)
2175	{
2176	smaller_sign = op0_sign;
2177	larger_sign = op1_sign;
2178	}
2179	else if (op1_small_p)
2180	{
2181	smaller_sign = op1_sign;
2182	larger_sign = op0_sign;
2183	}
2184	else if (op0_sign == op1_sign)
2185	{
2186	smaller_sign = op0_sign;
2187	larger_sign = op0_sign;
2188	}
2189
2190	if (!op0_small_p)
2191	do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
2192	NULL_RTX, NULL, large_op0,
2193	profile_probability::unlikely ());
2194
2195	if (!op1_small_p)
2196	do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2197	NULL_RTX, NULL, small_op0_large_op1,
2198	profile_probability::unlikely ());
2199
2200	/* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2201	hmode to mode, the multiplication will never overflow. We can
2202	do just one hmode x hmode => mode widening multiplication. */
2203	tree halfstype = build_nonstandard_integer_type (hprec, uns);
2204	ops.op0 = make_tree (halfstype, lopart0);
2205	ops.op1 = make_tree (halfstype, lopart1);
2206	ops.code = WIDEN_MULT_EXPR;
2207	ops.type = type;
2208	rtx thisres
2209	= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2210	emit_move_insn (res, thisres);
2211	emit_jump (done_label);
2212
2213	emit_label (small_op0_large_op1);
2214
2215	/* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2216	but op1 is not, just swap the arguments and handle it as op1
2217	sign/zero extended, op0 not. */
2218	rtx larger = gen_reg_rtx (mode);
2219	rtx hipart = gen_reg_rtx (hmode);
2220	rtx lopart = gen_reg_rtx (hmode);
2221	emit_move_insn (larger, op1);
2222	emit_move_insn (hipart, hipart1);
2223	emit_move_insn (lopart, lopart0);
2224	emit_jump (one_small_one_large);
2225
2226	emit_label (large_op0);
2227
2228	if (!op1_small_p)
2229	do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2230	NULL_RTX, NULL, both_ops_large,
2231	profile_probability::unlikely ());
2232
2233	/* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2234	but op0 is not, prepare larger, hipart and lopart pseudos and
2235	handle it together with small_op0_large_op1. */
2236	emit_move_insn (larger, op0);
2237	emit_move_insn (hipart, hipart0);
2238	emit_move_insn (lopart, lopart1);
2239
2240	emit_label (one_small_one_large);
2241
2242	/* lopart is the low part of the operand that is sign extended
2243	to mode, larger is the other operand, hipart is the
2244	high part of larger and lopart0 and lopart1 are the low parts
2245	of both operands.
2246	We perform lopart0 * lopart1 and lopart * hipart widening
2247	multiplications. */
2248	tree halfutype = build_nonstandard_integer_type (hprec, 1);
2249	ops.op0 = make_tree (halfutype, lopart0);
2250	ops.op1 = make_tree (halfutype, lopart1);
2251	rtx lo0xlo1
2252	= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2253
2254	ops.op0 = make_tree (halfutype, lopart);
2255	ops.op1 = make_tree (halfutype, hipart);
2256	rtx loxhi = gen_reg_rtx (mode);
2257	rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2258	emit_move_insn (loxhi, tem);
2259
2260	if (!uns)
2261	{
2262	/* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2263	if (larger_sign == 0)
2264	emit_jump (after_hipart_neg);
2265	else if (larger_sign != -1)
2266	do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2267	NULL_RTX, NULL, after_hipart_neg,
2268	profile_probability::even ());
2269
2270	tem = convert_modes (mode, oldmode: hmode, x: lopart, unsignedp: 1);
2271	tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2272	tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2273	1, OPTAB_WIDEN);
2274	emit_move_insn (loxhi, tem);
2275
2276	emit_label (after_hipart_neg);
2277
2278	/* if (lopart < 0) loxhi -= larger; */
2279	if (smaller_sign == 0)
2280	emit_jump (after_lopart_neg);
2281	else if (smaller_sign != -1)
2282	do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2283	NULL_RTX, NULL, after_lopart_neg,
2284	profile_probability::even ());
2285
2286	tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2287	1, OPTAB_WIDEN);
2288	emit_move_insn (loxhi, tem);
2289
2290	emit_label (after_lopart_neg);
2291	}
2292
2293	/* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2294	tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2295	tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2296	1, OPTAB_WIDEN);
2297	emit_move_insn (loxhi, tem);
2298
2299	/* if (loxhi >> (bitsize / 2)
2300	== (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2301	if (loxhi >> (bitsize / 2) == 0 (if uns). */
2302	rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2303	NULL_RTX, 0);
2304	hipartloxhi = convert_modes (mode: hmode, oldmode: mode, x: hipartloxhi, unsignedp: 0);
2305	rtx signbitloxhi = const0_rtx;
2306	if (!uns)
2307	signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2308	convert_modes (mode: hmode, oldmode: mode,
2309	x: loxhi, unsignedp: 0),
2310	hprec - 1, NULL_RTX, 0);
2311
2312	do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2313	NULL_RTX, NULL, do_overflow,
2314	profile_probability::very_unlikely ());
2315
2316	/* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2317	rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2318	NULL_RTX, 1);
2319	tem = convert_modes (mode, oldmode: hmode,
2320	x: convert_modes (mode: hmode, oldmode: mode, x: lo0xlo1, unsignedp: 1), unsignedp: 1);
2321
2322	tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2323	1, OPTAB_WIDEN);
2324	if (tem != res)
2325	emit_move_insn (res, tem);
2326	emit_jump (done_label);
2327
2328	emit_label (both_ops_large);
2329
2330	/* If both operands are large (not sign (!uns) or zero (uns)
2331	extended from hmode), then perform the full multiplication
2332	which will be the result of the operation.
2333	The only cases which don't overflow are for signed multiplication
2334	some cases where both hipart0 and highpart1 are 0 or -1.
2335	For unsigned multiplication when high parts are both non-zero
2336	this overflows always. */
2337	ops.code = MULT_EXPR;
2338	ops.op0 = make_tree (type, op0);
2339	ops.op1 = make_tree (type, op1);
2340	tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2341	emit_move_insn (res, tem);
2342
2343	if (!uns)
2344	{
2345	if (!op0_medium_p)
2346	{
2347	tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2348	NULL_RTX, 1, OPTAB_WIDEN);
2349	do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2350	NULL_RTX, NULL, do_error,
2351	profile_probability::very_unlikely ());
2352	}
2353
2354	if (!op1_medium_p)
2355	{
2356	tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2357	NULL_RTX, 1, OPTAB_WIDEN);
2358	do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2359	NULL_RTX, NULL, do_error,
2360	profile_probability::very_unlikely ());
2361	}
2362
2363	/* At this point hipart{0,1} are both in [-1, 0]. If they are
2364	the same, overflow happened if res is non-positive, if they
2365	are different, overflow happened if res is positive. */
2366	if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2367	emit_jump (hipart_different);
2368	else if (op0_sign == 1 || op1_sign == 1)
2369	do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2370	NULL_RTX, NULL, hipart_different,
2371	profile_probability::even ());
2372
2373	do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2374	NULL_RTX, NULL, do_error,
2375	profile_probability::very_unlikely ());
2376	emit_jump (done_label);
2377
2378	emit_label (hipart_different);
2379
2380	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2381	NULL_RTX, NULL, do_error,
2382	profile_probability::very_unlikely ());
2383	emit_jump (done_label);
2384	}
2385
2386	emit_label (do_overflow);
2387
2388	/* Overflow, do full multiplication and fallthru into do_error. */
2389	ops.op0 = make_tree (type, op0);
2390	ops.op1 = make_tree (type, op1);
2391	tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2392	emit_move_insn (res, tem);
2393	}
2394	else if (GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wmode)
2395	&& targetm.scalar_mode_supported_p (wmode))
2396	/* Even emitting a libcall is better than not detecting overflow
2397	at all. */
2398	goto twoxwider;
2399	else
2400	{
2401	gcc_assert (!is_ubsan);
2402	ops.code = MULT_EXPR;
2403	ops.type = type;
2404	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2405	emit_jump (done_label);
2406	}
2407	}
2408
2409	do_error_label:
2410	emit_label (do_error);
2411	if (is_ubsan)
2412	{
2413	/* Expand the ubsan builtin call. */
2414	push_temp_slots ();
2415	fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2416	arg0, arg1, datap);
2417	expand_normal (exp: fn);
2418	pop_temp_slots ();
2419	do_pending_stack_adjust ();
2420	}
2421	else if (lhs)
2422	expand_arith_set_overflow (lhs, target);
2423
2424	/* We're done. */
2425	emit_label (done_label);
2426
2427	/* u1 * u2 -> sr */
2428	if (uns0_p && uns1_p && !unsr_p)
2429	{
2430	rtx_code_label *all_done_label = gen_label_rtx ();
2431	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2432	NULL, all_done_label, profile_probability::very_likely ());
2433	expand_arith_set_overflow (lhs, target);
2434	emit_label (all_done_label);
2435	}
2436
2437	/* s1 * u2 -> sr */
2438	if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2439	{
2440	rtx_code_label *all_done_label = gen_label_rtx ();
2441	rtx_code_label *set_noovf = gen_label_rtx ();
2442	do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2443	NULL, all_done_label, profile_probability::very_likely ());
2444	expand_arith_set_overflow (lhs, target);
2445	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2446	NULL, set_noovf, profile_probability::very_likely ());
2447	do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2448	NULL, all_done_label, profile_probability::very_unlikely ());
2449	do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2450	all_done_label, profile_probability::very_unlikely ());
2451	emit_label (set_noovf);
2452	write_complex_part (target, const0_rtx, true, false);
2453	emit_label (all_done_label);
2454	}
2455
2456	if (lhs)
2457	{
2458	if (is_ubsan)
2459	expand_ubsan_result_store (lhs, target, mode, res, do_error);
2460	else
2461	expand_arith_overflow_result_store (lhs, target, mode, res);
2462	}
2463	}
2464
2465	/* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2466
2467	static void
2468	expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2469	tree arg0, tree arg1)
2470	{
2471	poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2472	rtx_code_label *loop_lab = NULL;
2473	rtx cntvar = NULL_RTX;
2474	tree cntv = NULL_TREE;
2475	tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2476	tree sz = TYPE_SIZE (eltype);
2477	tree data = NULL_TREE;
2478	tree resv = NULL_TREE;
2479	rtx lhsr = NULL_RTX;
2480	rtx resvr = NULL_RTX;
2481	unsigned HOST_WIDE_INT const_cnt = 0;
2482	bool use_loop_p = (!cnt.is_constant (const_value: &const_cnt) || const_cnt > 4);
2483
2484	if (lhs)
2485	{
2486	optab op;
2487	lhsr = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2488	if (!VECTOR_MODE_P (GET_MODE (lhsr))
2489	|| (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2490	optab_default)) == unknown_optab
2491	|| (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2492	== CODE_FOR_nothing))
2493	{
2494	if (MEM_P (lhsr))
2495	resv = make_tree (TREE_TYPE (lhs), lhsr);
2496	else
2497	{
2498	resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2499	resv = make_tree (TREE_TYPE (lhs), resvr);
2500	}
2501	}
2502	}
2503	if (use_loop_p)
2504	{
2505	do_pending_stack_adjust ();
2506	loop_lab = gen_label_rtx ();
2507	cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2508	cntv = make_tree (sizetype, cntvar);
2509	emit_move_insn (cntvar, const0_rtx);
2510	emit_label (loop_lab);
2511	}
2512	if (TREE_CODE (arg0) != VECTOR_CST)
2513	{
2514	rtx arg0r = expand_normal (exp: arg0);
2515	arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2516	}
2517	if (TREE_CODE (arg1) != VECTOR_CST)
2518	{
2519	rtx arg1r = expand_normal (exp: arg1);
2520	arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2521	}
2522	for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2523	{
2524	tree op0, op1, res = NULL_TREE;
2525	if (use_loop_p)
2526	{
2527	tree atype = build_array_type_nelts (eltype, cnt);
2528	op0 = uniform_vector_p (arg0);
2529	if (op0 == NULL_TREE)
2530	{
2531	op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2532	op0 = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: op0, arg1: cntv,
2533	NULL_TREE, NULL_TREE);
2534	}
2535	op1 = uniform_vector_p (arg1);
2536	if (op1 == NULL_TREE)
2537	{
2538	op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2539	op1 = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: op1, arg1: cntv,
2540	NULL_TREE, NULL_TREE);
2541	}
2542	if (resv)
2543	{
2544	res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2545	res = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: res, arg1: cntv,
2546	NULL_TREE, NULL_TREE);
2547	}
2548	}
2549	else
2550	{
2551	tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2552	op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2553	op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2554	if (resv)
2555	res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2556	bitpos);
2557	}
2558	switch (code)
2559	{
2560	case PLUS_EXPR:
2561	expand_addsub_overflow (loc, code: PLUS_EXPR, lhs: res, arg0: op0, arg1: op1,
2562	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, datap: &data);
2563	break;
2564	case MINUS_EXPR:
2565	if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2566	expand_neg_overflow (loc, lhs: res, arg1: op1, is_ubsan: true, datap: &data);
2567	else
2568	expand_addsub_overflow (loc, code: MINUS_EXPR, lhs: res, arg0: op0, arg1: op1,
2569	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, datap: &data);
2570	break;
2571	case MULT_EXPR:
2572	expand_mul_overflow (loc, lhs: res, arg0: op0, arg1: op1, unsr_p: false, uns0_p: false, uns1_p: false,
2573	is_ubsan: true, datap: &data);
2574	break;
2575	default:
2576	gcc_unreachable ();
2577	}
2578	}
2579	if (use_loop_p)
2580	{
2581	struct separate_ops ops;
2582	ops.code = PLUS_EXPR;
2583	ops.type = TREE_TYPE (cntv);
2584	ops.op0 = cntv;
2585	ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2586	ops.op2 = NULL_TREE;
2587	ops.location = loc;
2588	rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2589	EXPAND_NORMAL);
2590	if (ret != cntvar)
2591	emit_move_insn (cntvar, ret);
2592	rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2593	do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2594	TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2595	profile_probability::very_likely ());
2596	}
2597	if (lhs && resv == NULL_TREE)
2598	{
2599	struct separate_ops ops;
2600	ops.code = code;
2601	ops.type = TREE_TYPE (arg0);
2602	ops.op0 = arg0;
2603	ops.op1 = arg1;
2604	ops.op2 = NULL_TREE;
2605	ops.location = loc;
2606	rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2607	EXPAND_NORMAL);
2608	if (ret != lhsr)
2609	emit_move_insn (lhsr, ret);
2610	}
2611	else if (resvr)
2612	emit_move_insn (lhsr, resvr);
2613	}
2614
2615	/* Expand UBSAN_CHECK_ADD call STMT. */
2616
2617	static void
2618	expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2619	{
2620	location_t loc = gimple_location (g: stmt);
2621	tree lhs = gimple_call_lhs (gs: stmt);
2622	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2623	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2624	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2625	expand_vector_ubsan_overflow (loc, code: PLUS_EXPR, lhs, arg0, arg1);
2626	else
2627	expand_addsub_overflow (loc, code: PLUS_EXPR, lhs, arg0, arg1,
2628	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, NULL);
2629	}
2630
2631	/* Expand UBSAN_CHECK_SUB call STMT. */
2632
2633	static void
2634	expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2635	{
2636	location_t loc = gimple_location (g: stmt);
2637	tree lhs = gimple_call_lhs (gs: stmt);
2638	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2639	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2640	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2641	expand_vector_ubsan_overflow (loc, code: MINUS_EXPR, lhs, arg0, arg1);
2642	else if (integer_zerop (arg0))
2643	expand_neg_overflow (loc, lhs, arg1, is_ubsan: true, NULL);
2644	else
2645	expand_addsub_overflow (loc, code: MINUS_EXPR, lhs, arg0, arg1,
2646	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, NULL);
2647	}
2648
2649	/* Expand UBSAN_CHECK_MUL call STMT. */
2650
2651	static void
2652	expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2653	{
2654	location_t loc = gimple_location (g: stmt);
2655	tree lhs = gimple_call_lhs (gs: stmt);
2656	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2657	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2658	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2659	expand_vector_ubsan_overflow (loc, code: MULT_EXPR, lhs, arg0, arg1);
2660	else
2661	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true,
2662	NULL);
2663	}
2664
2665	/* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2666
2667	static void
2668	expand_arith_overflow (enum tree_code code, gimple *stmt)
2669	{
2670	tree lhs = gimple_call_lhs (gs: stmt);
2671	if (lhs == NULL_TREE)
2672	return;
2673	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2674	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2675	tree type = TREE_TYPE (TREE_TYPE (lhs));
2676	int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2677	int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2678	int unsr_p = TYPE_UNSIGNED (type);
2679	int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2680	int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2681	int precres = TYPE_PRECISION (type);
2682	location_t loc = gimple_location (g: stmt);
2683	if (!uns0_p && get_range_pos_neg (arg0) == 1)
2684	uns0_p = true;
2685	if (!uns1_p && get_range_pos_neg (arg1) == 1)
2686	uns1_p = true;
2687	int pr = get_min_precision (arg: arg0, sign: uns0_p ? UNSIGNED : SIGNED);
2688	prec0 = MIN (prec0, pr);
2689	pr = get_min_precision (arg: arg1, sign: uns1_p ? UNSIGNED : SIGNED);
2690	prec1 = MIN (prec1, pr);
2691
2692	/* If uns0_p && uns1_p, precop is minimum needed precision
2693	of unsigned type to hold the exact result, otherwise
2694	precop is minimum needed precision of signed type to
2695	hold the exact result. */
2696	int precop;
2697	if (code == MULT_EXPR)
2698	precop = prec0 + prec1 + (uns0_p != uns1_p);
2699	else
2700	{
2701	if (uns0_p == uns1_p)
2702	precop = MAX (prec0, prec1) + 1;
2703	else if (uns0_p)
2704	precop = MAX (prec0 + 1, prec1) + 1;
2705	else
2706	precop = MAX (prec0, prec1 + 1) + 1;
2707	}
2708	int orig_precres = precres;
2709
2710	do
2711	{
2712	if ((uns0_p && uns1_p)
2713	? ((precop + !unsr_p) <= precres
2714	/* u1 - u2 -> ur can overflow, no matter what precision
2715	the result has. */
2716	&& (code != MINUS_EXPR || !unsr_p))
2717	: (!unsr_p && precop <= precres))
2718	{
2719	/* The infinity precision result will always fit into result. */
2720	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2721	write_complex_part (target, const0_rtx, true, false);
2722	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2723	struct separate_ops ops;
2724	ops.code = code;
2725	ops.type = type;
2726	ops.op0 = fold_convert_loc (loc, type, arg0);
2727	ops.op1 = fold_convert_loc (loc, type, arg1);
2728	ops.op2 = NULL_TREE;
2729	ops.location = loc;
2730	rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2731	expand_arith_overflow_result_store (lhs, target, mode, res: tem);
2732	return;
2733	}
2734
2735	/* For operations with low precision, if target doesn't have them, start
2736	with precres widening right away, otherwise do it only if the most
2737	simple cases can't be used. */
2738	const int min_precision = targetm.min_arithmetic_precision ();
2739	if (orig_precres == precres && precres < min_precision)
2740	;
2741	else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2742	&& prec1 <= precres)
2743	|| ((!uns0_p || !uns1_p) && !unsr_p
2744	&& prec0 + uns0_p <= precres
2745	&& prec1 + uns1_p <= precres))
2746	{
2747	arg0 = fold_convert_loc (loc, type, arg0);
2748	arg1 = fold_convert_loc (loc, type, arg1);
2749	switch (code)
2750	{
2751	case MINUS_EXPR:
2752	if (integer_zerop (arg0) && !unsr_p)
2753	{
2754	expand_neg_overflow (loc, lhs, arg1, is_ubsan: false, NULL);
2755	return;
2756	}
2757	/* FALLTHRU */
2758	case PLUS_EXPR:
2759	expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2760	uns0_p: unsr_p, uns1_p: unsr_p, is_ubsan: false, NULL);
2761	return;
2762	case MULT_EXPR:
2763	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2764	uns0_p: unsr_p, uns1_p: unsr_p, is_ubsan: false, NULL);
2765	return;
2766	default:
2767	gcc_unreachable ();
2768	}
2769	}
2770
2771	/* For sub-word operations, retry with a wider type first. */
2772	if (orig_precres == precres && precop <= BITS_PER_WORD)
2773	{
2774	int p = MAX (min_precision, precop);
2775	scalar_int_mode m = smallest_int_mode_for_size (size: p);
2776	tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (mode: m),
2777	uns0_p && uns1_p
2778	&& unsr_p);
2779	p = TYPE_PRECISION (optype);
2780	if (p > precres)
2781	{
2782	precres = p;
2783	unsr_p = TYPE_UNSIGNED (optype);
2784	type = optype;
2785	continue;
2786	}
2787	}
2788
2789	if (prec0 <= precres && prec1 <= precres)
2790	{
2791	tree types[2];
2792	if (unsr_p)
2793	{
2794	types[0] = build_nonstandard_integer_type (precres, 0);
2795	types[1] = type;
2796	}
2797	else
2798	{
2799	types[0] = type;
2800	types[1] = build_nonstandard_integer_type (precres, 1);
2801	}
2802	arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2803	arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2804	if (code != MULT_EXPR)
2805	expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2806	uns0_p, uns1_p, is_ubsan: false, NULL);
2807	else
2808	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2809	uns0_p, uns1_p, is_ubsan: false, NULL);
2810	return;
2811	}
2812
2813	/* Retry with a wider type. */
2814	if (orig_precres == precres)
2815	{
2816	int p = MAX (prec0, prec1);
2817	scalar_int_mode m = smallest_int_mode_for_size (size: p);
2818	tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (mode: m),
2819	uns0_p && uns1_p
2820	&& unsr_p);
2821	p = TYPE_PRECISION (optype);
2822	if (p > precres)
2823	{
2824	precres = p;
2825	unsr_p = TYPE_UNSIGNED (optype);
2826	type = optype;
2827	continue;
2828	}
2829	}
2830
2831	gcc_unreachable ();
2832	}
2833	while (1);
2834	}
2835
2836	/* Expand ADD_OVERFLOW STMT. */
2837
2838	static void
2839	expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2840	{
2841	expand_arith_overflow (code: PLUS_EXPR, stmt);
2842	}
2843
2844	/* Expand SUB_OVERFLOW STMT. */
2845
2846	static void
2847	expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2848	{
2849	expand_arith_overflow (code: MINUS_EXPR, stmt);
2850	}
2851
2852	/* Expand MUL_OVERFLOW STMT. */
2853
2854	static void
2855	expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2856	{
2857	expand_arith_overflow (code: MULT_EXPR, stmt);
2858	}
2859
2860	/* Expand UADDC STMT. */
2861
2862	static void
2863	expand_UADDC (internal_fn ifn, gcall *stmt)
2864	{
2865	tree lhs = gimple_call_lhs (gs: stmt);
2866	tree arg1 = gimple_call_arg (gs: stmt, index: 0);
2867	tree arg2 = gimple_call_arg (gs: stmt, index: 1);
2868	tree arg3 = gimple_call_arg (gs: stmt, index: 2);
2869	tree type = TREE_TYPE (arg1);
2870	machine_mode mode = TYPE_MODE (type);
2871	insn_code icode = optab_handler (op: ifn == IFN_UADDC
2872	? uaddc5_optab : usubc5_optab, mode);
2873	rtx op1 = expand_normal (exp: arg1);
2874	rtx op2 = expand_normal (exp: arg2);
2875	rtx op3 = expand_normal (exp: arg3);
2876	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2877	rtx re = gen_reg_rtx (mode);
2878	rtx im = gen_reg_rtx (mode);
2879	class expand_operand ops[5];
2880	create_output_operand (op: &ops[0], x: re, mode);
2881	create_output_operand (op: &ops[1], x: im, mode);
2882	create_input_operand (op: &ops[2], value: op1, mode);
2883	create_input_operand (op: &ops[3], value: op2, mode);
2884	create_input_operand (op: &ops[4], value: op3, mode);
2885	expand_insn (icode, nops: 5, ops);
2886	write_complex_part (target, re, false, false);
2887	write_complex_part (target, im, true, false);
2888	}
2889
2890	/* Expand USUBC STMT. */
2891
2892	static void
2893	expand_USUBC (internal_fn ifn, gcall *stmt)
2894	{
2895	expand_UADDC (ifn, stmt);
2896	}
2897
2898	/* This should get folded in tree-vectorizer.cc. */
2899
2900	static void
2901	expand_LOOP_VECTORIZED (internal_fn, gcall *)
2902	{
2903	gcc_unreachable ();
2904	}
2905
2906	/* This should get folded in tree-vectorizer.cc. */
2907
2908	static void
2909	expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2910	{
2911	gcc_unreachable ();
2912	}
2913
2914	/* Return a memory reference of type TYPE for argument INDEX of STMT.
2915	Use argument INDEX + 1 to derive the second (TBAA) operand. */
2916
2917	static tree
2918	expand_call_mem_ref (tree type, gcall *stmt, int index)
2919	{
2920	tree addr = gimple_call_arg (gs: stmt, index);
2921	tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2922	unsigned int align = tree_to_shwi (gimple_call_arg (gs: stmt, index: index + 1));
2923	if (TYPE_ALIGN (type) != align)
2924	type = build_aligned_type (type, align);
2925
2926	tree tmp = addr;
2927	if (TREE_CODE (tmp) == SSA_NAME)
2928	{
2929	gimple *def = SSA_NAME_DEF_STMT (tmp);
2930	if (gimple_assign_single_p (gs: def))
2931	tmp = gimple_assign_rhs1 (gs: def);
2932	}
2933
2934	if (TREE_CODE (tmp) == ADDR_EXPR)
2935	{
2936	tree mem = TREE_OPERAND (tmp, 0);
2937	if (TREE_CODE (mem) == TARGET_MEM_REF
2938	&& types_compatible_p (TREE_TYPE (mem), type2: type))
2939	{
2940	tree offset = TMR_OFFSET (mem);
2941	if (type != TREE_TYPE (mem)
2942	|| alias_ptr_type != TREE_TYPE (offset)
2943	|| !integer_zerop (offset))
2944	{
2945	mem = copy_node (mem);
2946	TMR_OFFSET (mem) = wide_int_to_tree (type: alias_ptr_type,
2947	cst: wi::to_poly_wide (t: offset));
2948	TREE_TYPE (mem) = type;
2949	}
2950	return mem;
2951	}
2952	}
2953
2954	return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2955	}
2956
2957	/* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
2958	* OPTAB. */
2959
2960	static void
2961	expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
2962	{
2963	int i = 0;
2964	class expand_operand ops[5];
2965	tree type, lhs, rhs, maskt;
2966	rtx mem, target;
2967	insn_code icode;
2968
2969	maskt = gimple_call_arg (gs: stmt, index: internal_fn_mask_index (ifn));
2970	lhs = gimple_call_lhs (gs: stmt);
2971	if (lhs == NULL_TREE)
2972	return;
2973	type = TREE_TYPE (lhs);
2974	rhs = expand_call_mem_ref (type, stmt, index: 0);
2975
2976	if (optab == vec_mask_load_lanes_optab
2977	|| optab == vec_mask_len_load_lanes_optab)
2978	icode = get_multi_vector_move (array_type: type, optab);
2979	else if (optab == len_load_optab)
2980	icode = direct_optab_handler (op: optab, TYPE_MODE (type));
2981	else
2982	icode = convert_optab_handler (op: optab, TYPE_MODE (type),
2983	TYPE_MODE (TREE_TYPE (maskt)));
2984
2985	mem = expand_expr (exp: rhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2986	gcc_assert (MEM_P (mem));
2987	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2988	create_output_operand (op: &ops[i++], x: target, TYPE_MODE (type));
2989	create_fixed_operand (op: &ops[i++], x: mem);
2990	i = add_mask_and_len_args (ops, opno: i, stmt);
2991	expand_insn (icode, nops: i, ops);
2992
2993	if (!rtx_equal_p (target, ops[0].value))
2994	emit_move_insn (target, ops[0].value);
2995	}
2996
2997	#define expand_mask_load_optab_fn expand_partial_load_optab_fn
2998	#define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2999	#define expand_len_load_optab_fn expand_partial_load_optab_fn
3000	#define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
3001
3002	/* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3003	* OPTAB. */
3004
3005	static void
3006	expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
3007	{
3008	int i = 0;
3009	class expand_operand ops[5];
3010	tree type, lhs, rhs, maskt;
3011	rtx mem, reg;
3012	insn_code icode;
3013
3014	maskt = gimple_call_arg (gs: stmt, index: internal_fn_mask_index (ifn));
3015	rhs = gimple_call_arg (gs: stmt, index: internal_fn_stored_value_index (ifn));
3016	type = TREE_TYPE (rhs);
3017	lhs = expand_call_mem_ref (type, stmt, index: 0);
3018
3019	if (optab == vec_mask_store_lanes_optab
3020	|| optab == vec_mask_len_store_lanes_optab)
3021	icode = get_multi_vector_move (array_type: type, optab);
3022	else if (optab == len_store_optab)
3023	icode = direct_optab_handler (op: optab, TYPE_MODE (type));
3024	else
3025	icode = convert_optab_handler (op: optab, TYPE_MODE (type),
3026	TYPE_MODE (TREE_TYPE (maskt)));
3027
3028	mem = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3029	gcc_assert (MEM_P (mem));
3030	reg = expand_normal (exp: rhs);
3031	create_fixed_operand (op: &ops[i++], x: mem);
3032	create_input_operand (op: &ops[i++], value: reg, TYPE_MODE (type));
3033	i = add_mask_and_len_args (ops, opno: i, stmt);
3034	expand_insn (icode, nops: i, ops);
3035	}
3036
3037	#define expand_mask_store_optab_fn expand_partial_store_optab_fn
3038	#define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3039	#define expand_len_store_optab_fn expand_partial_store_optab_fn
3040	#define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3041
3042	/* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3043	The expansion of STMT happens based on OPTAB table associated. */
3044
3045	static void
3046	expand_vec_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3047	{
3048	class expand_operand ops[6];
3049	insn_code icode;
3050	tree lhs = gimple_call_lhs (gs: stmt);
3051	tree op0a = gimple_call_arg (gs: stmt, index: 0);
3052	tree op0b = gimple_call_arg (gs: stmt, index: 1);
3053	tree op1 = gimple_call_arg (gs: stmt, index: 2);
3054	tree op2 = gimple_call_arg (gs: stmt, index: 3);
3055	enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (gs: stmt, index: 4));
3056
3057	tree vec_cond_type = TREE_TYPE (lhs);
3058	tree op_mode = TREE_TYPE (op0a);
3059	bool unsignedp = TYPE_UNSIGNED (op_mode);
3060
3061	machine_mode mode = TYPE_MODE (vec_cond_type);
3062	machine_mode cmp_op_mode = TYPE_MODE (op_mode);
3063
3064	icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: cmp_op_mode);
3065	rtx comparison
3066	= vector_compare_rtx (VOIDmode, tcode, t_op0: op0a, t_op1: op0b, unsignedp, icode, opno: 4);
3067	/* vector_compare_rtx legitimizes operands, preserve equality when
3068	expanding op1/op2. */
3069	rtx rtx_op1, rtx_op2;
3070	if (operand_equal_p (op1, op0a))
3071	rtx_op1 = XEXP (comparison, 0);
3072	else if (operand_equal_p (op1, op0b))
3073	rtx_op1 = XEXP (comparison, 1);
3074	else
3075	rtx_op1 = expand_normal (exp: op1);
3076	if (operand_equal_p (op2, op0a))
3077	rtx_op2 = XEXP (comparison, 0);
3078	else if (operand_equal_p (op2, op0b))
3079	rtx_op2 = XEXP (comparison, 1);
3080	else
3081	rtx_op2 = expand_normal (exp: op2);
3082
3083	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3084	create_output_operand (op: &ops[0], x: target, mode);
3085	create_input_operand (op: &ops[1], value: rtx_op1, mode);
3086	create_input_operand (op: &ops[2], value: rtx_op2, mode);
3087	create_fixed_operand (op: &ops[3], x: comparison);
3088	create_fixed_operand (op: &ops[4], XEXP (comparison, 0));
3089	create_fixed_operand (op: &ops[5], XEXP (comparison, 1));
3090	expand_insn (icode, nops: 6, ops);
3091	if (!rtx_equal_p (ops[0].value, target))
3092	emit_move_insn (target, ops[0].value);
3093	}
3094
3095	/* Expand VCOND_MASK optab internal function.
3096	The expansion of STMT happens based on OPTAB table associated. */
3097
3098	static void
3099	expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3100	{
3101	class expand_operand ops[4];
3102
3103	tree lhs = gimple_call_lhs (gs: stmt);
3104	tree op0 = gimple_call_arg (gs: stmt, index: 0);
3105	tree op1 = gimple_call_arg (gs: stmt, index: 1);
3106	tree op2 = gimple_call_arg (gs: stmt, index: 2);
3107	tree vec_cond_type = TREE_TYPE (lhs);
3108
3109	machine_mode mode = TYPE_MODE (vec_cond_type);
3110	machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
3111	enum insn_code icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: mask_mode);
3112	rtx mask, rtx_op1, rtx_op2;
3113
3114	gcc_assert (icode != CODE_FOR_nothing);
3115
3116	mask = expand_normal (exp: op0);
3117	rtx_op1 = expand_normal (exp: op1);
3118	rtx_op2 = expand_normal (exp: op2);
3119
3120	mask = force_reg (mask_mode, mask);
3121	rtx_op1 = force_reg (mode, rtx_op1);
3122
3123	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3124	create_output_operand (op: &ops[0], x: target, mode);
3125	create_input_operand (op: &ops[1], value: rtx_op1, mode);
3126	create_input_operand (op: &ops[2], value: rtx_op2, mode);
3127	create_input_operand (op: &ops[3], value: mask, mode: mask_mode);
3128	expand_insn (icode, nops: 4, ops);
3129	if (!rtx_equal_p (ops[0].value, target))
3130	emit_move_insn (target, ops[0].value);
3131	}
3132
3133	/* Expand VEC_SET internal functions. */
3134
3135	static void
3136	expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3137	{
3138	tree lhs = gimple_call_lhs (gs: stmt);
3139	tree op0 = gimple_call_arg (gs: stmt, index: 0);
3140	tree op1 = gimple_call_arg (gs: stmt, index: 1);
3141	tree op2 = gimple_call_arg (gs: stmt, index: 2);
3142	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3143	rtx src = expand_normal (exp: op0);
3144
3145	machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
3146	scalar_mode innermode = GET_MODE_INNER (outermode);
3147
3148	rtx value = expand_normal (exp: op1);
3149	rtx pos = expand_normal (exp: op2);
3150
3151	class expand_operand ops[3];
3152	enum insn_code icode = optab_handler (op: optab, mode: outermode);
3153
3154	if (icode != CODE_FOR_nothing)
3155	{
3156	rtx temp = gen_reg_rtx (outermode);
3157	emit_move_insn (temp, src);
3158
3159	create_fixed_operand (op: &ops[0], x: temp);
3160	create_input_operand (op: &ops[1], value, mode: innermode);
3161	create_convert_operand_from (op: &ops[2], value: pos, TYPE_MODE (TREE_TYPE (op2)),
3162	unsigned_p: true);
3163	if (maybe_expand_insn (icode, nops: 3, ops))
3164	{
3165	emit_move_insn (target, temp);
3166	return;
3167	}
3168	}
3169	gcc_unreachable ();
3170	}
3171
3172	static void
3173	expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
3174	{
3175	}
3176
3177	static void
3178	expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
3179	{
3180	/* When guessing was done, the hints should be already stripped away. */
3181	gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
3182
3183	rtx target;
3184	tree lhs = gimple_call_lhs (gs: stmt);
3185	if (lhs)
3186	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3187	else
3188	target = const0_rtx;
3189	rtx val = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), target, VOIDmode, modifier: EXPAND_NORMAL);
3190	if (lhs && val != target)
3191	emit_move_insn (target, val);
3192	}
3193
3194	/* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3195	should never be called. */
3196
3197	static void
3198	expand_VA_ARG (internal_fn, gcall *)
3199	{
3200	gcc_unreachable ();
3201	}
3202
3203	/* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3204	dummy function should never be called. */
3205
3206	static void
3207	expand_VEC_CONVERT (internal_fn, gcall *)
3208	{
3209	gcc_unreachable ();
3210	}
3211
3212	/* Expand IFN_RAWMEMCHR internal function. */
3213
3214	void
3215	expand_RAWMEMCHR (internal_fn, gcall *stmt)
3216	{
3217	expand_operand ops[3];
3218
3219	tree lhs = gimple_call_lhs (gs: stmt);
3220	if (!lhs)
3221	return;
3222	machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
3223	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3224	create_output_operand (op: &ops[0], x: lhs_rtx, mode: lhs_mode);
3225
3226	tree mem = gimple_call_arg (gs: stmt, index: 0);
3227	rtx mem_rtx = get_memory_rtx (exp: mem, NULL);
3228	create_fixed_operand (op: &ops[1], x: mem_rtx);
3229
3230	tree pattern = gimple_call_arg (gs: stmt, index: 1);
3231	machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
3232	rtx pattern_rtx = expand_normal (exp: pattern);
3233	create_input_operand (op: &ops[2], value: pattern_rtx, mode);
3234
3235	insn_code icode = direct_optab_handler (op: rawmemchr_optab, mode);
3236
3237	expand_insn (icode, nops: 3, ops);
3238	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3239	emit_move_insn (lhs_rtx, ops[0].value);
3240	}
3241
3242	/* Expand the IFN_UNIQUE function according to its first argument. */
3243
3244	static void
3245	expand_UNIQUE (internal_fn, gcall *stmt)
3246	{
3247	rtx pattern = NULL_RTX;
3248	enum ifn_unique_kind kind
3249	= (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
3250
3251	switch (kind)
3252	{
3253	default:
3254	gcc_unreachable ();
3255
3256	case IFN_UNIQUE_UNSPEC:
3257	if (targetm.have_unique ())
3258	pattern = targetm.gen_unique ();
3259	break;
3260
3261	case IFN_UNIQUE_OACC_FORK:
3262	case IFN_UNIQUE_OACC_JOIN:
3263	if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3264	{
3265	tree lhs = gimple_call_lhs (gs: stmt);
3266	rtx target = const0_rtx;
3267
3268	if (lhs)
3269	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3270
3271	rtx data_dep = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
3272	rtx axis = expand_normal (exp: gimple_call_arg (gs: stmt, index: 2));
3273
3274	if (kind == IFN_UNIQUE_OACC_FORK)
3275	pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3276	else
3277	pattern = targetm.gen_oacc_join (target, data_dep, axis);
3278	}
3279	else
3280	gcc_unreachable ();
3281	break;
3282	}
3283
3284	if (pattern)
3285	emit_insn (pattern);
3286	}
3287
3288	/* Expand the IFN_DEFERRED_INIT function:
3289	LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3290
3291	Initialize the LHS with zero/pattern according to its second argument
3292	INIT_TYPE:
3293	if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3294	if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3295	to initialize;
3296	The LHS variable is initialized including paddings.
3297	The reasons to choose 0xFE for pattern initialization are:
3298	1. It is a non-canonical virtual address on x86_64, and at the
3299	high end of the i386 kernel address space.
3300	2. It is a very large float value (-1.694739530317379e+38).
3301	3. It is also an unusual number for integers. */
3302	#define INIT_PATTERN_VALUE 0xFE
3303	static void
3304	expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3305	{
3306	tree lhs = gimple_call_lhs (gs: stmt);
3307	tree var_size = gimple_call_arg (gs: stmt, index: 0);
3308	enum auto_init_type init_type
3309	= (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3310	bool reg_lhs = true;
3311
3312	tree var_type = TREE_TYPE (lhs);
3313	gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3314
3315	if (TREE_CODE (lhs) == SSA_NAME)
3316	reg_lhs = true;
3317	else
3318	{
3319	tree lhs_base = lhs;
3320	while (handled_component_p (t: lhs_base))
3321	lhs_base = TREE_OPERAND (lhs_base, 0);
3322	reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3323	|| non_mem_decl_p (lhs_base));
3324	/* If this expands to a register and the underlying decl is wrapped in
3325	a MEM_REF that just serves as an access type change expose the decl
3326	if it is of correct size. This avoids a situation as in PR103271
3327	if the target does not support a direct move to the registers mode. */
3328	if (reg_lhs
3329	&& TREE_CODE (lhs_base) == MEM_REF
3330	&& TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3331	&& DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3332	&& integer_zerop (TREE_OPERAND (lhs_base, 1))
3333	&& tree_fits_uhwi_p (var_size)
3334	&& tree_int_cst_equal
3335	(var_size,
3336	DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3337	{
3338	lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3339	var_type = TREE_TYPE (lhs);
3340	}
3341	}
3342
3343	if (!reg_lhs)
3344	{
3345	/* If the variable is not in register, expand to a memset
3346	to initialize it. */
3347	mark_addressable (lhs);
3348	tree var_addr = build_fold_addr_expr (lhs);
3349
3350	tree value = (init_type == AUTO_INIT_PATTERN)
3351	? build_int_cst (integer_type_node,
3352	INIT_PATTERN_VALUE)
3353	: integer_zero_node;
3354	tree m_call = build_call_expr (builtin_decl_implicit (fncode: BUILT_IN_MEMSET),
3355	3, var_addr, value, var_size);
3356	/* Expand this memset call. */
3357	expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3358	}
3359	else
3360	{
3361	/* If this variable is in a register use expand_assignment.
3362	For boolean scalars force zero-init. */
3363	tree init;
3364	scalar_int_mode var_mode;
3365	if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3366	&& tree_fits_uhwi_p (var_size)
3367	&& (init_type == AUTO_INIT_PATTERN
3368	|| !is_gimple_reg_type (type: var_type))
3369	&& int_mode_for_size (size: tree_to_uhwi (var_size) * BITS_PER_UNIT,
3370	limit: 0).exists (mode: &var_mode)
3371	&& have_insn_for (SET, var_mode))
3372	{
3373	unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3374	unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3375	memset (s: buf, c: (init_type == AUTO_INIT_PATTERN
3376	? INIT_PATTERN_VALUE : 0), n: total_bytes);
3377	tree itype = build_nonstandard_integer_type
3378	(total_bytes * BITS_PER_UNIT, 1);
3379	wide_int w = wi::from_buffer (buf, total_bytes);
3380	init = wide_int_to_tree (type: itype, cst: w);
3381	/* Pun the LHS to make sure its type has constant size
3382	unless it is an SSA name where that's already known. */
3383	if (TREE_CODE (lhs) != SSA_NAME)
3384	lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3385	else
3386	init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3387	}
3388	else
3389	/* Use zero-init also for variable-length sizes. */
3390	init = build_zero_cst (var_type);
3391
3392	expand_assignment (lhs, init, false);
3393	}
3394	}
3395
3396	/* The size of an OpenACC compute dimension. */
3397
3398	static void
3399	expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3400	{
3401	tree lhs = gimple_call_lhs (gs: stmt);
3402
3403	if (!lhs)
3404	return;
3405
3406	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3407	if (targetm.have_oacc_dim_size ())
3408	{
3409	rtx dim = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), NULL_RTX,
3410	VOIDmode, modifier: EXPAND_NORMAL);
3411	emit_insn (targetm.gen_oacc_dim_size (target, dim));
3412	}
3413	else
3414	emit_move_insn (target, GEN_INT (1));
3415	}
3416
3417	/* The position of an OpenACC execution engine along one compute axis. */
3418
3419	static void
3420	expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3421	{
3422	tree lhs = gimple_call_lhs (gs: stmt);
3423
3424	if (!lhs)
3425	return;
3426
3427	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3428	if (targetm.have_oacc_dim_pos ())
3429	{
3430	rtx dim = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), NULL_RTX,
3431	VOIDmode, modifier: EXPAND_NORMAL);
3432	emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3433	}
3434	else
3435	emit_move_insn (target, const0_rtx);
3436	}
3437
3438	/* This is expanded by oacc_device_lower pass. */
3439
3440	static void
3441	expand_GOACC_LOOP (internal_fn, gcall *)
3442	{
3443	gcc_unreachable ();
3444	}
3445
3446	/* This is expanded by oacc_device_lower pass. */
3447
3448	static void
3449	expand_GOACC_REDUCTION (internal_fn, gcall *)
3450	{
3451	gcc_unreachable ();
3452	}
3453
3454	/* This is expanded by oacc_device_lower pass. */
3455
3456	static void
3457	expand_GOACC_TILE (internal_fn, gcall *)
3458	{
3459	gcc_unreachable ();
3460	}
3461
3462	/* Set errno to EDOM. */
3463
3464	static void
3465	expand_SET_EDOM (internal_fn, gcall *)
3466	{
3467	#ifdef TARGET_EDOM
3468	#ifdef GEN_ERRNO_RTX
3469	rtx errno_rtx = GEN_ERRNO_RTX;
3470	#else
3471	rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3472	#endif
3473	emit_move_insn (errno_rtx,
3474	gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3475	#else
3476	gcc_unreachable ();
3477	#endif
3478	}
3479
3480	/* Expand atomic bit test and set. */
3481
3482	static void
3483	expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3484	{
3485	expand_ifn_atomic_bit_test_and (call);
3486	}
3487
3488	/* Expand atomic bit test and complement. */
3489
3490	static void
3491	expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3492	{
3493	expand_ifn_atomic_bit_test_and (call);
3494	}
3495
3496	/* Expand atomic bit test and reset. */
3497
3498	static void
3499	expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3500	{
3501	expand_ifn_atomic_bit_test_and (call);
3502	}
3503
3504	/* Expand atomic bit test and set. */
3505
3506	static void
3507	expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3508	{
3509	expand_ifn_atomic_compare_exchange (call);
3510	}
3511
3512	/* Expand atomic add fetch and cmp with 0. */
3513
3514	static void
3515	expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3516	{
3517	expand_ifn_atomic_op_fetch_cmp_0 (call);
3518	}
3519
3520	/* Expand atomic sub fetch and cmp with 0. */
3521
3522	static void
3523	expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3524	{
3525	expand_ifn_atomic_op_fetch_cmp_0 (call);
3526	}
3527
3528	/* Expand atomic and fetch and cmp with 0. */
3529
3530	static void
3531	expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3532	{
3533	expand_ifn_atomic_op_fetch_cmp_0 (call);
3534	}
3535
3536	/* Expand atomic or fetch and cmp with 0. */
3537
3538	static void
3539	expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3540	{
3541	expand_ifn_atomic_op_fetch_cmp_0 (call);
3542	}
3543
3544	/* Expand atomic xor fetch and cmp with 0. */
3545
3546	static void
3547	expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3548	{
3549	expand_ifn_atomic_op_fetch_cmp_0 (call);
3550	}
3551
3552	/* Expand LAUNDER to assignment, lhs = arg0. */
3553
3554	static void
3555	expand_LAUNDER (internal_fn, gcall *call)
3556	{
3557	tree lhs = gimple_call_lhs (gs: call);
3558
3559	if (!lhs)
3560	return;
3561
3562	expand_assignment (lhs, gimple_call_arg (gs: call, index: 0), false);
3563	}
3564
3565	/* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3566
3567	static void
3568	expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3569	{
3570	internal_fn ifn = gimple_call_internal_fn (gs: stmt);
3571	int rhs_index = internal_fn_stored_value_index (ifn);
3572	tree base = gimple_call_arg (gs: stmt, index: 0);
3573	tree offset = gimple_call_arg (gs: stmt, index: 1);
3574	tree scale = gimple_call_arg (gs: stmt, index: 2);
3575	tree rhs = gimple_call_arg (gs: stmt, index: rhs_index);
3576
3577	rtx base_rtx = expand_normal (exp: base);
3578	rtx offset_rtx = expand_normal (exp: offset);
3579	HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3580	rtx rhs_rtx = expand_normal (exp: rhs);
3581
3582	class expand_operand ops[8];
3583	int i = 0;
3584	create_address_operand (op: &ops[i++], value: base_rtx);
3585	create_input_operand (op: &ops[i++], value: offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3586	create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3587	create_integer_operand (&ops[i++], scale_int);
3588	create_input_operand (op: &ops[i++], value: rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3589	i = add_mask_and_len_args (ops, opno: i, stmt);
3590
3591	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (TREE_TYPE (rhs)),
3592	TYPE_MODE (TREE_TYPE (offset)));
3593	expand_insn (icode, nops: i, ops);
3594	}
3595
3596	/* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3597
3598	static void
3599	expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3600	{
3601	tree lhs = gimple_call_lhs (gs: stmt);
3602	tree base = gimple_call_arg (gs: stmt, index: 0);
3603	tree offset = gimple_call_arg (gs: stmt, index: 1);
3604	tree scale = gimple_call_arg (gs: stmt, index: 2);
3605
3606	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3607	rtx base_rtx = expand_normal (exp: base);
3608	rtx offset_rtx = expand_normal (exp: offset);
3609	HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3610
3611	int i = 0;
3612	class expand_operand ops[8];
3613	create_output_operand (op: &ops[i++], x: lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3614	create_address_operand (op: &ops[i++], value: base_rtx);
3615	create_input_operand (op: &ops[i++], value: offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3616	create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3617	create_integer_operand (&ops[i++], scale_int);
3618	i = add_mask_and_len_args (ops, opno: i, stmt);
3619	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (TREE_TYPE (lhs)),
3620	TYPE_MODE (TREE_TYPE (offset)));
3621	expand_insn (icode, nops: i, ops);
3622	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3623	emit_move_insn (lhs_rtx, ops[0].value);
3624	}
3625
3626	/* Helper for expand_DIVMOD. Return true if the sequence starting with
3627	INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3628
3629	static bool
3630	contains_call_div_mod (rtx_insn *insn)
3631	{
3632	subrtx_iterator::array_type array;
3633	for (; insn; insn = NEXT_INSN (insn))
3634	if (CALL_P (insn))
3635	return true;
3636	else if (INSN_P (insn))
3637	FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3638	switch (GET_CODE (*iter))
3639	{
3640	case CALL:
3641	case DIV:
3642	case UDIV:
3643	case MOD:
3644	case UMOD:
3645	return true;
3646	default:
3647	break;
3648	}
3649	return false;
3650	}
3651
3652	/* Expand DIVMOD() using:
3653	a) optab handler for udivmod/sdivmod if it is available.
3654	b) If optab_handler doesn't exist, generate call to
3655	target-specific divmod libfunc. */
3656
3657	static void
3658	expand_DIVMOD (internal_fn, gcall *call_stmt)
3659	{
3660	tree lhs = gimple_call_lhs (gs: call_stmt);
3661	tree arg0 = gimple_call_arg (gs: call_stmt, index: 0);
3662	tree arg1 = gimple_call_arg (gs: call_stmt, index: 1);
3663
3664	gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3665	tree type = TREE_TYPE (TREE_TYPE (lhs));
3666	machine_mode mode = TYPE_MODE (type);
3667	bool unsignedp = TYPE_UNSIGNED (type);
3668	optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3669
3670	rtx op0 = expand_normal (exp: arg0);
3671	rtx op1 = expand_normal (exp: arg1);
3672	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3673
3674	rtx quotient = NULL_RTX, remainder = NULL_RTX;
3675	rtx_insn *insns = NULL;
3676
3677	if (TREE_CODE (arg1) == INTEGER_CST)
3678	{
3679	/* For DIVMOD by integral constants, there could be efficient code
3680	expanded inline e.g. using shifts and plus/minus. Try to expand
3681	the division and modulo and if it emits any library calls or any
3682	{,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3683	divmod libcall. */
3684	scalar_int_mode int_mode;
3685	if (remainder == NULL_RTX
3686	&& optimize
3687	&& CONST_INT_P (op1)
3688	&& !pow2p_hwi (INTVAL (op1))
3689	&& is_int_mode (TYPE_MODE (type), int_mode: &int_mode)
3690	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3691	&& optab_handler (op: and_optab, mode: word_mode) != CODE_FOR_nothing
3692	&& optab_handler (op: add_optab, mode: word_mode) != CODE_FOR_nothing
3693	&& optimize_insn_for_speed_p ())
3694	{
3695	rtx_insn *last = get_last_insn ();
3696	remainder = NULL_RTX;
3697	quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3698	TYPE_UNSIGNED (type));
3699	if (quotient != NULL_RTX)
3700	{
3701	if (optab_handler (op: mov_optab, mode: int_mode) != CODE_FOR_nothing)
3702	{
3703	rtx_insn *move = emit_move_insn (quotient, quotient);
3704	set_dst_reg_note (move, REG_EQUAL,
3705	gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3706	? UDIV : DIV, int_mode,
3707	copy_rtx (op0), op1),
3708	quotient);
3709	move = emit_move_insn (remainder, remainder);
3710	set_dst_reg_note (move, REG_EQUAL,
3711	gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3712	? UMOD : MOD, int_mode,
3713	copy_rtx (op0), op1),
3714	quotient);
3715	}
3716	}
3717	else
3718	delete_insns_since (last);
3719	}
3720
3721	if (remainder == NULL_RTX)
3722	{
3723	struct separate_ops ops;
3724	ops.code = TRUNC_DIV_EXPR;
3725	ops.type = type;
3726	ops.op0 = make_tree (ops.type, op0);
3727	ops.op1 = arg1;
3728	ops.op2 = NULL_TREE;
3729	ops.location = gimple_location (g: call_stmt);
3730	start_sequence ();
3731	quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3732	if (contains_call_div_mod (insn: get_insns ()))
3733	quotient = NULL_RTX;
3734	else
3735	{
3736	ops.code = TRUNC_MOD_EXPR;
3737	remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3738	EXPAND_NORMAL);
3739	if (contains_call_div_mod (insn: get_insns ()))
3740	remainder = NULL_RTX;
3741	}
3742	if (remainder)
3743	insns = get_insns ();
3744	end_sequence ();
3745	}
3746	}
3747
3748	if (remainder)
3749	emit_insn (insns);
3750
3751	/* Check if optab_handler exists for divmod_optab for given mode. */
3752	else if (optab_handler (op: tab, mode) != CODE_FOR_nothing)
3753	{
3754	quotient = gen_reg_rtx (mode);
3755	remainder = gen_reg_rtx (mode);
3756	expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3757	}
3758
3759	/* Generate call to divmod libfunc if it exists. */
3760	else if (rtx libfunc = optab_libfunc (tab, mode))
3761	targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3762	&quotient, &remainder);
3763
3764	else
3765	gcc_unreachable ();
3766
3767	/* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3768	expand_expr (exp: build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3769	make_tree (TREE_TYPE (arg0), quotient),
3770	make_tree (TREE_TYPE (arg1), remainder)),
3771	target, VOIDmode, modifier: EXPAND_NORMAL);
3772	}
3773
3774	/* Expand a NOP. */
3775
3776	static void
3777	expand_NOP (internal_fn, gcall *)
3778	{
3779	/* Nothing. But it shouldn't really prevail. */
3780	}
3781
3782	/* Coroutines, all should have been processed at this stage. */
3783
3784	static void
3785	expand_CO_FRAME (internal_fn, gcall *)
3786	{
3787	gcc_unreachable ();
3788	}
3789
3790	static void
3791	expand_CO_YIELD (internal_fn, gcall *)
3792	{
3793	gcc_unreachable ();
3794	}
3795
3796	static void
3797	expand_CO_SUSPN (internal_fn, gcall *)
3798	{
3799	gcc_unreachable ();
3800	}
3801
3802	static void
3803	expand_CO_ACTOR (internal_fn, gcall *)
3804	{
3805	gcc_unreachable ();
3806	}
3807
3808	/* Expand a call to FN using the operands in STMT. FN has a single
3809	output operand and NARGS input operands. */
3810
3811	static void
3812	expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3813	unsigned int nargs)
3814	{
3815	tree_pair types = direct_internal_fn_types (fn, stmt);
3816	insn_code icode = direct_optab_handler (op: optab, TYPE_MODE (types.first));
3817	expand_fn_using_insn (stmt, icode, noutputs: 1, ninputs: nargs);
3818	}
3819
3820	/* Expand WHILE_ULT call STMT using optab OPTAB. */
3821
3822	static void
3823	expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3824	{
3825	expand_operand ops[4];
3826	tree rhs_type[2];
3827
3828	tree lhs = gimple_call_lhs (gs: stmt);
3829	tree lhs_type = TREE_TYPE (lhs);
3830	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3831	create_output_operand (op: &ops[0], x: lhs_rtx, TYPE_MODE (lhs_type));
3832
3833	for (unsigned int i = 0; i < 2; ++i)
3834	{
3835	tree rhs = gimple_call_arg (gs: stmt, index: i);
3836	rhs_type[i] = TREE_TYPE (rhs);
3837	rtx rhs_rtx = expand_normal (exp: rhs);
3838	create_input_operand (op: &ops[i + 1], value: rhs_rtx, TYPE_MODE (rhs_type[i]));
3839	}
3840
3841	int opcnt;
3842	if (!VECTOR_MODE_P (TYPE_MODE (lhs_type)))
3843	{
3844	/* When the mask is an integer mode the exact vector length may not
3845	be clear to the backend, so we pass it in operand[3].
3846	Use the vector in arg2 for the most reliable intended size. */
3847	tree type = TREE_TYPE (gimple_call_arg (stmt, 2));
3848	create_integer_operand (&ops[3], TYPE_VECTOR_SUBPARTS (node: type));
3849	opcnt = 4;
3850	}
3851	else
3852	/* The mask has a vector type so the length operand is unnecessary. */
3853	opcnt = 3;
3854
3855	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (rhs_type[0]),
3856	TYPE_MODE (lhs_type));
3857
3858	expand_insn (icode, nops: opcnt, ops);
3859	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3860	emit_move_insn (lhs_rtx, ops[0].value);
3861	}
3862
3863	/* Expand a call to a convert-like optab using the operands in STMT.
3864	FN has a single output operand and NARGS input operands. */
3865
3866	static void
3867	expand_convert_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab,
3868	unsigned int nargs)
3869	{
3870	tree_pair types = direct_internal_fn_types (fn, stmt);
3871	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (types.first),
3872	TYPE_MODE (types.second));
3873	expand_fn_using_insn (stmt, icode, noutputs: 1, ninputs: nargs);
3874	}
3875
3876	/* Expanders for optabs that can use expand_direct_optab_fn. */
3877
3878	#define expand_unary_optab_fn(FN, STMT, OPTAB) \
3879	expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3880
3881	#define expand_binary_optab_fn(FN, STMT, OPTAB) \
3882	expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3883
3884	#define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3885	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3886
3887	#define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3888	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3889
3890	#define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3891	expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3892
3893	#define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3894	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3895
3896	#define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3897	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3898
3899	#define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3900	expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3901
3902	#define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3903	expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3904
3905	#define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3906	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3907
3908	#define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3909	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3910
3911	#define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3912	expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3913
3914	#define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3915	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3916
3917	#define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
3918	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3919
3920	#define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3921	expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3922
3923	/* Expanders for optabs that can use expand_convert_optab_fn. */
3924
3925	#define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3926	expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3927
3928	#define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
3929	expand_convert_optab_fn (FN, STMT, OPTAB, 2)
3930
3931	/* RETURN_TYPE and ARGS are a return type and argument list that are
3932	in principle compatible with FN (which satisfies direct_internal_fn_p).
3933	Return the types that should be used to determine whether the
3934	target supports FN. */
3935
3936	tree_pair
3937	direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3938	{
3939	const direct_internal_fn_info &info = direct_internal_fn (fn);
3940	tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3941	tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3942	return tree_pair (type0, type1);
3943	}
3944
3945	/* CALL is a call whose return type and arguments are in principle
3946	compatible with FN (which satisfies direct_internal_fn_p). Return the
3947	types that should be used to determine whether the target supports FN. */
3948
3949	tree_pair
3950	direct_internal_fn_types (internal_fn fn, gcall *call)
3951	{
3952	const direct_internal_fn_info &info = direct_internal_fn (fn);
3953	tree op0 = (info.type0 < 0
3954	? gimple_call_lhs (gs: call)
3955	: gimple_call_arg (gs: call, index: info.type0));
3956	tree op1 = (info.type1 < 0
3957	? gimple_call_lhs (gs: call)
3958	: gimple_call_arg (gs: call, index: info.type1));
3959	return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
3960	}
3961
3962	/* Return true if OPTAB is supported for TYPES (whose modes should be
3963	the same) when the optimization type is OPT_TYPE. Used for simple
3964	direct optabs. */
3965
3966	static bool
3967	direct_optab_supported_p (direct_optab optab, tree_pair types,
3968	optimization_type opt_type)
3969	{
3970	machine_mode mode = TYPE_MODE (types.first);
3971	gcc_checking_assert (mode == TYPE_MODE (types.second));
3972	return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
3973	}
3974
3975	/* Return true if OPTAB is supported for TYPES, where the first type
3976	is the destination and the second type is the source. Used for
3977	convert optabs. */
3978
3979	static bool
3980	convert_optab_supported_p (convert_optab optab, tree_pair types,
3981	optimization_type opt_type)
3982	{
3983	return (convert_optab_handler (optab, TYPE_MODE (types.first),
3984	TYPE_MODE (types.second), opt_type)
3985	!= CODE_FOR_nothing);
3986	}
3987
3988	/* Return true if load/store lanes optab OPTAB is supported for
3989	array type TYPES.first when the optimization type is OPT_TYPE. */
3990
3991	static bool
3992	multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
3993	optimization_type opt_type)
3994	{
3995	gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
3996	machine_mode imode = TYPE_MODE (types.first);
3997	machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
3998	return (convert_optab_handler (optab, imode, vmode, opt_type)
3999	!= CODE_FOR_nothing);
4000	}
4001
4002	#define direct_unary_optab_supported_p direct_optab_supported_p
4003	#define direct_unary_convert_optab_supported_p convert_optab_supported_p
4004	#define direct_binary_optab_supported_p direct_optab_supported_p
4005	#define direct_ternary_optab_supported_p direct_optab_supported_p
4006	#define direct_cond_unary_optab_supported_p direct_optab_supported_p
4007	#define direct_cond_binary_optab_supported_p direct_optab_supported_p
4008	#define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4009	#define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4010	#define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4011	#define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4012	#define direct_mask_load_optab_supported_p convert_optab_supported_p
4013	#define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4014	#define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4015	#define direct_gather_load_optab_supported_p convert_optab_supported_p
4016	#define direct_len_load_optab_supported_p direct_optab_supported_p
4017	#define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4018	#define direct_mask_store_optab_supported_p convert_optab_supported_p
4019	#define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4020	#define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4021	#define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4022	#define direct_vec_cond_optab_supported_p convert_optab_supported_p
4023	#define direct_scatter_store_optab_supported_p convert_optab_supported_p
4024	#define direct_len_store_optab_supported_p direct_optab_supported_p
4025	#define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4026	#define direct_while_optab_supported_p convert_optab_supported_p
4027	#define direct_fold_extract_optab_supported_p direct_optab_supported_p
4028	#define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4029	#define direct_fold_left_optab_supported_p direct_optab_supported_p
4030	#define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4031	#define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4032	#define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4033	#define direct_vec_set_optab_supported_p direct_optab_supported_p
4034	#define direct_vec_extract_optab_supported_p convert_optab_supported_p
4035
4036	/* Return the optab used by internal function FN. */
4037
4038	optab
4039	direct_internal_fn_optab (internal_fn fn, tree_pair types)
4040	{
4041	switch (fn)
4042	{
4043	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4044	case IFN_##CODE: break;
4045	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4046	case IFN_##CODE: return OPTAB##_optab;
4047	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4048	UNSIGNED_OPTAB, TYPE) \
4049	case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4050	? UNSIGNED_OPTAB ## _optab \
4051	: SIGNED_OPTAB ## _optab);
4052	#include "internal-fn.def"
4053
4054	case IFN_LAST:
4055	break;
4056	}
4057	gcc_unreachable ();
4058	}
4059
4060	/* Return the optab used by internal function FN. */
4061
4062	static optab
4063	direct_internal_fn_optab (internal_fn fn)
4064	{
4065	switch (fn)
4066	{
4067	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4068	case IFN_##CODE: break;
4069	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4070	case IFN_##CODE: return OPTAB##_optab;
4071	#include "internal-fn.def"
4072
4073	case IFN_LAST:
4074	break;
4075	}
4076	gcc_unreachable ();
4077	}
4078
4079	/* Return true if FN is supported for the types in TYPES when the
4080	optimization type is OPT_TYPE. The types are those associated with
4081	the "type0" and "type1" fields of FN's direct_internal_fn_info
4082	structure. */
4083
4084	bool
4085	direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
4086	optimization_type opt_type)
4087	{
4088	switch (fn)
4089	{
4090	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4091	case IFN_##CODE: break;
4092	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4093	case IFN_##CODE: \
4094	return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4095	opt_type);
4096	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4097	UNSIGNED_OPTAB, TYPE) \
4098	case IFN_##CODE: \
4099	{ \
4100	optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4101	? UNSIGNED_OPTAB ## _optab \
4102	: SIGNED_OPTAB ## _optab); \
4103	return direct_##TYPE##_optab_supported_p (which_optab, types, \
4104	opt_type); \
4105	}
4106	#include "internal-fn.def"
4107
4108	case IFN_LAST:
4109	break;
4110	}
4111	gcc_unreachable ();
4112	}
4113
4114	/* Return true if FN is supported for type TYPE when the optimization
4115	type is OPT_TYPE. The caller knows that the "type0" and "type1"
4116	fields of FN's direct_internal_fn_info structure are the same. */
4117
4118	bool
4119	direct_internal_fn_supported_p (internal_fn fn, tree type,
4120	optimization_type opt_type)
4121	{
4122	const direct_internal_fn_info &info = direct_internal_fn (fn);
4123	gcc_checking_assert (info.type0 == info.type1);
4124	return direct_internal_fn_supported_p (fn, types: tree_pair (type, type), opt_type);
4125	}
4126
4127	/* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4128	given that STMT is a call to a direct internal function. */
4129
4130	bool
4131	direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
4132	{
4133	internal_fn fn = gimple_call_internal_fn (gs: stmt);
4134	tree_pair types = direct_internal_fn_types (fn, call: stmt);
4135	return direct_internal_fn_supported_p (fn, types, opt_type);
4136	}
4137
4138	/* Return true if FN is a binary operation and if FN is commutative. */
4139
4140	bool
4141	commutative_binary_fn_p (internal_fn fn)
4142	{
4143	switch (fn)
4144	{
4145	case IFN_AVG_FLOOR:
4146	case IFN_AVG_CEIL:
4147	case IFN_MULH:
4148	case IFN_MULHS:
4149	case IFN_MULHRS:
4150	case IFN_FMIN:
4151	case IFN_FMAX:
4152	case IFN_COMPLEX_MUL:
4153	case IFN_UBSAN_CHECK_ADD:
4154	case IFN_UBSAN_CHECK_MUL:
4155	case IFN_ADD_OVERFLOW:
4156	case IFN_MUL_OVERFLOW:
4157	case IFN_VEC_WIDEN_PLUS:
4158	case IFN_VEC_WIDEN_PLUS_LO:
4159	case IFN_VEC_WIDEN_PLUS_HI:
4160	case IFN_VEC_WIDEN_PLUS_EVEN:
4161	case IFN_VEC_WIDEN_PLUS_ODD:
4162	return true;
4163
4164	default:
4165	return false;
4166	}
4167	}
4168
4169	/* Return true if FN is a ternary operation and if its first two arguments
4170	are commutative. */
4171
4172	bool
4173	commutative_ternary_fn_p (internal_fn fn)
4174	{
4175	switch (fn)
4176	{
4177	case IFN_FMA:
4178	case IFN_FMS:
4179	case IFN_FNMA:
4180	case IFN_FNMS:
4181	case IFN_UADDC:
4182	return true;
4183
4184	default:
4185	return false;
4186	}
4187	}
4188
4189	/* Return true if FN is an associative binary operation. */
4190
4191	bool
4192	associative_binary_fn_p (internal_fn fn)
4193	{
4194	switch (fn)
4195	{
4196	case IFN_FMIN:
4197	case IFN_FMAX:
4198	return true;
4199
4200	default:
4201	return false;
4202	}
4203	}
4204
4205	/* If FN is commutative in two consecutive arguments, return the
4206	index of the first, otherwise return -1. */
4207
4208	int
4209	first_commutative_argument (internal_fn fn)
4210	{
4211	switch (fn)
4212	{
4213	case IFN_COND_ADD:
4214	case IFN_COND_MUL:
4215	case IFN_COND_MIN:
4216	case IFN_COND_MAX:
4217	case IFN_COND_FMIN:
4218	case IFN_COND_FMAX:
4219	case IFN_COND_AND:
4220	case IFN_COND_IOR:
4221	case IFN_COND_XOR:
4222	case IFN_COND_FMA:
4223	case IFN_COND_FMS:
4224	case IFN_COND_FNMA:
4225	case IFN_COND_FNMS:
4226	case IFN_COND_LEN_ADD:
4227	case IFN_COND_LEN_MUL:
4228	case IFN_COND_LEN_MIN:
4229	case IFN_COND_LEN_MAX:
4230	case IFN_COND_LEN_FMIN:
4231	case IFN_COND_LEN_FMAX:
4232	case IFN_COND_LEN_AND:
4233	case IFN_COND_LEN_IOR:
4234	case IFN_COND_LEN_XOR:
4235	case IFN_COND_LEN_FMA:
4236	case IFN_COND_LEN_FMS:
4237	case IFN_COND_LEN_FNMA:
4238	case IFN_COND_LEN_FNMS:
4239	return 1;
4240
4241	default:
4242	if (commutative_binary_fn_p (fn)
4243	|| commutative_ternary_fn_p (fn))
4244	return 0;
4245	return -1;
4246	}
4247	}
4248
4249	/* Return true if this CODE describes an internal_fn that returns a vector with
4250	elements twice as wide as the element size of the input vectors. */
4251
4252	bool
4253	widening_fn_p (code_helper code)
4254	{
4255	if (!code.is_fn_code ())
4256	return false;
4257
4258	if (!internal_fn_p (code: (combined_fn) code))
4259	return false;
4260
4261	internal_fn fn = as_internal_fn (code: (combined_fn) code);
4262	switch (fn)
4263	{
4264	#undef DEF_INTERNAL_WIDENING_OPTAB_FN
4265	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4266	case IFN_##NAME: \
4267	case IFN_##NAME##_HI: \
4268	case IFN_##NAME##_LO: \
4269	case IFN_##NAME##_EVEN: \
4270	case IFN_##NAME##_ODD: \
4271	return true;
4272	#include "internal-fn.def"
4273	#undef DEF_INTERNAL_WIDENING_OPTAB_FN
4274
4275	default:
4276	return false;
4277	}
4278	}
4279
4280	/* Return true if IFN_SET_EDOM is supported. */
4281
4282	bool
4283	set_edom_supported_p (void)
4284	{
4285	#ifdef TARGET_EDOM
4286	return true;
4287	#else
4288	return false;
4289	#endif
4290	}
4291
4292	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4293	static void \
4294	expand_##CODE (internal_fn fn, gcall *stmt) \
4295	{ \
4296	expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4297	}
4298	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4299	UNSIGNED_OPTAB, TYPE) \
4300	static void \
4301	expand_##CODE (internal_fn fn, gcall *stmt) \
4302	{ \
4303	tree_pair types = direct_internal_fn_types (fn, stmt); \
4304	optab which_optab = direct_internal_fn_optab (fn, types); \
4305	expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4306	}
4307	#include "internal-fn.def"
4308	#undef DEF_INTERNAL_OPTAB_FN
4309	#undef DEF_INTERNAL_SIGNED_OPTAB_FN
4310
4311	/* Routines to expand each internal function, indexed by function number.
4312	Each routine has the prototype:
4313
4314	expand_<NAME> (gcall *stmt)
4315
4316	where STMT is the statement that performs the call. */
4317	static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4318
4319	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4320	#include "internal-fn.def"
4321	0
4322	};
4323
4324	/* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4325	that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4326	for each such IFN_COND_##SUFFIX. */
4327	#define FOR_EACH_CODE_MAPPING(T) \
4328	T (PLUS_EXPR, ADD) \
4329	T (MINUS_EXPR, SUB) \
4330	T (MULT_EXPR, MUL) \
4331	T (TRUNC_DIV_EXPR, DIV) \
4332	T (TRUNC_MOD_EXPR, MOD) \
4333	T (RDIV_EXPR, RDIV) \
4334	T (MIN_EXPR, MIN) \
4335	T (MAX_EXPR, MAX) \
4336	T (BIT_AND_EXPR, AND) \
4337	T (BIT_IOR_EXPR, IOR) \
4338	T (BIT_XOR_EXPR, XOR) \
4339	T (LSHIFT_EXPR, SHL) \
4340	T (RSHIFT_EXPR, SHR) \
4341	T (NEGATE_EXPR, NEG)
4342
4343	/* Return a function that only performs CODE when a certain condition is met
4344	and that uses a given fallback value otherwise. For example, if CODE is
4345	a binary operation associated with conditional function FN:
4346
4347	LHS = FN (COND, A, B, ELSE)
4348
4349	is equivalent to the C expression:
4350
4351	LHS = COND ? A CODE B : ELSE;
4352
4353	operating elementwise if the operands are vectors.
4354
4355	Return IFN_LAST if no such function exists. */
4356
4357	internal_fn
4358	get_conditional_internal_fn (tree_code code)
4359	{
4360	switch (code)
4361	{
4362	#define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4363	FOR_EACH_CODE_MAPPING(CASE)
4364	#undef CASE
4365	default:
4366	return IFN_LAST;
4367	}
4368	}
4369
4370	/* If IFN implements the conditional form of a tree code, return that
4371	tree code, otherwise return ERROR_MARK. */
4372
4373	tree_code
4374	conditional_internal_fn_code (internal_fn ifn)
4375	{
4376	switch (ifn)
4377	{
4378	#define CASE(CODE, IFN) \
4379	case IFN_COND_##IFN: \
4380	case IFN_COND_LEN_##IFN: \
4381	return CODE;
4382	FOR_EACH_CODE_MAPPING (CASE)
4383	#undef CASE
4384	default:
4385	return ERROR_MARK;
4386	}
4387	}
4388
4389	/* Like get_conditional_internal_fn, but return a function that
4390	additionally restricts the operation to the leading elements
4391	of a vector. The number of elements to process is given by a length
4392	and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4393	elements are taken from the fallback ("else") argument.
4394
4395	For example, if CODE is a binary operation associated with FN:
4396
4397	LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4398
4399	is equivalent to the C code:
4400
4401	for (int i = 0; i < NUNITS; i++)
4402	{
4403	if (i < LEN + BIAS && COND[i])
4404	LHS[i] = A[i] CODE B[i];
4405	else
4406	LHS[i] = ELSE[i];
4407	}
4408	*/
4409
4410	internal_fn
4411	get_conditional_len_internal_fn (tree_code code)
4412	{
4413	switch (code)
4414	{
4415	#define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4416	FOR_EACH_CODE_MAPPING(CASE)
4417	#undef CASE
4418	default:
4419	return IFN_LAST;
4420	}
4421	}
4422
4423	/* Invoke T(IFN) for each internal function IFN that also has an
4424	IFN_COND_* form. */
4425	#define FOR_EACH_COND_FN_PAIR(T) \
4426	T (FMAX) \
4427	T (FMIN) \
4428	T (FMA) \
4429	T (FMS) \
4430	T (FNMA) \
4431	T (FNMS)
4432
4433	/* Return a function that only performs internal function FN when a
4434	certain condition is met and that uses a given fallback value otherwise.
4435	In other words, the returned function FN' is such that:
4436
4437	LHS = FN' (COND, A1, ... An, ELSE)
4438
4439	is equivalent to the C expression:
4440
4441	LHS = COND ? FN (A1, ..., An) : ELSE;
4442
4443	operating elementwise if the operands are vectors.
4444
4445	Return IFN_LAST if no such function exists. */
4446
4447	internal_fn
4448	get_conditional_internal_fn (internal_fn fn)
4449	{
4450	switch (fn)
4451	{
4452	#define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4453	FOR_EACH_COND_FN_PAIR(CASE)
4454	#undef CASE
4455	default:
4456	return IFN_LAST;
4457	}
4458	}
4459
4460	/* If there exists an internal function like IFN that operates on vectors,
4461	but with additional length and bias parameters, return the internal_fn
4462	for that function, otherwise return IFN_LAST. */
4463	internal_fn
4464	get_len_internal_fn (internal_fn fn)
4465	{
4466	switch (fn)
4467	{
4468	#undef DEF_INTERNAL_COND_FN
4469	#undef DEF_INTERNAL_SIGNED_COND_FN
4470	#define DEF_INTERNAL_COND_FN(NAME, ...) \
4471	case IFN_COND_##NAME: \
4472	return IFN_COND_LEN_##NAME;
4473	#define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4474	case IFN_COND_##NAME: \
4475	return IFN_COND_LEN_##NAME;
4476	#include "internal-fn.def"
4477	#undef DEF_INTERNAL_COND_FN
4478	#undef DEF_INTERNAL_SIGNED_COND_FN
4479	default:
4480	return IFN_LAST;
4481	}
4482	}
4483
4484	/* If IFN implements the conditional form of an unconditional internal
4485	function, return that unconditional function, otherwise return IFN_LAST. */
4486
4487	internal_fn
4488	get_unconditional_internal_fn (internal_fn ifn)
4489	{
4490	switch (ifn)
4491	{
4492	#define CASE(NAME) \
4493	case IFN_COND_##NAME: \
4494	case IFN_COND_LEN_##NAME: \
4495	return IFN_##NAME;
4496	FOR_EACH_COND_FN_PAIR (CASE)
4497	#undef CASE
4498	default:
4499	return IFN_LAST;
4500	}
4501	}
4502
4503	/* Return true if STMT can be interpreted as a conditional tree code
4504	operation of the form:
4505
4506	LHS = COND ? OP (RHS1, ...) : ELSE;
4507
4508	operating elementwise if the operands are vectors. This includes
4509	the case of an all-true COND, so that the operation always happens.
4510
4511	There is an alternative approach to interpret the STMT when the operands
4512	are vectors which is the operation predicated by both conditional mask
4513	and loop control length, the equivalent C code:
4514
4515	for (int i = 0; i < NUNTIS; i++)
4516	{
4517	if (i < LEN + BIAS && COND[i])
4518	LHS[i] = A[i] CODE B[i];
4519	else
4520	LHS[i] = ELSE[i];
4521	}
4522
4523	When returning true, set:
4524
4525	- *COND_OUT to the condition COND, or to NULL_TREE if the condition
4526	is known to be all-true
4527	- *CODE_OUT to the tree code
4528	- OPS[I] to operand I of *CODE_OUT
4529	- *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4530	condition is known to be all true.
4531	- *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4532	- *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4533
4534	bool
4535	can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4536	tree_code *code_out,
4537	tree (&ops)[3], tree *else_out,
4538	tree *len, tree *bias)
4539	{
4540	*len = NULL_TREE;
4541	*bias = NULL_TREE;
4542	if (gassign *assign = dyn_cast <gassign *> (p: stmt))
4543	{
4544	*cond_out = NULL_TREE;
4545	*code_out = gimple_assign_rhs_code (gs: assign);
4546	ops[0] = gimple_assign_rhs1 (gs: assign);
4547	ops[1] = gimple_assign_rhs2 (gs: assign);
4548	ops[2] = gimple_assign_rhs3 (gs: assign);
4549	*else_out = NULL_TREE;
4550	return true;
4551	}
4552	if (gcall *call = dyn_cast <gcall *> (p: stmt))
4553	if (gimple_call_internal_p (gs: call))
4554	{
4555	internal_fn ifn = gimple_call_internal_fn (gs: call);
4556	tree_code code = conditional_internal_fn_code (ifn);
4557	int len_index = internal_fn_len_index (ifn);
4558	int cond_nargs = len_index >= 0 ? 4 : 2;
4559	if (code != ERROR_MARK)
4560	{
4561	*cond_out = gimple_call_arg (gs: call, index: 0);
4562	*code_out = code;
4563	unsigned int nops = gimple_call_num_args (gs: call) - cond_nargs;
4564	for (unsigned int i = 0; i < 3; ++i)
4565	ops[i] = i < nops ? gimple_call_arg (gs: call, index: i + 1) : NULL_TREE;
4566	*else_out = gimple_call_arg (gs: call, index: nops + 1);
4567	if (len_index < 0)
4568	{
4569	if (integer_truep (*cond_out))
4570	{
4571	*cond_out = NULL_TREE;
4572	*else_out = NULL_TREE;
4573	}
4574	}
4575	else
4576	{
4577	*len = gimple_call_arg (gs: call, index: len_index);
4578	*bias = gimple_call_arg (gs: call, index: len_index + 1);
4579	}
4580	return true;
4581	}
4582	}
4583	return false;
4584	}
4585
4586	/* Return true if IFN is some form of load from memory. */
4587
4588	bool
4589	internal_load_fn_p (internal_fn fn)
4590	{
4591	switch (fn)
4592	{
4593	case IFN_MASK_LOAD:
4594	case IFN_LOAD_LANES:
4595	case IFN_MASK_LOAD_LANES:
4596	case IFN_MASK_LEN_LOAD_LANES:
4597	case IFN_GATHER_LOAD:
4598	case IFN_MASK_GATHER_LOAD:
4599	case IFN_MASK_LEN_GATHER_LOAD:
4600	case IFN_LEN_LOAD:
4601	case IFN_MASK_LEN_LOAD:
4602	return true;
4603
4604	default:
4605	return false;
4606	}
4607	}
4608
4609	/* Return true if IFN is some form of store to memory. */
4610
4611	bool
4612	internal_store_fn_p (internal_fn fn)
4613	{
4614	switch (fn)
4615	{
4616	case IFN_MASK_STORE:
4617	case IFN_STORE_LANES:
4618	case IFN_MASK_STORE_LANES:
4619	case IFN_MASK_LEN_STORE_LANES:
4620	case IFN_SCATTER_STORE:
4621	case IFN_MASK_SCATTER_STORE:
4622	case IFN_MASK_LEN_SCATTER_STORE:
4623	case IFN_LEN_STORE:
4624	case IFN_MASK_LEN_STORE:
4625	return true;
4626
4627	default:
4628	return false;
4629	}
4630	}
4631
4632	/* Return true if IFN is some form of gather load or scatter store. */
4633
4634	bool
4635	internal_gather_scatter_fn_p (internal_fn fn)
4636	{
4637	switch (fn)
4638	{
4639	case IFN_GATHER_LOAD:
4640	case IFN_MASK_GATHER_LOAD:
4641	case IFN_MASK_LEN_GATHER_LOAD:
4642	case IFN_SCATTER_STORE:
4643	case IFN_MASK_SCATTER_STORE:
4644	case IFN_MASK_LEN_SCATTER_STORE:
4645	return true;
4646
4647	default:
4648	return false;
4649	}
4650	}
4651
4652	/* If FN takes a vector len argument, return the index of that argument,
4653	otherwise return -1. */
4654
4655	int
4656	internal_fn_len_index (internal_fn fn)
4657	{
4658	switch (fn)
4659	{
4660	case IFN_LEN_LOAD:
4661	case IFN_LEN_STORE:
4662	return 2;
4663
4664	case IFN_MASK_LEN_GATHER_LOAD:
4665	case IFN_MASK_LEN_SCATTER_STORE:
4666	case IFN_COND_LEN_FMA:
4667	case IFN_COND_LEN_FMS:
4668	case IFN_COND_LEN_FNMA:
4669	case IFN_COND_LEN_FNMS:
4670	return 5;
4671
4672	case IFN_COND_LEN_ADD:
4673	case IFN_COND_LEN_SUB:
4674	case IFN_COND_LEN_MUL:
4675	case IFN_COND_LEN_DIV:
4676	case IFN_COND_LEN_MOD:
4677	case IFN_COND_LEN_RDIV:
4678	case IFN_COND_LEN_MIN:
4679	case IFN_COND_LEN_MAX:
4680	case IFN_COND_LEN_FMIN:
4681	case IFN_COND_LEN_FMAX:
4682	case IFN_COND_LEN_AND:
4683	case IFN_COND_LEN_IOR:
4684	case IFN_COND_LEN_XOR:
4685	case IFN_COND_LEN_SHL:
4686	case IFN_COND_LEN_SHR:
4687	return 4;
4688
4689	case IFN_COND_LEN_NEG:
4690	case IFN_MASK_LEN_LOAD:
4691	case IFN_MASK_LEN_STORE:
4692	case IFN_MASK_LEN_LOAD_LANES:
4693	case IFN_MASK_LEN_STORE_LANES:
4694	case IFN_VCOND_MASK_LEN:
4695	return 3;
4696
4697	default:
4698	return -1;
4699	}
4700	}
4701
4702	/* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4703	argument that is used when the condition is false. Return -1 otherwise. */
4704
4705	int
4706	internal_fn_else_index (internal_fn fn)
4707	{
4708	switch (fn)
4709	{
4710	case IFN_COND_NEG:
4711	case IFN_COND_NOT:
4712	case IFN_COND_LEN_NEG:
4713	case IFN_COND_LEN_NOT:
4714	return 2;
4715
4716	case IFN_COND_ADD:
4717	case IFN_COND_SUB:
4718	case IFN_COND_MUL:
4719	case IFN_COND_DIV:
4720	case IFN_COND_MOD:
4721	case IFN_COND_MIN:
4722	case IFN_COND_MAX:
4723	case IFN_COND_FMIN:
4724	case IFN_COND_FMAX:
4725	case IFN_COND_AND:
4726	case IFN_COND_IOR:
4727	case IFN_COND_XOR:
4728	case IFN_COND_SHL:
4729	case IFN_COND_SHR:
4730	case IFN_COND_LEN_ADD:
4731	case IFN_COND_LEN_SUB:
4732	case IFN_COND_LEN_MUL:
4733	case IFN_COND_LEN_DIV:
4734	case IFN_COND_LEN_MOD:
4735	case IFN_COND_LEN_MIN:
4736	case IFN_COND_LEN_MAX:
4737	case IFN_COND_LEN_FMIN:
4738	case IFN_COND_LEN_FMAX:
4739	case IFN_COND_LEN_AND:
4740	case IFN_COND_LEN_IOR:
4741	case IFN_COND_LEN_XOR:
4742	case IFN_COND_LEN_SHL:
4743	case IFN_COND_LEN_SHR:
4744	return 3;
4745
4746	case IFN_COND_FMA:
4747	case IFN_COND_FMS:
4748	case IFN_COND_FNMA:
4749	case IFN_COND_FNMS:
4750	case IFN_COND_LEN_FMA:
4751	case IFN_COND_LEN_FMS:
4752	case IFN_COND_LEN_FNMA:
4753	case IFN_COND_LEN_FNMS:
4754	return 4;
4755
4756	default:
4757	return -1;
4758	}
4759
4760	return -1;
4761	}
4762
4763	/* If FN takes a vector mask argument, return the index of that argument,
4764	otherwise return -1. */
4765
4766	int
4767	internal_fn_mask_index (internal_fn fn)
4768	{
4769	switch (fn)
4770	{
4771	case IFN_MASK_LOAD:
4772	case IFN_MASK_LOAD_LANES:
4773	case IFN_MASK_LEN_LOAD_LANES:
4774	case IFN_MASK_STORE:
4775	case IFN_MASK_STORE_LANES:
4776	case IFN_MASK_LEN_STORE_LANES:
4777	case IFN_MASK_LEN_LOAD:
4778	case IFN_MASK_LEN_STORE:
4779	return 2;
4780
4781	case IFN_MASK_GATHER_LOAD:
4782	case IFN_MASK_SCATTER_STORE:
4783	case IFN_MASK_LEN_GATHER_LOAD:
4784	case IFN_MASK_LEN_SCATTER_STORE:
4785	return 4;
4786
4787	case IFN_VCOND_MASK_LEN:
4788	return 0;
4789
4790	default:
4791	return (conditional_internal_fn_code (ifn: fn) != ERROR_MARK
4792	|| get_unconditional_internal_fn (ifn: fn) != IFN_LAST ? 0 : -1);
4793	}
4794	}
4795
4796	/* If FN takes a value that should be stored to memory, return the index
4797	of that argument, otherwise return -1. */
4798
4799	int
4800	internal_fn_stored_value_index (internal_fn fn)
4801	{
4802	switch (fn)
4803	{
4804	case IFN_MASK_STORE:
4805	case IFN_MASK_STORE_LANES:
4806	case IFN_SCATTER_STORE:
4807	case IFN_MASK_SCATTER_STORE:
4808	case IFN_MASK_LEN_SCATTER_STORE:
4809	return 3;
4810
4811	case IFN_LEN_STORE:
4812	return 4;
4813
4814	case IFN_MASK_LEN_STORE:
4815	case IFN_MASK_LEN_STORE_LANES:
4816	return 5;
4817
4818	default:
4819	return -1;
4820	}
4821	}
4822
4823	/* Return true if the target supports gather load or scatter store function
4824	IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4825	while for stores it is the vector type of the stored data argument.
4826	MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4827	or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4828	offset from the shared base address of each loaded or stored element.
4829	SCALE is the amount by which these offsets should be multiplied
4830	*after* they have been extended to address width. */
4831
4832	bool
4833	internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
4834	tree memory_element_type,
4835	tree offset_vector_type, int scale)
4836	{
4837	if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
4838	TYPE_SIZE (memory_element_type)))
4839	return false;
4840	if (maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: vector_type),
4841	b: TYPE_VECTOR_SUBPARTS (node: offset_vector_type)))
4842	return false;
4843	optab optab = direct_internal_fn_optab (fn: ifn);
4844	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (vector_type),
4845	TYPE_MODE (offset_vector_type));
4846	int output_ops = internal_load_fn_p (fn: ifn) ? 1 : 0;
4847	bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
4848	return (icode != CODE_FOR_nothing
4849	&& insn_operand_matches (icode, opno: 2 + output_ops, GEN_INT (unsigned_p))
4850	&& insn_operand_matches (icode, opno: 3 + output_ops, GEN_INT (scale)));
4851	}
4852
4853	/* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4854	for pointers of type TYPE when the accesses have LENGTH bytes and their
4855	common byte alignment is ALIGN. */
4856
4857	bool
4858	internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
4859	poly_uint64 length, unsigned int align)
4860	{
4861	machine_mode mode = TYPE_MODE (type);
4862	optab optab = direct_internal_fn_optab (fn: ifn);
4863	insn_code icode = direct_optab_handler (op: optab, mode);
4864	if (icode == CODE_FOR_nothing)
4865	return false;
4866	rtx length_rtx = immed_wide_int_const (length, mode);
4867	return (insn_operand_matches (icode, opno: 3, operand: length_rtx)
4868	&& insn_operand_matches (icode, opno: 4, GEN_INT (align)));
4869	}
4870
4871	/* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4872	or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4873	-1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4874	{len_,mask_len_}store). If none of the biases match what the backend
4875	provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
4876
4877	signed char
4878	internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
4879	{
4880	optab optab = direct_internal_fn_optab (fn: ifn);
4881	insn_code icode = direct_optab_handler (op: optab, mode);
4882	int bias_no = 3;
4883
4884	if (icode == CODE_FOR_nothing)
4885	{
4886	machine_mode mask_mode;
4887	if (!targetm.vectorize.get_mask_mode (mode).exists (mode: &mask_mode))
4888	return VECT_PARTIAL_BIAS_UNSUPPORTED;
4889	if (ifn == IFN_LEN_LOAD)
4890	{
4891	/* Try MASK_LEN_LOAD. */
4892	optab = direct_internal_fn_optab (fn: IFN_MASK_LEN_LOAD);
4893	}
4894	else
4895	{
4896	/* Try MASK_LEN_STORE. */
4897	optab = direct_internal_fn_optab (fn: IFN_MASK_LEN_STORE);
4898	}
4899	icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: mask_mode);
4900	bias_no = 4;
4901	}
4902
4903	if (icode != CODE_FOR_nothing)
4904	{
4905	/* For now we only support biases of 0 or -1. Try both of them. */
4906	if (insn_operand_matches (icode, opno: bias_no, GEN_INT (0)))
4907	return 0;
4908	if (insn_operand_matches (icode, opno: bias_no, GEN_INT (-1)))
4909	return -1;
4910	}
4911
4912	return VECT_PARTIAL_BIAS_UNSUPPORTED;
4913	}
4914
4915	/* Expand STMT as though it were a call to internal function FN. */
4916
4917	void
4918	expand_internal_call (internal_fn fn, gcall *stmt)
4919	{
4920	internal_fn_expanders[fn] (fn, stmt);
4921	}
4922
4923	/* Expand STMT, which is a call to internal function FN. */
4924
4925	void
4926	expand_internal_call (gcall *stmt)
4927	{
4928	expand_internal_call (fn: gimple_call_internal_fn (gs: stmt), stmt);
4929	}
4930
4931	/* If TYPE is a vector type, return true if IFN is a direct internal
4932	function that is supported for that type. If TYPE is a scalar type,
4933	return true if IFN is a direct internal function that is supported for
4934	the target's preferred vector version of TYPE. */
4935
4936	bool
4937	vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
4938	{
4939	if (VECTOR_MODE_P (TYPE_MODE (type)))
4940	return direct_internal_fn_supported_p (fn: ifn, type, opt_type: OPTIMIZE_FOR_SPEED);
4941
4942	scalar_mode smode;
4943	if (VECTOR_TYPE_P (type)
4944	|| !is_a <scalar_mode> (TYPE_MODE (type), result: &smode))
4945	return false;
4946
4947	machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
4948	if (VECTOR_MODE_P (vmode))
4949	{
4950	tree vectype = build_vector_type_for_mode (type, vmode);
4951	if (direct_internal_fn_supported_p (fn: ifn, type: vectype, opt_type: OPTIMIZE_FOR_SPEED))
4952	return true;
4953	}
4954
4955	auto_vector_modes vector_modes;
4956	targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
4957	for (machine_mode base_mode : vector_modes)
4958	if (related_vector_mode (base_mode, smode).exists (mode: &vmode))
4959	{
4960	tree vectype = build_vector_type_for_mode (type, vmode);
4961	if (direct_internal_fn_supported_p (fn: ifn, type: vectype, opt_type: OPTIMIZE_FOR_SPEED))
4962	return true;
4963	}
4964
4965	return false;
4966	}
4967
4968	void
4969	expand_SHUFFLEVECTOR (internal_fn, gcall *)
4970	{
4971	gcc_unreachable ();
4972	}
4973
4974	void
4975	expand_PHI (internal_fn, gcall *)
4976	{
4977	gcc_unreachable ();
4978	}
4979
4980	void
4981	expand_SPACESHIP (internal_fn, gcall *stmt)
4982	{
4983	tree lhs = gimple_call_lhs (gs: stmt);
4984	tree rhs1 = gimple_call_arg (gs: stmt, index: 0);
4985	tree rhs2 = gimple_call_arg (gs: stmt, index: 1);
4986	tree type = TREE_TYPE (rhs1);
4987
4988	do_pending_stack_adjust ();
4989
4990	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
4991	rtx op1 = expand_normal (exp: rhs1);
4992	rtx op2 = expand_normal (exp: rhs2);
4993
4994	class expand_operand ops[3];
4995	create_output_operand (op: &ops[0], x: target, TYPE_MODE (TREE_TYPE (lhs)));
4996	create_input_operand (op: &ops[1], value: op1, TYPE_MODE (type));
4997	create_input_operand (op: &ops[2], value: op2, TYPE_MODE (type));
4998	insn_code icode = optab_handler (op: spaceship_optab, TYPE_MODE (type));
4999	expand_insn (icode, nops: 3, ops);
5000	if (!rtx_equal_p (target, ops[0].value))
5001	emit_move_insn (target, ops[0].value);
5002	}
5003
5004	void
5005	expand_ASSUME (internal_fn, gcall *)
5006	{
5007	}
5008
5009	void
5010	expand_MASK_CALL (internal_fn, gcall *)
5011	{
5012	/* This IFN should only exist between ifcvt and vect passes. */
5013	gcc_unreachable ();
5014	}
5015
5016	void
5017	expand_MULBITINT (internal_fn, gcall *stmt)
5018	{
5019	rtx_mode_t args[6];
5020	for (int i = 0; i < 6; i++)
5021	args[i] = rtx_mode_t (expand_normal (exp: gimple_call_arg (gs: stmt, index: i)),
5022	(i & 1) ? SImode : ptr_mode);
5023	rtx fun = init_one_libfunc ("__mulbitint3");
5024	emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 6, args);
5025	}
5026
5027	void
5028	expand_DIVMODBITINT (internal_fn, gcall *stmt)
5029	{
5030	rtx_mode_t args[8];
5031	for (int i = 0; i < 8; i++)
5032	args[i] = rtx_mode_t (expand_normal (exp: gimple_call_arg (gs: stmt, index: i)),
5033	(i & 1) ? SImode : ptr_mode);
5034	rtx fun = init_one_libfunc ("__divmodbitint4");
5035	emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 8, args);
5036	}
5037
5038	void
5039	expand_FLOATTOBITINT (internal_fn, gcall *stmt)
5040	{
5041	machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt, 2)));
5042	rtx arg0 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
5043	rtx arg1 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
5044	rtx arg2 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 2));
5045	const char *mname = GET_MODE_NAME (mode);
5046	unsigned mname_len = strlen (s: mname);
5047	int len = 12 + mname_len;
5048	if (DECIMAL_FLOAT_MODE_P (mode))
5049	len += 4;
5050	char *libfunc_name = XALLOCAVEC (char, len);
5051	char *p = libfunc_name;
5052	const char *q;
5053	if (DECIMAL_FLOAT_MODE_P (mode))
5054	{
5055	#if ENABLE_DECIMAL_BID_FORMAT
5056	memcpy (dest: p, src: "__bid_fix", n: 9);
5057	#else
5058	memcpy (p, "__dpd_fix", 9);
5059	#endif
5060	p += 9;
5061	}
5062	else
5063	{
5064	memcpy (dest: p, src: "__fix", n: 5);
5065	p += 5;
5066	}
5067	for (q = mname; *q; q++)
5068	*p++ = TOLOWER (*q);
5069	memcpy (dest: p, src: "bitint", n: 7);
5070	rtx fun = init_one_libfunc (libfunc_name);
5071	emit_library_call (fun, fn_type: LCT_NORMAL, VOIDmode, arg1: arg0, arg1_mode: ptr_mode, arg2: arg1,
5072	SImode, arg3: arg2, arg3_mode: mode);
5073	}
5074
5075	void
5076	expand_BITINTTOFLOAT (internal_fn, gcall *stmt)
5077	{
5078	tree lhs = gimple_call_lhs (gs: stmt);
5079	if (!lhs)
5080	return;
5081	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
5082	rtx arg0 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
5083	rtx arg1 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
5084	const char *mname = GET_MODE_NAME (mode);
5085	unsigned mname_len = strlen (s: mname);
5086	int len = 14 + mname_len;
5087	if (DECIMAL_FLOAT_MODE_P (mode))
5088	len += 4;
5089	char *libfunc_name = XALLOCAVEC (char, len);
5090	char *p = libfunc_name;
5091	const char *q;
5092	if (DECIMAL_FLOAT_MODE_P (mode))
5093	{
5094	#if ENABLE_DECIMAL_BID_FORMAT
5095	memcpy (dest: p, src: "__bid_floatbitint", n: 17);
5096	#else
5097	memcpy (p, "__dpd_floatbitint", 17);
5098	#endif
5099	p += 17;
5100	}
5101	else
5102	{
5103	memcpy (dest: p, src: "__floatbitint", n: 13);
5104	p += 13;
5105	}
5106	for (q = mname; *q; q++)
5107	*p++ = TOLOWER (*q);
5108	*p = '\0';
5109	rtx fun = init_one_libfunc (libfunc_name);
5110	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5111	rtx val = emit_library_call_value (fun, value: target, fn_type: LCT_PURE, outmode: mode,
5112	arg1: arg0, arg1_mode: ptr_mode, arg2: arg1, SImode);
5113	if (val != target)
5114	emit_move_insn (target, val);
5115	}
5116